From 1597d3a8b395815781ae1f7547d0843623f147da Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Tue, 2 Jun 2026 07:04:14 +0000 Subject: [PATCH] Add batch inference for sweeping many models over many datasets `mmeval/run.py` evaluates a single model on a single dataset. This adds a thin scheduler on top of it so a whole set of (model x dataset) combinations can run from one command, with GPUs allocated automatically from each model's estimated VRAM. The framework's single-task path is untouched: `mmeval/run.py`, `mmeval/registry.py`, and `mmeval/utils/argparser.py` are unchanged. The feature is purely additive. - mmeval/batch_infer.py: orchestrator. Selects models (--models / --model-series, with type/quantization/api/modality filters and --sort-by), expands dataset specs (mmeval_hf@/evalkit@/local@/tsv@), and invokes run.py once per task. Estimates VRAM, picks the fewest GPUs that fit each model and fills the rest with parallel shards, skips models that don't fit, and prints a summary. Unrecognized flags are forwarded to run.py, so all generation and inference options work unchanged. --list-models previews the selection. - mmeval/utils/gpu.py: GPU discovery (CUDA_VISIBLE_DEVICES / nvidia-smi, incl. MIG), a parameter+KV-cache VRAM estimate, and the allocation policy. - mmeval/model_metadata.json: companion to registry.py, a JSON object keyed by model name adding the size/architecture/modality fields the scheduler needs (registry.py stays the source of truth for which models exist and how to run them). - scripts/update_model_metadata.py: refreshes the auto-derived metadata fields from HuggingFace and cross-checks the file against registry.py. - docs: a Batch Inference section in USAGE.md and a README feature note. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 2 +- docs/en/USAGE.md | 112 +++++++++ mmeval/batch_infer.py | 376 +++++++++++++++++++++++++++++++ mmeval/model_metadata.json | 249 ++++++++++++++++++++ mmeval/utils/gpu.py | 187 +++++++++++++++ scripts/update_model_metadata.py | 214 ++++++++++++++++++ 6 files changed, 1139 insertions(+), 1 deletion(-) create mode 100644 mmeval/batch_infer.py create mode 100644 mmeval/model_metadata.json create mode 100644 mmeval/utils/gpu.py create mode 100644 scripts/update_model_metadata.py diff --git a/README.md b/README.md index d22ecb9..bd5307c 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ - **Simple** -- Set up a conda environment and run inference with a single command; no complex configuration or pipeline assembly required - **Self-contained** -- Each model series has its own complete inference implementation in a single file and its own conda environment, eliminating dependency conflicts -- **Scalable** -- Automatic multi-GPU parallel scheduling with worker-based data sharding; lazy loading for memory-efficient processing of large datasets +- **Scalable** -- Automatic multi-GPU parallel scheduling with worker-based data sharding; lazy loading for memory-efficient processing of large datasets. Sweep many models over many datasets in one command with VRAM-aware GPU allocation via [batch inference](docs/en/USAGE.md#batch-inference) - **Modular** -- Pluggable dataset loaders (local JSON, HuggingFace, VLMEvalKit TSV), Jinja2 prompt templates, and a registry-based model system that can be extended independently ## Installation diff --git a/docs/en/USAGE.md b/docs/en/USAGE.md index eb1874d..6c48ad3 100644 --- a/docs/en/USAGE.md +++ b/docs/en/USAGE.md @@ -195,6 +195,118 @@ All arguments are passed to `python mmeval/run.py` and organized into the follow --- +## Batch Inference + +`mmeval/run.py` evaluates **one** model on **one** dataset. To sweep many +models over many datasets in a single command, use `mmeval/batch_infer.py`. It is +a thin scheduler on top of `run.py`: for each model it estimates peak VRAM, +decides how many GPUs a shard needs and how many shards can run in parallel, then +calls `run.py` once per dataset. + +```bash +# Two models on one HuggingFace benchmark, GPUs allocated automatically +python mmeval/batch_infer.py \ + --models Qwen2.5-VL-7B-Instruct InternVL3-8B \ + --datasets mmeval_hf@mm-eval/MMBench-en-V11:test \ + --out-dir work_dirs/batch +``` + +Results are written to `///result.json`, the same +per-run layout `run.py` produces. + +### Selecting models + +Choose models by name or by series (the two are mutually exclusive); pass `all` +to select everything. Selection is then narrowed by filters: + +```bash +# Every model in two series, newest first +python mmeval/batch_infer.py --model-series qwenvl2d5 internvl3 --sort-by newest ... + +# All instruct-tuned, non-quantized local models that support video +python mmeval/batch_infer.py --models all --modalities multi_video_interleave ... +``` + +| Filter | Default | Description | +|--------|---------|-------------| +| `--model-type` | `non-pretrain` | `pretrain`, `instruct`, `reasoning`, `non-pretrain`, or `all` | +| `--quantization` | `false` | `true`, `false`, or `all` | +| `--api` | `false` | `false` = local only, `true` = API only, `all` = both | +| `--modalities` | -- | keep models supporting at least one listed modality | +| `--sort-by` | registry order | `newest`, `oldest`, `largest`, `smallest` | + +Preview the resolved selection without running anything: + +```bash +python mmeval/batch_infer.py --model-series qwenvl2d5 --list-models +``` + +### Specifying datasets + +`--datasets` accepts one or more specs in the form `@`: + +| Spec | Expands to | +|------|------------| +| `mmeval_hf@org/name:split` | `--dataset mmeval_hf@org/name --split split` | +| `evalkit@Name` | `--dataset evalkit@Name` | +| `local@/path/file.json[:/img_dir]` | `--dataset local@json --infile … [--img_dir …]` | +| `tsv@name-or-path-or-url` | `--dataset name-or-path-or-url` | + +### GPU allocation + +Each model's VRAM is estimated from its size and architecture (recorded in +`mmeval/model_metadata.json`). A model is given the fewest GPUs that fit it, and +the remaining GPUs run additional shards in parallel; a model that cannot fit on +the available GPUs is **skipped** (reported in the summary). Override the +detected per-GPU VRAM — or disable specific GPUs — with `--gpu-memory` (values in +GiB, in visible-GPU order, `0` to disable): + +```bash +# 80GiB on GPU0/GPU1, GPU2 disabled, 40GiB on GPU3 +python mmeval/batch_infer.py ... --gpu-memory 80 80 0 40 +``` + +API models need no GPU; control their concurrency with `--api-parallel`. + +### Forwarding generation/inference options + +Any flag `batch_infer` does not recognize is forwarded verbatim to every `run.py` +invocation, so all [generation](#generation-arguments) and +[inference](#inference-arguments) arguments work unchanged: + +```bash +python mmeval/batch_infer.py --models all --datasets evalkit@MMBench_dev_en \ + --out-dir work_dirs/batch --max_new_tokens 256 --sample_num 50 --circular +``` + +(The per-task `--model_name_or_path`, `--dataset`, `--out_dir`, +`--gpu_per_parallel`, and `--parallel_per_task` are managed by `batch_infer` and +cannot be forwarded.) Add `--save-log` to write a JSON run log to `--out-dir`. + +### Maintaining the model metadata + +`mmeval/model_metadata.json` is a companion to `mmeval/registry.py`: `registry.py` +remains the source of truth for which models exist and which environment runs +them, while this file adds the size/architecture/modality fields the scheduler +needs. It is a JSON object keyed by model name; for example: + +```json +"Qwen2.5-VL-7B-Instruct": {"hf_path": "Qwen/Qwen2.5-VL-7B-Instruct", "series": "qwenvl2d5", + "model_type": "instruct", "api_model": false, "quantization": false, + "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8292166656, + "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, + "num_key_value_heads": 4, "max_position_embeddings": 128000, "required_gpu_count": 1} +``` + +After adding models to `registry.py`, refresh the auto-derived fields (parameter +count, architecture, etc.) and check the two stay in sync with: + +```bash +HF_TOKEN=hf_xxx python scripts/update_model_metadata.py +``` + +--- + ## Prompt Template System Simple-MMEval uses [Jinja2](https://jinja.palletsprojects.com/) templates to construct prompts from structured sample data. Templates are resolved in the following priority order: diff --git a/mmeval/batch_infer.py b/mmeval/batch_infer.py new file mode 100644 index 0000000..3a63992 --- /dev/null +++ b/mmeval/batch_infer.py @@ -0,0 +1,376 @@ +"""Batch inference: run many (model x dataset) combinations in one command. + +This is a thin scheduler on top of ``mmeval/run.py``. For each selected model it +estimates peak VRAM, decides how many GPUs a shard needs and how many shards can +run in parallel, then invokes ``mmeval/run.py`` for every dataset. Model metadata +(used for VRAM sizing and filtering) lives in ``mmeval/model_metadata.json``; the +model -> series/env resolution stays in ``mmeval/registry.py`` and is handled by +``run.py`` itself. + +Examples: + # Two models on one HF benchmark, auto GPU allocation + python mmeval/batch_infer.py \\ + --models Qwen2.5-VL-7B-Instruct InternVL3-8B \\ + --datasets mmeval_hf@mm-eval/MMBench-en-V11:test \\ + --out-dir work_dirs/batch + + # A whole series over several datasets, forwarding gen kwargs to run.py + python mmeval/batch_infer.py \\ + --model-series qwenvl2d5 \\ + --datasets evalkit@LLaVABench local@/data/q.json:/data/imgs \\ + --out-dir work_dirs/batch --max_new_tokens 256 +""" + +import argparse +import json +import os +import subprocess +import sys +import time +from collections import Counter +from datetime import datetime +from pathlib import Path + +from mmeval.utils.gpu import ( + estimate_max_tokens, + estimate_vram_gb, + enabled_gpu_indices, + get_gpu_indices, + gpu_allocation, + min_gpu_vram, + parse_gpu_memory, +) + +BASE_DIR = Path(__file__).resolve().parent.parent +METADATA_JSON = Path(__file__).with_name("model_metadata.json") +RUN_SCRIPT = Path(__file__).with_name("run.py") + +# run.py flags that batch_infer sets per task; users must not pass them through. +MANAGED_FLAGS = { + "model_name_or_path", "dataset", "split", "infile", "img_dir", + "out_dir", "gpu_per_parallel", "parallel_per_task", "rank", +} + + +# --------------------------------------------------------------------------- # +# Model metadata +# --------------------------------------------------------------------------- # +def load_models() -> list[dict]: + """Load the metadata JSON as a list of model dicts (model name folded in).""" + with METADATA_JSON.open(encoding="utf-8") as f: + data = json.load(f) + return [{"model_name": name, **fields} for name, fields in data.items()] + + +def model_vram_gb(model: dict) -> float: + """Estimated peak VRAM (GiB) for one shard; API models need none.""" + if model.get("api_model"): + return 0.0 + return estimate_vram_gb(model, estimate_max_tokens(model)) + + +def filter_models(models: list[dict], args) -> list[dict]: + """Apply --model-type / --quantization / --api / --modalities filters.""" + result = [] + for m in models: + if args.model_type == "non-pretrain": + if m["model_type"] == "pretrain": + continue + elif args.model_type != "all" and m["model_type"] != args.model_type: + continue + if args.quantization != "all" and m["quantization"] != (args.quantization == "true"): + continue + if args.api != "all" and m["api_model"] != (args.api == "true"): + continue + if args.modalities and not set(m["modalities"]).intersection(args.modalities): + continue + result.append(m) + return result + + +def sort_models(models: list[dict], key: str | None) -> list[dict]: + """Order models for scheduling. None keeps registry order.""" + if key in ("newest", "oldest"): + return sorted(models, key=lambda m: m.get("created_at") or "0", reverse=(key == "newest")) + if key in ("largest", "smallest"): + def params(m): + return m.get("parameter_count") or 0 + return sorted(models, key=params, reverse=(key == "largest")) + return models + + +def select_models(args, all_models: list[dict]) -> list[dict]: + """Resolve --models / --model-series into a concrete model list, then filter.""" + by_name = {m["model_name"]: m for m in all_models} + + if args.models: + if [s.lower() for s in args.models] == ["all"]: + selected = list(all_models) + else: + unknown = [n for n in args.models if n not in by_name] + if unknown: + sys.exit(f"Unknown model(s): {', '.join(unknown)} " + f"(use --list-models to inspect the registry).") + selected = [by_name[n] for n in args.models] + else: + if [s.lower() for s in args.model_series] == ["all"]: + selected = list(all_models) + else: + wanted = set(args.model_series) + unknown = sorted(wanted.difference(m["series"] for m in all_models)) + if unknown: + sys.exit(f"Unknown model series: {', '.join(unknown)}") + selected = [m for m in all_models if m["series"] in wanted] + + filtered = sort_models(filter_models(selected, args), args.sort_by) + if not filtered and selected: + print(f"Note: all {len(selected)} selected model(s) were removed by filters " + f"(--model-type={args.model_type} --quantization={args.quantization} --api={args.api}" + + (f" --modalities={' '.join(args.modalities)}" if args.modalities else "") + ").") + return filtered + + +# --------------------------------------------------------------------------- # +# Dataset specs +# --------------------------------------------------------------------------- # +def parse_dataset_spec(spec: str) -> tuple[list[str], str]: + """Translate a dataset spec into (run.py args, short label). + + Supported forms: + mmeval_hf@org/name:split HuggingFace mm-eval dataset (split required) + evalkit@Name VLMEvalKit dataset by name + local@/path/file.json[:/img_dir] local JSON, optional image dir + tsv@name-or-path-or-url raw --dataset value (.tsv name, path, or URL) + """ + spec = spec.strip() + dtype, sep, rest = spec.partition("@") + if not sep or not rest: + sys.exit(f"Invalid dataset spec '{spec}'. Expected '@', " + f"e.g. mmeval_hf@mm-eval/MMBench-en-V11:test") + + if dtype == "mmeval_hf": + if ":" not in rest: + sys.exit(f"mmeval_hf spec needs a ':' suffix, got '{spec}' " + f"(e.g. mmeval_hf@mm-eval/MMBench-en-V11:test).") + path, split = rest.rsplit(":", 1) + return ["--dataset", f"mmeval_hf@{path}", "--split", split], f"{path.split('/')[-1]}_{split}" + + if dtype == "evalkit": + return ["--dataset", f"evalkit@{rest}"], rest + + if dtype == "local": + if ":" in rest: + infile, img_dir = rest.rsplit(":", 1) + return ["--dataset", "local@json", "--infile", infile, "--img_dir", img_dir], Path(infile).stem + return ["--dataset", "local@json", "--infile", rest], Path(rest).stem + + if dtype == "tsv": + return ["--dataset", rest], Path(rest).stem + + sys.exit(f"Unknown dataset type '{dtype}' in '{spec}'. " + f"Use one of: mmeval_hf, evalkit, local, tsv.") + + +# --------------------------------------------------------------------------- # +# Execution +# --------------------------------------------------------------------------- # +def run_task(model: dict, ds_args: list[str], label: str, out_base: Path, + enabled_gpus: list[str], gpu_vram_gb: float, passthrough: list[str], + api_parallel: int) -> dict: + """Run one (model, dataset) task via mmeval/run.py.""" + name = model["model_name"] + is_api = bool(model.get("api_model")) + + if is_api: + vram, alloc = 0.0, {"can_run": True, "gpu_per_parallel": 0, + "parallel_per_task": max(1, api_parallel)} + else: + vram = model_vram_gb(model) + alloc = gpu_allocation(len(enabled_gpus), vram, gpu_vram_gb) + + base = {"model": name, "series": model["series"], "dataset": label, + "estimated_vram_gb": round(vram, 1)} + if not alloc["can_run"]: + print(f" skipped: {alloc['reason']}") + return {**base, "status": "skipped", "reason": alloc["reason"], "duration": 0.0} + + out_dir = out_base / label / name + out_dir.mkdir(parents=True, exist_ok=True) + + cmd = [ + sys.executable, str(RUN_SCRIPT), + "--model_name_or_path", model.get("hf_path") or name, + *ds_args, + "--out_dir", str(out_dir), + "--gpu_per_parallel", str(alloc["gpu_per_parallel"]), + "--parallel_per_task", str(alloc["parallel_per_task"]), + *passthrough, + ] + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join(filter(None, [str(BASE_DIR), env.get("PYTHONPATH", "")])) + # Constrain run.py to exactly the GPUs we accounted for (honors --gpu-memory). + if not is_api: + env["CUDA_VISIBLE_DEVICES"] = ",".join(enabled_gpus) + + start = time.time() + status = "error" + try: + status = "success" if subprocess.run(cmd, cwd=str(BASE_DIR), env=env).returncode == 0 else "failed" + except Exception as exc: # pragma: no cover - defensive + print(f" error: {exc}") + duration = time.time() - start + print(f" {status} ({duration:.0f}s)") + return {**base, "status": status, + "gpu_per_parallel": alloc["gpu_per_parallel"], + "parallel_per_task": alloc["parallel_per_task"], + "duration": round(duration, 1)} + + +def list_models(models: list[dict]) -> None: + """Print matching models grouped by series.""" + by_series: dict[str, list[dict]] = {} + for m in models: + by_series.setdefault(m["series"], []).append(m) + print(f"\nMatching models: {len(models)}\n") + for series in sorted(by_series): + print(f"{series}:") + for m in by_series[series]: + params = m.get("parameter_count") or 0 + p_str = f"{params / 1e9:.1f}B" if params > 0 else "" + quant = "Q" if m.get("quantization") else " " + api = "API" if m.get("api_model") else " " + print(f" {m['model_name']:<48} {model_vram_gb(m):>4.0f}GB {m.get('model_type', '?'):<10} " + f"{quant} {api} {p_str}") + print() + + +def print_summary(results: list[dict]) -> None: + """Print a one-line tally plus details for non-successful tasks.""" + counts = Counter(r["status"] for r in results) + total_time = sum(r.get("duration", 0) for r in results) + print(f"\n{'=' * 60}") + print(f"Batch inference summary: {len(results)} task(s) " + f"✅ {counts['success']} ❌ {counts['failed']} " + f"⏭️ {counts['skipped']} 💥 {counts['error']} " + f"({total_time:.0f}s / {total_time / 60:.1f}m)") + for r in results: + if r["status"] in ("failed", "error"): + print(f" ❌ {r['model']} ({r['dataset']}): {r['status']}") + elif r["status"] == "skipped": + print(f" ⏭️ {r['model']} ({r['dataset']}): {r.get('reason', '')}") + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="batch_infer", + description="Run inference over many (model x dataset) combinations, " + "scheduling GPUs automatically from estimated VRAM.", + formatter_class=argparse.RawDescriptionHelpFormatter, + # Disable prefix matching so run.py flags (e.g. --dataset, a prefix of + # --datasets) fall through to passthrough instead of being swallowed. + allow_abbrev=False, + ) + sel = p.add_mutually_exclusive_group() + sel.add_argument("--models", nargs="+", metavar="NAME", + help="Model name(s) from the registry, or 'all'. " + "Mutually exclusive with --model-series.") + sel.add_argument("--model-series", nargs="+", metavar="SERIES", + help="Model series name(s), or 'all'. " + "Mutually exclusive with --models.") + p.add_argument("--datasets", nargs="+", metavar="SPEC", + help="Dataset spec(s): mmeval_hf@org/name:split | evalkit@Name | " + "local@/path.json[:/img_dir] | tsv@name-or-path-or-url.") + p.add_argument("--out-dir", "--out_dir", dest="out_dir", metavar="DIR", + help="Base output directory; results go to ///.") + p.add_argument("--model-type", default="non-pretrain", + choices=["pretrain", "instruct", "reasoning", "non-pretrain", "all"], + help="Filter models by type (default: non-pretrain).") + p.add_argument("--quantization", default="false", choices=["true", "false", "all"], + help="Filter by quantization (default: false).") + p.add_argument("--api", default="false", choices=["true", "false", "all"], + help="Select API vs local models (default: false = local only).") + p.add_argument("--modalities", nargs="+", metavar="MODALITY", + help="Keep models supporting at least one of these modalities.") + p.add_argument("--sort-by", choices=["newest", "oldest", "largest", "smallest"], + help="Order models for scheduling (default: registry order).") + p.add_argument("--gpu-memory", nargs="+", type=float, metavar="GIB", + help="Per-GPU VRAM in GiB, in visible-GPU order; 0 disables a GPU. " + "Default: auto-detected. Example: --gpu-memory 80 80 0 40") + p.add_argument("--api-parallel", type=int, default=1, metavar="N", + help="Concurrent shards for API models (no GPU needed; default: 1).") + p.add_argument("--list-models", action="store_true", + help="Print the resolved model list and exit.") + p.add_argument("--save-log", action="store_true", + help="Write a JSON run log (args + per-task results) into --out-dir.") + return p + + +def main() -> int: + args, passthrough = build_parser().parse_known_args() + + bad = [tok for tok in passthrough if tok.startswith("--") + and tok.lstrip("-").split("=", 1)[0].replace("-", "_") in MANAGED_FLAGS] + if bad: + sys.exit(f"These flags are managed by batch_infer and cannot be passed through: " + f"{', '.join(sorted(set(bad)))}.") + + all_models = load_models() + if not (args.models or args.model_series): + if args.list_models: + args.models = ["all"] + else: + sys.exit("Select models with --models or --model-series " + "(or use --list-models to inspect the registry).") + + models = select_models(args, all_models) + if not models: + sys.exit("No models match the given selection/filters.") + + if args.list_models: + list_models(models) + return 0 + + if not args.datasets: + sys.exit("--datasets is required (e.g. --datasets mmeval_hf@mm-eval/MMBench-en-V11:test).") + if not args.out_dir: + sys.exit("--out-dir is required.") + + datasets = [parse_dataset_spec(s) for s in args.datasets] + out_base = Path(args.out_dir) + out_base.mkdir(parents=True, exist_ok=True) + + visible = get_gpu_indices() + gpu_memory = parse_gpu_memory(args.gpu_memory, len(visible)) + enabled = enabled_gpu_indices(visible, gpu_memory) + vram_gb = min_gpu_vram(gpu_memory) + + tasks = [(m, ds_args, label) for ds_args, label in datasets for m in models] + print(f"Batch inference: {len(tasks)} task(s) — {len(models)} model(s) x {len(datasets)} dataset(s)") + if gpu_memory: + mem = ", ".join(f"GPU{i}:{v:.0f}GiB" for i, v in enumerate(gpu_memory)) + print(f"GPUs: {len(enabled)}/{len(visible)} enabled (min VRAM {vram_gb:.0f}GiB) [{mem}]") + else: + print("GPUs: none visible (API models only)") + print(f"Output: {out_base}\n") + + results = [] + for i, (model, ds_args, label) in enumerate(tasks, 1): + print(f"[{i}/{len(tasks)}] {model['model_name']} ({label})") + results.append(run_task(model, ds_args, label, out_base, enabled, + vram_gb, passthrough, args.api_parallel)) + print() + + print_summary(results) + + if args.save_log: + log_path = out_base / f"batch_log_{datetime.now():%Y%m%d_%H%M%S}.json" + with log_path.open("w", encoding="utf-8") as f: + json.dump({"args": vars(args), "passthrough": passthrough, "results": results}, f, indent=2) + print(f"\nLog saved to {log_path}") + + failed = sum(r["status"] in ("failed", "error") for r in results) + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/mmeval/model_metadata.json b/mmeval/model_metadata.json new file mode 100644 index 0000000..0ce6186 --- /dev/null +++ b/mmeval/model_metadata.json @@ -0,0 +1,249 @@ +{ + "Bunny-Llama-3-8B-V": {"hf_path": "BAAI/Bunny-Llama-3-8B-V", "series": "bunnyllama3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 8479990848, "created_at": "2024-04-21", "torch_dtype": "float16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 8192, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Cosmos-Reason2-2B": {"hf_path": "nvidia/Cosmos-Reason2-2B", "series": "cosmos_reason2", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2438696960, "created_at": "2025-12-12", "torch_dtype": null, "num_hidden_layers": 28, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Cosmos-Reason2-8B": {"hf_path": "nvidia/Cosmos-Reason2-8B", "series": "cosmos_reason2", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8767123696, "created_at": "2025-12-12", "torch_dtype": null, "num_hidden_layers": 36, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 2}, + "InternVL-Chat-V1-1": {"hf_path": "OpenGVLab/InternVL-Chat-V1-1", "series": "internvl_chat", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "single_video_start", "text"], "parameter_count": null, "created_at": "2024-01-24", "torch_dtype": "bfloat16", "num_hidden_layers": 40, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 40, "max_position_embeddings": 4096, "image_tokens": 1024, "max_dynamic_patch": 1, "required_gpu_count": 1}, + "InternVL-Chat-V1-2": {"hf_path": "OpenGVLab/InternVL-Chat-V1-2", "series": "internvl_chat", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "single_video_start", "text"], "parameter_count": 40068684160, "created_at": "2024-02-11", "torch_dtype": "bfloat16", "num_hidden_layers": 60, "hidden_size": 7168, "num_attention_heads": 56, "num_key_value_heads": 8, "max_position_embeddings": 4096, "image_tokens": 1024, "max_dynamic_patch": 1, "required_gpu_count": 1}, + "InternVL-Chat-V1-2-Plus": {"hf_path": "OpenGVLab/InternVL-Chat-V1-2-Plus", "series": "internvl_chat", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "single_video_start", "text"], "parameter_count": 40068684160, "created_at": "2024-02-21", "torch_dtype": "bfloat16", "num_hidden_layers": 60, "hidden_size": 7168, "num_attention_heads": 56, "num_key_value_heads": 8, "max_position_embeddings": 4096, "image_tokens": 1024, "max_dynamic_patch": 1, "required_gpu_count": 1}, + "InternVL-Chat-V1-5": {"hf_path": "OpenGVLab/InternVL-Chat-V1-5", "series": "internvl_chat1d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "single_video_start", "text"], "parameter_count": 25514186112, "created_at": "2024-04-18", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 6144, "num_attention_heads": 48, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL2-1B": {"hf_path": "OpenGVLab/InternVL2-1B", "series": "internvl2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 938158976, "created_at": "2024-07-08", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 896, "num_attention_heads": 14, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2-26B": {"hf_path": "OpenGVLab/InternVL2-26B", "series": "internvl2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 25514186112, "created_at": "2024-06-27", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 6144, "num_attention_heads": 48, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL2-2B": {"hf_path": "OpenGVLab/InternVL2-2B", "series": "internvl2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2205754368, "created_at": "2024-06-27", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2-40B": {"hf_path": "OpenGVLab/InternVL2-40B", "series": "internvl2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 40068684160, "created_at": "2024-06-27", "torch_dtype": "bfloat16", "num_hidden_layers": 60, "hidden_size": 7168, "num_attention_heads": 56, "num_key_value_heads": 8, "max_position_embeddings": 8192, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL2-4B": {"hf_path": "OpenGVLab/InternVL2-4B", "series": "internvl2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 4146855936, "created_at": "2024-06-27", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 3072, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": 131072, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2-8B": {"hf_path": "OpenGVLab/InternVL2-8B", "series": "internvl2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8075365376, "created_at": "2024-06-27", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2-Llama3-76B": {"hf_path": "OpenGVLab/InternVL2-Llama3-76B", "series": "internvl2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 76262358400, "created_at": "2024-07-15", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 8192, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 4}, + "InternVL2_5-1B": {"hf_path": "OpenGVLab/InternVL2_5-1B", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 938193024, "created_at": "2024-11-20", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 896, "num_attention_heads": 14, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2_5-1B-MPO": {"hf_path": "OpenGVLab/InternVL2_5-1B-MPO", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 938193024, "created_at": "2024-12-20", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 896, "num_attention_heads": 14, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2_5-26B": {"hf_path": "OpenGVLab/InternVL2_5-26B", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 25514186112, "created_at": "2024-11-20", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 6144, "num_attention_heads": 48, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL2_5-26B-MPO": {"hf_path": "OpenGVLab/InternVL2_5-26B-MPO", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 25514186112, "created_at": "2024-12-20", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 6144, "num_attention_heads": 48, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL2_5-2B": {"hf_path": "OpenGVLab/InternVL2_5-2B", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2205754368, "created_at": "2024-11-20", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2_5-2B-MPO": {"hf_path": "OpenGVLab/InternVL2_5-2B-MPO", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2205754368, "created_at": "2024-12-20", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2_5-38B": {"hf_path": "OpenGVLab/InternVL2_5-38B", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 38388164992, "created_at": "2024-11-20", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL2_5-38B-MPO": {"hf_path": "OpenGVLab/InternVL2_5-38B-MPO", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 38388164992, "created_at": "2024-12-20", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL2_5-4B": {"hf_path": "OpenGVLab/InternVL2_5-4B", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 3712637952, "created_at": "2024-11-20", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2_5-4B-MPO": {"hf_path": "OpenGVLab/InternVL2_5-4B-MPO", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 3712637952, "created_at": "2024-12-20", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2_5-78B": {"hf_path": "OpenGVLab/InternVL2_5-78B", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 78408318336, "created_at": "2024-12-02", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 4}, + "InternVL2_5-78B-MPO": {"hf_path": "OpenGVLab/InternVL2_5-78B-MPO", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 78408318336, "created_at": "2024-12-20", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 4}, + "InternVL2_5-8B": {"hf_path": "OpenGVLab/InternVL2_5-8B", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8075365376, "created_at": "2024-11-20", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL2_5-8B-MPO": {"hf_path": "OpenGVLab/InternVL2_5-8B-MPO", "series": "internvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8075365376, "created_at": "2024-12-20", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-14B": {"hf_path": "OpenGVLab/InternVL3-14B", "series": "internvl3", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 15117256704, "created_at": "2025-04-10", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3-14B-Instruct": {"hf_path": "OpenGVLab/InternVL3-14B-Instruct", "series": "internvl3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 15117256704, "created_at": "2025-04-16", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3-14B-Pretrained": {"hf_path": "OpenGVLab/InternVL3-14B-Pretrained", "series": "internvl3", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 15117256704, "created_at": "2025-04-17", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3-1B": {"hf_path": "OpenGVLab/InternVL3-1B", "series": "internvl3", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 938193024, "created_at": "2025-04-10", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 896, "num_attention_heads": 14, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-1B-Instruct": {"hf_path": "OpenGVLab/InternVL3-1B-Instruct", "series": "internvl3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 938193024, "created_at": "2025-04-16", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 896, "num_attention_heads": 14, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-1B-Pretrained": {"hf_path": "OpenGVLab/InternVL3-1B-Pretrained", "series": "internvl3", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 938193024, "created_at": "2025-04-17", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 896, "num_attention_heads": 14, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-2B": {"hf_path": "OpenGVLab/InternVL3-2B", "series": "internvl3", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2088957440, "created_at": "2025-04-10", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 1536, "num_attention_heads": 12, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-2B-Instruct": {"hf_path": "OpenGVLab/InternVL3-2B-Instruct", "series": "internvl3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2088957440, "created_at": "2025-04-16", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 1536, "num_attention_heads": 12, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-2B-Pretrained": {"hf_path": "OpenGVLab/InternVL3-2B-Pretrained", "series": "internvl3", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2088957440, "created_at": "2025-04-17", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 1536, "num_attention_heads": 12, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-38B": {"hf_path": "OpenGVLab/InternVL3-38B", "series": "internvl3", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 38388164992, "created_at": "2025-04-10", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3-38B-Instruct": {"hf_path": "OpenGVLab/InternVL3-38B-Instruct", "series": "internvl3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 38388164992, "created_at": "2025-04-16", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3-38B-Pretrained": {"hf_path": "OpenGVLab/InternVL3-38B-Pretrained", "series": "internvl3", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 38388164992, "created_at": "2025-04-17", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3-78B": {"hf_path": "OpenGVLab/InternVL3-78B", "series": "internvl3", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 78408318336, "created_at": "2025-04-10", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 4}, + "InternVL3-78B-Instruct": {"hf_path": "OpenGVLab/InternVL3-78B-Instruct", "series": "internvl3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 78408318336, "created_at": "2025-04-16", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 4}, + "InternVL3-78B-Pretrained": {"hf_path": "OpenGVLab/InternVL3-78B-Pretrained", "series": "internvl3", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 78408318336, "created_at": "2025-04-17", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 4}, + "InternVL3-8B": {"hf_path": "OpenGVLab/InternVL3-8B", "series": "internvl3", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 7944373760, "created_at": "2025-04-10", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-8B-Instruct": {"hf_path": "OpenGVLab/InternVL3-8B-Instruct", "series": "internvl3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 7944373760, "created_at": "2025-04-16", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-8B-Pretrained": {"hf_path": "OpenGVLab/InternVL3-8B-Pretrained", "series": "internvl3", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 7944373760, "created_at": "2025-04-17", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-9B": {"hf_path": "OpenGVLab/InternVL3-9B", "series": "internvl3", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 9138793472, "created_at": "2025-04-10", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-9B-Instruct": {"hf_path": "OpenGVLab/InternVL3-9B-Instruct", "series": "internvl3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 9138793472, "created_at": "2025-04-16", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3-9B-Pretrained": {"hf_path": "OpenGVLab/InternVL3-9B-Pretrained", "series": "internvl3", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 9138793472, "created_at": "2025-04-17", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-14B": {"hf_path": "OpenGVLab/InternVL3_5-14B", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 15119523840, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 40, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-14B-Instruct": {"hf_path": "OpenGVLab/InternVL3_5-14B-Instruct", "series": "internvl3d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 15119523840, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 40, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-14B-MPO": {"hf_path": "OpenGVLab/InternVL3_5-14B-MPO", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 15119523840, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 40, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-14B-Pretrained": {"hf_path": "OpenGVLab/InternVL3_5-14B-Pretrained", "series": "internvl3d5", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 15119523840, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 40, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-1B": {"hf_path": "OpenGVLab/InternVL3_5-1B", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 1060897792, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 1024, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-1B-Instruct": {"hf_path": "OpenGVLab/InternVL3_5-1B-Instruct", "series": "internvl3d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 1060897792, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 1024, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-1B-MPO": {"hf_path": "OpenGVLab/InternVL3_5-1B-MPO", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 1060897792, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 1024, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-1B-Pretrained": {"hf_path": "OpenGVLab/InternVL3_5-1B-Pretrained", "series": "internvl3d5", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 1060897792, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 1024, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-241B-A28B": {"hf_path": "OpenGVLab/InternVL3_5-241B-A28B", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 240699370368, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 94, "hidden_size": 4096, "num_attention_heads": 64, "num_key_value_heads": 4, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 8}, + "InternVL3_5-241B-A28B-Instruct": {"hf_path": "OpenGVLab/InternVL3_5-241B-A28B-Instruct", "series": "internvl3d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 240699370368, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 94, "hidden_size": 4096, "num_attention_heads": 64, "num_key_value_heads": 4, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 8}, + "InternVL3_5-241B-A28B-MPO": {"hf_path": "OpenGVLab/InternVL3_5-241B-A28B-MPO", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 240699370368, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 94, "hidden_size": 4096, "num_attention_heads": 64, "num_key_value_heads": 4, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 8}, + "InternVL3_5-241B-A28B-Pretrained": {"hf_path": "OpenGVLab/InternVL3_5-241B-A28B-Pretrained", "series": "internvl3d5", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 240699370368, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 94, "hidden_size": 4096, "num_attention_heads": 64, "num_key_value_heads": 4, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 8}, + "InternVL3_5-2B": {"hf_path": "OpenGVLab/InternVL3_5-2B", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2348347392, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-2B-Instruct": {"hf_path": "OpenGVLab/InternVL3_5-2B-Instruct", "series": "internvl3d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2348347392, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-2B-MPO": {"hf_path": "OpenGVLab/InternVL3_5-2B-MPO", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2348347392, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-2B-Pretrained": {"hf_path": "OpenGVLab/InternVL3_5-2B-Pretrained", "series": "internvl3d5", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2348347392, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-30B-A3B": {"hf_path": "OpenGVLab/InternVL3_5-30B-A3B", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 30848730112, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 2048, "num_attention_heads": 32, "num_key_value_heads": 4, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3_5-30B-A3B-Instruct": {"hf_path": "OpenGVLab/InternVL3_5-30B-A3B-Instruct", "series": "internvl3d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 30848730112, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 2048, "num_attention_heads": 32, "num_key_value_heads": 4, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3_5-30B-A3B-MPO": {"hf_path": "OpenGVLab/InternVL3_5-30B-A3B-MPO", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 30848730112, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 2048, "num_attention_heads": 32, "num_key_value_heads": 4, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3_5-30B-A3B-Pretrained": {"hf_path": "OpenGVLab/InternVL3_5-30B-A3B-Pretrained", "series": "internvl3d5", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 30848730112, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 2048, "num_attention_heads": 32, "num_key_value_heads": 4, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3_5-38B": {"hf_path": "OpenGVLab/InternVL3_5-38B", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 38390405504, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3_5-38B-Instruct": {"hf_path": "OpenGVLab/InternVL3_5-38B-Instruct", "series": "internvl3d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 38390405504, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3_5-38B-MPO": {"hf_path": "OpenGVLab/InternVL3_5-38B-MPO", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 38390405504, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3_5-38B-Pretrained": {"hf_path": "OpenGVLab/InternVL3_5-38B-Pretrained", "series": "internvl3d5", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 38390405504, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "InternVL3_5-4B": {"hf_path": "OpenGVLab/InternVL3_5-4B", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 4732489216, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 2560, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-4B-Instruct": {"hf_path": "OpenGVLab/InternVL3_5-4B-Instruct", "series": "internvl3d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 4732489216, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 2560, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-4B-MPO": {"hf_path": "OpenGVLab/InternVL3_5-4B-MPO", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 4732489216, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 2560, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-4B-Pretrained": {"hf_path": "OpenGVLab/InternVL3_5-4B-Pretrained", "series": "internvl3d5", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 4732489216, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 2560, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-8B": {"hf_path": "OpenGVLab/InternVL3_5-8B", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8528318464, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-8B-Instruct": {"hf_path": "OpenGVLab/InternVL3_5-8B-Instruct", "series": "internvl3d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8528318464, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-8B-MPO": {"hf_path": "OpenGVLab/InternVL3_5-8B-MPO", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8528318464, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-8B-Pretrained": {"hf_path": "OpenGVLab/InternVL3_5-8B-Pretrained", "series": "internvl3d5", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8528318464, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "InternVL3_5-GPT-OSS-20B-A4B-Preview": {"hf_path": "OpenGVLab/InternVL3_5-GPT-OSS-20B-A4B-Preview", "series": "internvl3d5", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 392282304, "created_at": "2025-08-25", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 2880, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 131072, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 2}, + "Janus-1.3B": {"hf_path": "deepseek-ai/Janus-1.3B", "series": "janus", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 2089297547, "created_at": "2024-10-18", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Janus-Pro-1B": {"hf_path": "deepseek-ai/Janus-Pro-1B", "series": "janus_pro", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": "2025-01-26", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Janus-Pro-7B": {"hf_path": "deepseek-ai/Janus-Pro-7B", "series": "janus_pro", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": "2025-01-26", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "JanusFlow-1.3B": {"hf_path": "deepseek-ai/JanusFlow-1.3B", "series": "janus_flow", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 2046367504, "created_at": "2024-11-12", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "LLaVA-OneVision-1.5-8B-Instruct": {"hf_path": "lmms-lab/LLaVA-OneVision-1.5-8B-Instruct", "series": "llava_ov_1d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_video_start", "text"], "parameter_count": 8527214624, "created_at": "2025-09-16", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Llama-3.2-11B-Vision-Instruct": {"hf_path": "meta-llama/Llama-3.2-11B-Vision-Instruct", "series": "llama3d2_vision", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 10670220835, "created_at": "2024-09-18", "torch_dtype": "bfloat16", "num_hidden_layers": 40, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 131072, "image_tokens": 1600, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Llama-3.2-90B-Vision-Instruct": {"hf_path": "meta-llama/Llama-3.2-90B-Vision-Instruct", "series": "llama3d2_vision", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 88593355323, "created_at": "2024-09-19", "torch_dtype": "bfloat16", "num_hidden_layers": 100, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 131072, "image_tokens": 1600, "max_dynamic_patch": null, "required_gpu_count": 4}, + "Llama-4-Maverick-17B-128E": {"hf_path": "meta-llama/Llama-4-Maverick-17B-128E", "series": "llama4", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 401583781376, "created_at": "2025-04-02", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 8}, + "Llama-4-Maverick-17B-128E-Instruct": {"hf_path": "meta-llama/Llama-4-Maverick-17B-128E-Instruct", "series": "llama4", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 401583781376, "created_at": "2025-04-01", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 1048576, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 8}, + "Llama-4-Scout-17B-16E": {"hf_path": "meta-llama/Llama-4-Scout-17B-16E", "series": "llama4", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 108641793536, "created_at": "2025-04-02", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 4}, + "Llama-4-Scout-17B-16E-Instruct": {"hf_path": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "series": "llama4", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 108641793536, "created_at": "2025-04-02", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 10485760, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 4}, + "Mantis-8B-Fuyu": {"hf_path": "TIGER-Lab/Mantis-8B-Fuyu", "series": "mantis_fuyu", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 9408254976, "created_at": "2024-05-03", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Mantis-8B-Idefics2": {"hf_path": "TIGER-Lab/Mantis-8B-Idefics2", "series": "mantis_idefics2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 8402768112, "created_at": "2024-05-23", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 4900, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Mantis-8B-clip-llama3": {"hf_path": "TIGER-Lab/Mantis-8B-clip-llama3", "series": "mantis", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 8354764800, "created_at": "2024-05-03", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": 8, "max_position_embeddings": 8192, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Mantis-8B-siglip-llama3": {"hf_path": "TIGER-Lab/Mantis-8B-siglip-llama3", "series": "mantis", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 8480007232, "created_at": "2024-05-03", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": 8, "max_position_embeddings": 8192, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Mantis-bakllava-7b": {"hf_path": "TIGER-Lab/Mantis-bakllava-7b", "series": "mantis_llava", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 7566743552, "created_at": "2024-04-12", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Mantis-llava-7b": {"hf_path": "TIGER-Lab/Mantis-llava-7b", "series": "mantis_llava", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 7063427072, "created_at": "2024-04-13", "torch_dtype": "float16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": 4096, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Mini-InternVL-Chat-2B-V1-5": {"hf_path": "OpenGVLab/Mini-InternVL-Chat-2B-V1-5", "series": "internvl_chat1d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "single_video_start", "text"], "parameter_count": 2205754368, "created_at": "2024-05-13", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "Mini-InternVL-Chat-4B-V1-5": {"hf_path": "OpenGVLab/Mini-InternVL-Chat-4B-V1-5", "series": "internvl_chat1d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "single_video_start", "text"], "parameter_count": 4146855936, "created_at": "2024-05-28", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 3072, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": 131072, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "Ovis1.5-Gemma2-9B": {"hf_path": "AIDC-AI/Ovis1.5-Gemma2-9B", "series": "ovis1d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 11359445052, "created_at": "2024-07-30", "torch_dtype": "bfloat16", "num_hidden_layers": 42, "hidden_size": 3584, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 8192, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Ovis1.5-Llama3-8B": {"hf_path": "AIDC-AI/Ovis1.5-Llama3-8B", "series": "ovis1d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": null, "created_at": "2024-07-24", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 8192, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Ovis1.6-Gemma2-27B": {"hf_path": "AIDC-AI/Ovis1.6-Gemma2-27B", "series": "ovis1d6_27b", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 28863552566, "created_at": "2024-11-26", "torch_dtype": "bfloat16", "num_hidden_layers": 46, "hidden_size": 4608, "num_attention_heads": 32, "num_key_value_heads": 16, "max_position_embeddings": 8192, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Ovis1.6-Gemma2-9B": {"hf_path": "AIDC-AI/Ovis1.6-Gemma2-9B", "series": "ovis1d6", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 10206910518, "created_at": "2024-09-18", "torch_dtype": "bfloat16", "num_hidden_layers": 42, "hidden_size": 3584, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 8192, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Ovis1.6-Llama3.2-3B": {"hf_path": "AIDC-AI/Ovis1.6-Llama3.2-3B", "series": "ovis1d6", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 4144399926, "created_at": "2024-10-16", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3072, "num_attention_heads": 24, "num_key_value_heads": 8, "max_position_embeddings": 131072, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Ovis2.5-2B": {"hf_path": "AIDC-AI/Ovis2.5-2B", "series": "ovis2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "single_video_start", "text"], "parameter_count": 2570429672, "created_at": "2025-08-15", "torch_dtype": "float32", "num_hidden_layers": 28, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Ovis2.5-9B": {"hf_path": "AIDC-AI/Ovis2.5-9B", "series": "ovis2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "single_video_start", "text"], "parameter_count": 9174807784, "created_at": "2025-08-15", "torch_dtype": "float32", "num_hidden_layers": 36, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 40960, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Phi-3-vision-128k-instruct": {"hf_path": "microsoft/Phi-3-vision-128k-instruct", "series": "phi3v", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "single_image_start", "text"], "parameter_count": 4146621440, "created_at": "2024-05-19", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 3072, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": 131072, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Phi-3.5-vision-instruct": {"hf_path": "microsoft/Phi-3.5-vision-instruct", "series": "phi3v", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "single_image_start", "text"], "parameter_count": 4146621440, "created_at": "2024-08-16", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 3072, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": 131072, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Phi-4-multimodal-instruct": {"hf_path": "microsoft/Phi-4-multimodal-instruct", "series": "phi4mm", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 5574460384, "created_at": "2025-02-24", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 3072, "num_attention_heads": 24, "num_key_value_heads": 8, "max_position_embeddings": 131072, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2-VL-2B-Instruct": {"hf_path": "Qwen/Qwen2-VL-2B-Instruct", "series": "qwenvl2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2208985600, "created_at": "2024-08-28", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 1536, "num_attention_heads": 12, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2-VL-2B-Instruct-AWQ": {"hf_path": "Qwen/Qwen2-VL-2B-Instruct-AWQ", "series": "qwenvl2", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2442359296, "created_at": "2024-08-29", "torch_dtype": "float16", "num_hidden_layers": 28, "hidden_size": 1536, "num_attention_heads": 12, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2-VL-2B-Instruct-GPTQ-Int4": {"hf_path": "Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4", "series": "qwenvl2", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2208985600, "created_at": "2024-08-29", "torch_dtype": "float16", "num_hidden_layers": 28, "hidden_size": 1536, "num_attention_heads": 12, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2-VL-72B-Instruct": {"hf_path": "Qwen/Qwen2-VL-72B-Instruct", "series": "qwenvl2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 73405560320, "created_at": "2024-09-17", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 4}, + "Qwen2-VL-72B-Instruct-AWQ": {"hf_path": "Qwen/Qwen2-VL-72B-Instruct-AWQ", "series": "qwenvl2", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 73657218560, "created_at": "2024-09-17", "torch_dtype": "float16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Qwen2-VL-72B-Instruct-GPTQ-Int4": {"hf_path": "Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4", "series": "qwenvl2", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 73657218560, "created_at": "2024-09-17", "torch_dtype": "float16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 4}, + "Qwen2-VL-7B-Instruct": {"hf_path": "Qwen/Qwen2-VL-7B-Instruct", "series": "qwenvl2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8291375616, "created_at": "2024-08-28", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2-VL-7B-Instruct-AWQ": {"hf_path": "Qwen/Qwen2-VL-7B-Instruct-AWQ", "series": "qwenvl2", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8291375616, "created_at": "2024-08-29", "torch_dtype": "float16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2-VL-7B-Instruct-GPTQ-Int4": {"hf_path": "Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4", "series": "qwenvl2", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8291375616, "created_at": "2024-08-29", "torch_dtype": "float16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2.5-Omni-3B": {"hf_path": "Qwen/Qwen2.5-Omni-3B", "series": "qwenvl2d5_omni", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 5537120672, "created_at": "2025-04-30", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2.5-Omni-7B": {"hf_path": "Qwen/Qwen2.5-Omni-7B", "series": "qwenvl2d5_omni", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 10732225440, "created_at": "2025-03-22", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2.5-Omni-7B-GPTQ-Int4": {"hf_path": "Qwen/Qwen2.5-Omni-7B-GPTQ-Int4", "series": "qwenvl2d5_omni", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 10732225440, "created_at": "2025-05-14", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2.5-VL-32B-Instruct": {"hf_path": "Qwen/Qwen2.5-VL-32B-Instruct", "series": "qwenvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 33452718336, "created_at": "2025-03-21", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Qwen2.5-VL-32B-Instruct-AWQ": {"hf_path": "Qwen/Qwen2.5-VL-32B-Instruct-AWQ", "series": "qwenvl2d5", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 33452718336, "created_at": "2025-03-26", "torch_dtype": "bfloat16", "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 8, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Qwen2.5-VL-3B-Instruct": {"hf_path": "Qwen/Qwen2.5-VL-3B-Instruct", "series": "qwenvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 3754622976, "created_at": "2025-01-26", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 2, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2.5-VL-3B-Instruct-AWQ": {"hf_path": "Qwen/Qwen2.5-VL-3B-Instruct-AWQ", "series": "qwenvl2d5", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 3754622976, "created_at": "2025-02-13", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 2, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2.5-VL-72B-Instruct": {"hf_path": "Qwen/Qwen2.5-VL-72B-Instruct", "series": "qwenvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 73410777344, "created_at": "2025-01-27", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 4}, + "Qwen2.5-VL-72B-Instruct-AWQ": {"hf_path": "Qwen/Qwen2.5-VL-72B-Instruct-AWQ", "series": "qwenvl2d5", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 73662435584, "created_at": "2025-02-13", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 4}, + "Qwen2.5-VL-7B-Instruct": {"hf_path": "Qwen/Qwen2.5-VL-7B-Instruct", "series": "qwenvl2d5", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8292166656, "created_at": "2025-01-26", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen2.5-VL-7B-Instruct-AWQ": {"hf_path": "Qwen/Qwen2.5-VL-7B-Instruct-AWQ", "series": "qwenvl2d5", "model_type": "instruct", "api_model": false, "quantization": true, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8292166656, "created_at": "2025-02-15", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen3-Omni-30B-A3B-Captioner": {"hf_path": "Qwen/Qwen3-Omni-30B-A3B-Captioner", "series": "qwen3_omni", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "multi_video_interleave", "text"], "parameter_count": 31719205488, "created_at": "2025-09-15", "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Qwen3-Omni-30B-A3B-Instruct": {"hf_path": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "series": "qwen3_omni", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "multi_video_interleave", "text"], "parameter_count": 35259818545, "created_at": "2025-09-20", "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Qwen3-Omni-30B-A3B-Thinking": {"hf_path": "Qwen/Qwen3-Omni-30B-A3B-Thinking", "series": "qwen3_omni", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "multi_video_interleave", "text"], "parameter_count": 31719205488, "created_at": "2025-09-15", "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Qwen3-VL-235B-A22B-Instruct": {"hf_path": "Qwen/Qwen3-VL-235B-A22B-Instruct", "series": "qwen3_vl", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 235670022896, "created_at": "2025-09-22", "torch_dtype": null, "num_hidden_layers": 94, "hidden_size": 4096, "num_attention_heads": 64, "num_key_value_heads": 4, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 8}, + "Qwen3-VL-235B-A22B-Thinking": {"hf_path": "Qwen/Qwen3-VL-235B-A22B-Thinking", "series": "qwen3_vl", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 235670022896, "created_at": "2025-09-22", "torch_dtype": null, "num_hidden_layers": 94, "hidden_size": 4096, "num_attention_heads": 64, "num_key_value_heads": 4, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 8}, + "Qwen3-VL-2B-Instruct": {"hf_path": "Qwen/Qwen3-VL-2B-Instruct", "series": "qwen3_vl", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2127532032, "created_at": "2025-10-19", "torch_dtype": null, "num_hidden_layers": 28, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen3-VL-2B-Thinking": {"hf_path": "Qwen/Qwen3-VL-2B-Thinking", "series": "qwen3_vl", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 2127532032, "created_at": "2025-10-19", "torch_dtype": null, "num_hidden_layers": 28, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen3-VL-30B-A3B-Instruct": {"hf_path": "Qwen/Qwen3-VL-30B-A3B-Instruct", "series": "qwen3_vl", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 31070754032, "created_at": "2025-09-30", "torch_dtype": null, "num_hidden_layers": 48, "hidden_size": 2048, "num_attention_heads": 32, "num_key_value_heads": 4, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Qwen3-VL-30B-A3B-Thinking": {"hf_path": "Qwen/Qwen3-VL-30B-A3B-Thinking", "series": "qwen3_vl", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 31070754032, "created_at": "2025-09-30", "torch_dtype": null, "num_hidden_layers": 48, "hidden_size": 2048, "num_attention_heads": 32, "num_key_value_heads": 4, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Qwen3-VL-32B-Instruct": {"hf_path": "Qwen/Qwen3-VL-32B-Instruct", "series": "qwen3_vl", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 33357390064, "created_at": "2025-10-19", "torch_dtype": null, "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Qwen3-VL-32B-Thinking": {"hf_path": "Qwen/Qwen3-VL-32B-Thinking", "series": "qwen3_vl", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 33357390064, "created_at": "2025-10-19", "torch_dtype": null, "num_hidden_layers": 64, "hidden_size": 5120, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 2}, + "Qwen3-VL-4B-Instruct": {"hf_path": "Qwen/Qwen3-VL-4B-Instruct", "series": "qwen3_vl", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 4437815808, "created_at": "2025-10-11", "torch_dtype": null, "num_hidden_layers": 36, "hidden_size": 2560, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen3-VL-4B-Thinking": {"hf_path": "Qwen/Qwen3-VL-4B-Thinking", "series": "qwen3_vl", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 4437815808, "created_at": "2025-10-11", "torch_dtype": null, "num_hidden_layers": 36, "hidden_size": 2560, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen3-VL-8B-Instruct": {"hf_path": "Qwen/Qwen3-VL-8B-Instruct", "series": "qwen3_vl", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8767123696, "created_at": "2025-10-11", "torch_dtype": null, "num_hidden_layers": 36, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Qwen3-VL-8B-Thinking": {"hf_path": "Qwen/Qwen3-VL-8B-Thinking", "series": "qwen3_vl", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": 8767123696, "created_at": "2025-10-11", "torch_dtype": null, "num_hidden_layers": 36, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 262144, "image_tokens": 784, "max_dynamic_patch": null, "required_gpu_count": 1}, + "R1-Onevision-7B": {"hf_path": "Fancy-MLLM/R1-Onevision-7B", "series": "r1_onevision", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 8292166656, "created_at": "2025-02-11", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "SmolVLM-Instruct": {"hf_path": "HuggingFaceTB/SmolVLM-Instruct", "series": "smolvlm", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 2246272880, "created_at": "2024-11-18", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 2048, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": 16384, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 1}, + "SmolVLM-Instruct-Base": {"hf_path": "HuggingFaceTB/SmolVLM-Base", "series": "smolvlm", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 2246272880, "created_at": "2024-11-22", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 2048, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": 16384, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 1}, + "SmolVLM-Instruct-DPO": {"hf_path": "HuggingFaceTB/SmolVLM-Instruct-DPO", "series": "smolvlm", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": null, "created_at": "2024-11-26", "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "SmolVLM-Sythetic": {"hf_path": "HuggingFaceTB/SmolVLM-Synthetic", "series": "smolvlm", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_start", "text"], "parameter_count": 2246272880, "created_at": "2024-11-22", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 2048, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": 16384, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 1}, + "VLAA-Thinker-Qwen2.5VL-3B": {"hf_path": "UCSC-VLAA/VLAA-Thinker-Qwen2.5VL-3B", "series": "vlaa_thinking", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 3754622976, "created_at": "2025-03-31", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 2, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "VLAA-Thinker-Qwen2.5VL-7B": {"hf_path": "UCSC-VLAA/VLAA-Thinker-Qwen2.5VL-7B", "series": "vlaa_thinking", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 8292166656, "created_at": "2025-03-31", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 128000, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "VLAA-Thinker-Qwen2VL-2B": {"hf_path": "UCSC-VLAA/VLAA-Thinker-Qwen2VL-2B", "series": "vlaa_thinking", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 2208985600, "created_at": "2025-04-11", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 1536, "num_attention_heads": 12, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "VLAA-Thinker-Qwen2VL-7B": {"hf_path": "UCSC-VLAA/VLAA-Thinker-Qwen2VL-7B", "series": "vlaa_thinking", "model_type": "reasoning", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 8291375616, "created_at": "2025-04-11", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "VLAA-Thinker-Qwen2VL-7B-Zero": {"hf_path": "UCSC-VLAA/VLAA-Thinker-Qwen2VL-7B-Zero", "series": "vlaa_thinking", "model_type": "pretrain", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 8291375616, "created_at": "2025-04-11", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "VideoLLaMA2-7B": {"hf_path": "DAMO-NLP-SG/VideoLLaMA2-7B", "series": "videollama2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_video_start", "text"], "parameter_count": 8034428160, "created_at": "2024-06-11", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Vintern-1B-v2": {"hf_path": "5CD-AI/Vintern-1B-v2", "series": "vintern", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 938158976, "created_at": "2024-08-09", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 896, "num_attention_heads": 14, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 12, "required_gpu_count": 1}, + "Vintern-1B-v3_5": {"hf_path": "5CD-AI/Vintern-1B-v3_5", "series": "vintern", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 938193024, "created_at": "2025-01-11", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 896, "num_attention_heads": 14, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 4, "required_gpu_count": 1}, + "Vintern-3B-beta": {"hf_path": "5CD-AI/Vintern-3B-beta", "series": "vintern", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 3712637952, "created_at": "2024-10-22", "torch_dtype": "bfloat16", "num_hidden_layers": 36, "hidden_size": 2048, "num_attention_heads": 16, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": 4, "required_gpu_count": 1}, + "WeMM": {"hf_path": "mm-eval/WeMM", "series": "wemm", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start"], "parameter_count": null, "created_at": "2025-10-14", "torch_dtype": "float16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 4900, "max_dynamic_patch": null, "required_gpu_count": 1}, + "WeMM-Chat-2k-CN": {"hf_path": "mm-eval/WeMM-Chat-2k-CN", "series": "wemm", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start"], "parameter_count": null, "created_at": "2025-10-14", "torch_dtype": "float16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 19600, "max_dynamic_patch": null, "required_gpu_count": 1}, + "WeMM-Chat-CN": {"hf_path": "mm-eval/WeMM-Chat-CN", "series": "wemm", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start"], "parameter_count": null, "created_at": "2025-10-14", "torch_dtype": "float16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 4900, "max_dynamic_patch": null, "required_gpu_count": 1}, + "Xinyuan-VL-2B": {"hf_path": "Cylingo/Xinyuan-VL-2B", "series": "xinyuanvl", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 2208985600, "created_at": "2024-09-24", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 1536, "num_attention_heads": 12, "num_key_value_heads": 2, "max_position_embeddings": 32768, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "blip2-flan-t5-xl": {"hf_path": "Salesforce/blip2-flan-t5-xl", "series": "blip2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 3942446592, "created_at": "2023-02-06", "torch_dtype": "float32", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "blip2-flan-t5-xxl": {"hf_path": "Salesforce/blip2-flan-t5-xxl", "series": "blip2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 12229596672, "created_at": "2023-02-09", "torch_dtype": "float32", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 2}, + "cambrian-13b": {"hf_path": "nyu-visionx/cambrian-13b", "series": "cambrian", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 13350085632, "created_at": "2024-06-16", "torch_dtype": "float32", "num_hidden_layers": 40, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 40, "max_position_embeddings": 4096, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "cambrian-34b": {"hf_path": "nyu-visionx/cambrian-34b", "series": "cambrian", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 34764002304, "created_at": "2024-06-16", "torch_dtype": "float32", "num_hidden_layers": 60, "hidden_size": 7168, "num_attention_heads": 56, "num_key_value_heads": 8, "max_position_embeddings": 4096, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 2}, + "cambrian-8b": {"hf_path": "nyu-visionx/cambrian-8b", "series": "cambrian", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 8333022208, "created_at": "2024-06-16", "torch_dtype": "float32", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 8, "max_position_embeddings": 8192, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "cambrian-phi3-3b": {"hf_path": "nyu-visionx/cambrian-phi3-3b", "series": "cambrian", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 4094477312, "created_at": "2024-06-24", "torch_dtype": "float32", "num_hidden_layers": 32, "hidden_size": 3072, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": 4096, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "claude-haiku-4-5": {"hf_path": null, "series": "anthropic_claude", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "claude-opus-4-0": {"hf_path": null, "series": "anthropic_claude", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "claude-opus-4-1": {"hf_path": null, "series": "anthropic_claude", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "claude-opus-4-5": {"hf_path": null, "series": "anthropic_claude", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "claude-opus-4-6": {"hf_path": null, "series": "anthropic_claude", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "claude-sonnet-4-0": {"hf_path": null, "series": "anthropic_claude", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "claude-sonnet-4-5": {"hf_path": null, "series": "anthropic_claude", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "claude-sonnet-4-6": {"hf_path": null, "series": "anthropic_claude", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "doubao-seed-1-6-flash-250828": {"hf_path": null, "series": "doubao_ark", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "doubao-seed-1-6-lite-251015": {"hf_path": null, "series": "doubao_ark", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "doubao-seed-1-6-vision-250815": {"hf_path": null, "series": "doubao_ark", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "doubao-seed-1-8-251228": {"hf_path": null, "series": "doubao_ark", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "doubao-seed-2-0-code-preview-260215": {"hf_path": null, "series": "doubao_ark", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "doubao-seed-2-0-lite-260215": {"hf_path": null, "series": "doubao_ark", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "doubao-seed-2-0-mini-260215": {"hf_path": null, "series": "doubao_ark", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "doubao-seed-2-0-pro-260215": {"hf_path": null, "series": "doubao_ark", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "doubao-seed-code-preview-251028": {"hf_path": null, "series": "doubao_ark", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "fuyu-8b": {"hf_path": "adept/fuyu-8b", "series": "fuyu", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start"], "parameter_count": 9408238592, "created_at": "2023-10-17", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "gemini-2.5-flash": {"hf_path": null, "series": "google_gemini", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gemini-2.5-flash-lite": {"hf_path": null, "series": "google_gemini", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gemini-2.5-pro": {"hf_path": null, "series": "google_gemini", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gemini-3-flash-preview": {"hf_path": null, "series": "google_gemini", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gemini-3-pro-image-preview": {"hf_path": null, "series": "google_gemini", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gemini-3-pro-preview": {"hf_path": null, "series": "google_gemini", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_video_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gemma-3-12b-it": {"hf_path": "google/gemma-3-12b-it", "series": "gemma3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 12187325040, "created_at": "2025-03-01", "torch_dtype": "bfloat16", "num_hidden_layers": 48, "hidden_size": 3840, "num_attention_heads": 16, "num_key_value_heads": 8, "max_position_embeddings": null, "image_tokens": 4096, "max_dynamic_patch": null, "required_gpu_count": 1}, + "gemma-3-27b-it": {"hf_path": "google/gemma-3-27b-it", "series": "gemma3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 27432406640, "created_at": "2025-03-01", "torch_dtype": "bfloat16", "num_hidden_layers": 62, "hidden_size": 5376, "num_attention_heads": 32, "num_key_value_heads": 16, "max_position_embeddings": null, "image_tokens": 4096, "max_dynamic_patch": null, "required_gpu_count": 1}, + "gemma-3-4b-it": {"hf_path": "google/gemma-3-4b-it", "series": "gemma3", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 4300079472, "created_at": "2025-02-20", "torch_dtype": "bfloat16", "num_hidden_layers": 34, "hidden_size": 2560, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": 4096, "max_dynamic_patch": null, "required_gpu_count": 1}, + "glm-4v-9b": {"hf_path": "THUDM/glm-4v-9b", "series": "glm_4v", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 13906320928, "created_at": "2024-06-04", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": 6400, "max_dynamic_patch": null, "required_gpu_count": 1}, + "gpt-4.1": {"hf_path": null, "series": "openai_gpt", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gpt-4.1-mini": {"hf_path": null, "series": "openai_gpt", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gpt-4.1-nano": {"hf_path": null, "series": "openai_gpt", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gpt-4o": {"hf_path": null, "series": "openai_gpt", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gpt-4o-mini": {"hf_path": null, "series": "openai_gpt", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gpt-5": {"hf_path": null, "series": "openai_gpt", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gpt-5-mini": {"hf_path": null, "series": "openai_gpt", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gpt-5-nano": {"hf_path": null, "series": "openai_gpt", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gpt-5.1": {"hf_path": null, "series": "openai_gpt", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "gpt-5.2": {"hf_path": null, "series": "openai_gpt", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "grok-2-vision-1212": {"hf_path": null, "series": "xai_grok", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "grok-4-0709": {"hf_path": null, "series": "xai_grok", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "grok-4-1-fast-non-reasoning": {"hf_path": null, "series": "xai_grok", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "grok-4-1-fast-reasoning": {"hf_path": null, "series": "xai_grok", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "grok-4-fast-non-reasoning": {"hf_path": null, "series": "xai_grok", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "grok-4-fast-reasoning": {"hf_path": null, "series": "xai_grok", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "hunyuan-large-vision": {"hf_path": null, "series": "hunyuan_vision", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "hunyuan-t1-vision": {"hf_path": null, "series": "hunyuan_vision", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "hunyuan-turbos-vision": {"hf_path": null, "series": "hunyuan_vision", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "hunyuan-vision": {"hf_path": null, "series": "hunyuan_vision", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "hunyuan-vision-1.5-instruct": {"hf_path": null, "series": "hunyuan_vision", "model_type": "instruct", "api_model": true, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": null, "created_at": null, "torch_dtype": null, "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 0}, + "instructblip-flan-t5-xl": {"hf_path": "Salesforce/instructblip-flan-t5-xl", "series": "instructblip", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start"], "parameter_count": 4022969088, "created_at": "2023-05-28", "torch_dtype": "float32", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "instructblip-flan-t5-xxl": {"hf_path": "Salesforce/instructblip-flan-t5-xxl", "series": "instructblip", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start"], "parameter_count": 12310119168, "created_at": "2023-06-03", "torch_dtype": "float32", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "instructblip-vicuna-13b": {"hf_path": "Salesforce/instructblip-vicuna-13b", "series": "instructblip", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start"], "parameter_count": 14192093952, "created_at": "2023-06-03", "torch_dtype": "float16", "num_hidden_layers": 40, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 40, "max_position_embeddings": null, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "instructblip-vicuna-7b": {"hf_path": "Salesforce/instructblip-vicuna-7b", "series": "instructblip", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start"], "parameter_count": 7913726720, "created_at": "2023-05-22", "torch_dtype": "float16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": null, "image_tokens": 1024, "max_dynamic_patch": null, "required_gpu_count": 1}, + "internlm-xcomposer-7b": {"hf_path": "internlm/internlm-xcomposer-7b", "series": "internlm_xcomposer", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": null, "created_at": "2023-09-26", "torch_dtype": "float32", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": null, "max_position_embeddings": 2048, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llama3-llava-next-8b-hf": {"hf_path": "llava-hf/llama3-llava-next-8b-hf", "series": "llava_next", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 8355276800, "created_at": "2024-07-19", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": 8, "max_position_embeddings": 8192, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-1.5-13b-hf": {"hf_path": "llava-hf/llava-1.5-13b-hf", "series": "llava", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 13351494656, "created_at": "2023-12-05", "torch_dtype": "float16", "num_hidden_layers": 40, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 40, "max_position_embeddings": 4096, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-1.5-7b-hf": {"hf_path": "llava-hf/llava-1.5-7b-hf", "series": "llava", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 7063427072, "created_at": "2023-12-05", "torch_dtype": "float16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": 4096, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-next-110b-hf": {"hf_path": "llava-hf/llava-next-110b-hf", "series": "llava_next", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 111588943872, "created_at": "2024-07-19", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 4}, + "llava-next-72b-hf": {"hf_path": "llava-hf/llava-next-72b-hf", "series": "llava_next", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 72666949632, "created_at": "2024-07-19", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 64, "max_position_embeddings": 32768, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 4}, + "llava-next-interleave-qwen-0.5b": {"hf_path": "lmms-lab/llava-next-interleave-qwen-0.5b", "series": "llava_next_interleave", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 863966752, "created_at": "2024-06-27", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 1024, "num_attention_heads": 16, "num_key_value_heads": 16, "max_position_embeddings": 32768, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-next-interleave-qwen-7b": {"hf_path": "lmms-lab/llava-next-interleave-qwen-7b", "series": "llava_next_interleave", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 8140580384, "created_at": "2024-06-19", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": 32768, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-next-interleave-qwen-7b-dpo": {"hf_path": "lmms-lab/llava-next-interleave-qwen-7b-dpo", "series": "llava_next_interleave", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 8140580384, "created_at": "2024-06-26", "torch_dtype": "bfloat16", "num_hidden_layers": 32, "hidden_size": 4096, "num_attention_heads": 32, "num_key_value_heads": 32, "max_position_embeddings": 32768, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-onevision-qwen2-0.5b-ov-hf": {"hf_path": "llava-hf/llava-onevision-qwen2-0.5b-ov-hf", "series": "llava_ov", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_video_start", "text"], "parameter_count": 893675552, "created_at": "2024-08-13", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 896, "num_attention_heads": 14, "num_key_value_heads": 2, "max_position_embeddings": null, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-onevision-qwen2-0.5b-si-hf": {"hf_path": "llava-hf/llava-onevision-qwen2-0.5b-si-hf", "series": "llava_ov", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_video_start", "text"], "parameter_count": 893675552, "created_at": "2024-08-13", "torch_dtype": "bfloat16", "num_hidden_layers": 24, "hidden_size": 896, "num_attention_heads": 14, "num_key_value_heads": 2, "max_position_embeddings": null, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-onevision-qwen2-72b-ov-chat-hf": {"hf_path": "llava-hf/llava-onevision-qwen2-72b-ov-chat-hf", "series": "llava_ov", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_video_start", "text"], "parameter_count": 73181570592, "created_at": "2024-09-16", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": null, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 4}, + "llava-onevision-qwen2-72b-ov-hf": {"hf_path": "llava-hf/llava-onevision-qwen2-72b-ov-hf", "series": "llava_ov", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_video_start", "text"], "parameter_count": 73181570592, "created_at": "2024-08-13", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": null, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 4}, + "llava-onevision-qwen2-72b-si-hf": {"hf_path": "llava-hf/llava-onevision-qwen2-72b-si-hf", "series": "llava_ov", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_video_start", "text"], "parameter_count": 73181570592, "created_at": "2024-08-13", "torch_dtype": "bfloat16", "num_hidden_layers": 80, "hidden_size": 8192, "num_attention_heads": 64, "num_key_value_heads": 8, "max_position_embeddings": null, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 4}, + "llava-onevision-qwen2-7b-ov-chat-hf": {"hf_path": "llava-hf/llava-onevision-qwen2-7b-ov-chat-hf", "series": "llava_ov", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_video_start", "text"], "parameter_count": 8030807584, "created_at": "2024-09-16", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": null, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-onevision-qwen2-7b-ov-hf": {"hf_path": "llava-hf/llava-onevision-qwen2-7b-ov-hf", "series": "llava_ov", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_video_start", "text"], "parameter_count": 8030807584, "created_at": "2024-08-13", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": null, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-onevision-qwen2-7b-si-hf": {"hf_path": "llava-hf/llava-onevision-qwen2-7b-si-hf", "series": "llava_ov", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_video_start", "text"], "parameter_count": 8030807584, "created_at": "2024-08-13", "torch_dtype": "bfloat16", "num_hidden_layers": 28, "hidden_size": 3584, "num_attention_heads": 28, "num_key_value_heads": 4, "max_position_embeddings": null, "image_tokens": 729, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-v1.6-34b-hf": {"hf_path": "llava-hf/llava-v1.6-34b-hf", "series": "llava_next", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 34752083968, "created_at": "2024-03-17", "torch_dtype": "bfloat16", "num_hidden_layers": 60, "hidden_size": 7168, "num_attention_heads": 56, "num_key_value_heads": 8, "max_position_embeddings": 4096, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 2}, + "llava-v1.6-mistral-7b-hf": {"hf_path": "llava-hf/llava-v1.6-mistral-7b-hf", "series": "llava_next", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 7566747648, "created_at": "2024-02-20", "torch_dtype": "float16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": 8, "max_position_embeddings": 32768, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-v1.6-vicuna-13b-hf": {"hf_path": "llava-hf/llava-v1.6-vicuna-13b-hf", "series": "llava_next", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 13351499776, "created_at": "2024-03-17", "torch_dtype": "float16", "num_hidden_layers": 40, "hidden_size": 5120, "num_attention_heads": 40, "num_key_value_heads": 40, "max_position_embeddings": 4096, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 1}, + "llava-v1.6-vicuna-7b-hf": {"hf_path": "llava-hf/llava-v1.6-vicuna-7b-hf", "series": "llava_next", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 7063431168, "created_at": "2024-03-17", "torch_dtype": "float16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": 4096, "image_tokens": 576, "max_dynamic_patch": null, "required_gpu_count": 1}, + "moondream1": {"hf_path": "vikhyatk/moondream1", "series": "moondream1", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 1857482608, "created_at": "2024-01-20", "torch_dtype": "float16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "moondream2": {"hf_path": "vikhyatk/moondream2", "series": "moondream2", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["single_image_start", "text"], "parameter_count": 1927237104, "created_at": "2024-03-04", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1}, + "xgen-mm-phi3-mini-instruct-interleave-r-v1.5": {"hf_path": "Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5", "series": "xgen", "model_type": "instruct", "api_model": false, "quantization": false, "modalities": ["multi_image_interleave", "text"], "parameter_count": 4359257219, "created_at": "2024-08-09", "torch_dtype": "bfloat16", "num_hidden_layers": null, "hidden_size": null, "num_attention_heads": null, "num_key_value_heads": null, "max_position_embeddings": null, "image_tokens": null, "max_dynamic_patch": null, "required_gpu_count": 1} +} diff --git a/mmeval/utils/gpu.py b/mmeval/utils/gpu.py new file mode 100644 index 0000000..581909e --- /dev/null +++ b/mmeval/utils/gpu.py @@ -0,0 +1,187 @@ +"""GPU discovery and VRAM-aware allocation helpers for batch inference. + +These helpers let ``mmeval/batch_infer.py`` decide, for each model, how many +GPUs a single inference shard needs and how many shards can run in parallel, +based on an estimate of the model's peak VRAM usage. +""" + +import math +import os +import re +import subprocess + +# Bytes per parameter for common weight dtypes (KV cache is always fp16/bf16). +DTYPE_BYTES = {"bfloat16": 2, "float16": 2, "float32": 4, "int8": 1, "int4": 0.5, "fp8": 1} + +DEFAULT_IMAGE_TOKENS = 1024 +DEFAULT_TEXT_TOKENS = 2048 +DEFAULT_MAX_CONTEXT = 32768 +VRAM_OVERHEAD_GB = 3.0 # activations, fragmentation, runtime overhead +VRAM_FALLBACK_GB_PER_GPU = 40.0 # used when parameter_count is unknown +VRAM_USABLE_FRACTION = 0.9 # leave headroom on each GPU +DETECTED_VRAM_FALLBACK_GB = 80.0 # assumed per-GPU VRAM when detection fails + + +def _get_visible_devices() -> str: + """Return CUDA_VISIBLE_DEVICES, falling back to NVIDIA_VISIBLE_DEVICES.""" + if "CUDA_VISIBLE_DEVICES" in os.environ: + return os.environ["CUDA_VISIBLE_DEVICES"].strip() + return os.environ.get("NVIDIA_VISIBLE_DEVICES", "").strip() + + +def _visibility_explicitly_disabled() -> bool: + """True when CUDA_VISIBLE_DEVICES is set to empty (CUDA = disable all GPUs).""" + return "CUDA_VISIBLE_DEVICES" in os.environ and not os.environ["CUDA_VISIBLE_DEVICES"].strip() + + +def _nvidia_smi_vram(*extra_args: str) -> float | None: + """Query nvidia-smi for the minimum per-GPU VRAM in GiB. None on failure.""" + try: + out = subprocess.run( + ["nvidia-smi", *extra_args, + "--query-gpu=memory.total", "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=5, + ) + if out.returncode == 0: + vals = [int(v) for v in out.stdout.strip().split("\n") if v.strip()] + if vals: + return min(vals) / 1024 # MiB -> GiB + except (FileNotFoundError, subprocess.TimeoutExpired, ValueError): + pass + return None + + +def _parse_mig_vram_gb() -> float | None: + """Parse the smallest MIG slice VRAM from nvidia-smi -L profiles (e.g. '1g.10gb').""" + try: + out = subprocess.run(["nvidia-smi", "-L"], capture_output=True, text=True, timeout=5) + if out.returncode == 0: + sizes = re.findall(r"\d+g\.(\d+)gb", out.stdout, re.IGNORECASE) + if sizes: + return float(min(int(s) for s in sizes)) + except (FileNotFoundError, subprocess.TimeoutExpired): + pass + return None + + +def get_gpu_indices() -> list[str]: + """Return the visible GPU tokens as strings (respects CUDA_VISIBLE_DEVICES).""" + if _visibility_explicitly_disabled(): + return [] + cvd = _get_visible_devices() + if cvd and cvd.lower() != "all": + # Opaque tokens: numeric ids and GPU/MIG UUIDs are both supported. + return [x.strip() for x in cvd.split(",") if x.strip()] + + try: + out = subprocess.run(["nvidia-smi", "-L"], capture_output=True, text=True, timeout=5) + if out.returncode != 0: + return [] + lines = [ln.strip() for ln in out.stdout.strip().splitlines() if ln.strip()] + # MIG mode: CUDA sees MIG instances, not parent GPUs. + mig_lines = [ln for ln in lines if "MIG" in ln and "Device" in ln] + if mig_lines: + return [str(i) for i in range(len(mig_lines))] + gpu_count = len([ln for ln in lines if ln.startswith("GPU") and ":" in ln]) + return [str(i) for i in range(gpu_count)] + except (FileNotFoundError, subprocess.TimeoutExpired): + return [] + + +def get_gpu_vram_gb() -> float: + """Detect the minimum per-GPU VRAM in GiB, falling back to a safe default.""" + if _visibility_explicitly_disabled(): + return 0.0 + cvd = _get_visible_devices() + if cvd and cvd.lower() != "all": + cvd_norm = ",".join(x.strip() for x in cvd.split(",") if x.strip()) + detected = _nvidia_smi_vram("-i", cvd_norm) + if detected is not None: + return detected + # -i can fail for MIG UUIDs; try MIG profiles, then the safe fallback. + return _parse_mig_vram_gb() or DETECTED_VRAM_FALLBACK_GB + return _nvidia_smi_vram() or DETECTED_VRAM_FALLBACK_GB + + +def estimate_max_tokens(model: dict, num_images: int = 1) -> int: + """Estimate the sequence length (image + text tokens) for VRAM sizing.""" + image_tokens = int(model.get("image_tokens") or DEFAULT_IMAGE_TOKENS) + max_dynamic_patch = int(model.get("max_dynamic_patch") or 1) + total = image_tokens * max_dynamic_patch * num_images + DEFAULT_TEXT_TOKENS + max_ctx = int(model.get("max_position_embeddings") or DEFAULT_MAX_CONTEXT) + return min(total, max_ctx) + + +def estimate_vram_gb(model: dict, max_tokens: int, batch_size: int = 1) -> float: + """Estimate peak VRAM (GiB) for one inference shard of ``model``.""" + params = int(model.get("parameter_count") or 0) + if params == 0: + # Architecture unknown: fall back to the curated GPU count. + required = model.get("required_gpu_count") + return (float(required) if str(required).isdigit() else 1.0) * VRAM_FALLBACK_GB_PER_GPU + + weight_bytes = DTYPE_BYTES.get(model.get("torch_dtype") or "bfloat16", 2) + weights_gb = params * weight_bytes / (1024 ** 3) + + # KV cache (GQA-aware), always fp16/bf16 regardless of weight quantization. + num_layers = int(model.get("num_hidden_layers") or 32) + hidden_size = int(model.get("hidden_size") or 4096) + num_attn_heads = int(model.get("num_attention_heads") or 32) + num_kv_heads = int(model.get("num_key_value_heads") or num_attn_heads) + head_dim = -(-hidden_size // num_attn_heads) if num_attn_heads > 0 else hidden_size + kv_cache_gb = 2 * batch_size * num_layers * (num_kv_heads * head_dim) * max_tokens * 2 / (1024 ** 3) + + return weights_gb + kv_cache_gb + VRAM_OVERHEAD_GB + + +def parse_gpu_memory(value: list[float] | None, visible_gpu_count: int) -> list[float]: + """Resolve ``--gpu-memory`` into a per-visible-GPU VRAM list (GiB).""" + if visible_gpu_count <= 0: + return [] + if value is None: + return [get_gpu_vram_gb()] * visible_gpu_count + + parsed = [float(v) for v in value] + if any(v < 0 for v in parsed): + raise ValueError("--gpu-memory values must be >= 0") + if len(parsed) < visible_gpu_count: + parsed.extend([0.0] * (visible_gpu_count - len(parsed))) + elif len(parsed) > visible_gpu_count: + print(f"Warning: --gpu-memory has {len(parsed)} values but only " + f"{visible_gpu_count} GPU(s) visible; dropping extras: {parsed[visible_gpu_count:]}") + return parsed[:visible_gpu_count] + + +def min_gpu_vram(gpu_memory: list[float]) -> float: + """Minimum VRAM among enabled GPUs (those with VRAM > 0); 0.0 if none.""" + enabled = [v for v in gpu_memory if v > 0] + return min(enabled) if enabled else 0.0 + + +def enabled_gpu_indices(visible: list[str], gpu_memory: list[float]) -> list[str]: + """Visible GPU tokens whose configured VRAM is > 0.""" + return [idx for idx, vram in zip(visible, gpu_memory) if vram > 0] + + +def gpu_allocation(available_gpus: int, estimated_vram_gb: float, per_gpu_vram_gb: float) -> dict: + """Plan GPU usage for one model. + + Returns a dict with ``can_run`` plus, when runnable, ``gpu_per_parallel`` + (GPUs per shard) and ``parallel_per_task`` (concurrent shards). When not + runnable it carries a human-readable ``reason``. + """ + if available_gpus <= 0: + return {"can_run": False, "reason": "no enabled GPU"} + if per_gpu_vram_gb <= 0: + return {"can_run": False, "reason": "all GPUs disabled (--gpu-memory)"} + + if estimated_vram_gb <= 0: + return {"can_run": True, "gpu_per_parallel": 1, "parallel_per_task": available_gpus} + + gpus_needed = max(1, math.ceil(estimated_vram_gb / (per_gpu_vram_gb * VRAM_USABLE_FRACTION))) + if gpus_needed > available_gpus: + return {"can_run": False, + "reason": f"need {gpus_needed} GPU(s) ({estimated_vram_gb:.1f}GiB), have {available_gpus}"} + return {"can_run": True, + "gpu_per_parallel": gpus_needed, + "parallel_per_task": available_gpus // gpus_needed} diff --git a/scripts/update_model_metadata.py b/scripts/update_model_metadata.py new file mode 100644 index 0000000..6583e77 --- /dev/null +++ b/scripts/update_model_metadata.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Refresh auto-derivable fields in ``mmeval/model_metadata.json`` from HuggingFace. + +The metadata file is a companion to ``mmeval/registry.py`` used by +``mmeval/batch_infer.py`` for VRAM-aware scheduling and model filtering. It is a +JSON object keyed by model name; each value carries the size/architecture and +modality fields the scheduler needs. Curated fields (hf_path, series, model_type, +api_model, quantization, modalities, required_gpu_count) are left untouched; only +the architecture/size fields fetched from HuggingFace are updated. + +This script also cross-checks the file against ``mmeval/registry.py`` (the +framework's source of truth) and reports models present in one but not the other, +so the two stay in sync. + +Usage: + HF_TOKEN=hf_xxx python scripts/update_model_metadata.py +""" + +import json +import os +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +METADATA_JSON = REPO_ROOT / "mmeval" / "model_metadata.json" + +# Fields refreshed from HuggingFace; everything else is curated by hand. +AUTO_FIELDS = [ + "parameter_count", "created_at", "torch_dtype", + "num_hidden_layers", "hidden_size", "num_attention_heads", + "num_key_value_heads", "max_position_embeddings", + "image_tokens", "max_dynamic_patch", +] + +# Per-model field order for stable, readable output. +FIELD_ORDER = [ + "hf_path", "series", "model_type", "api_model", "quantization", "modalities", + "parameter_count", "created_at", "torch_dtype", + "num_hidden_layers", "hidden_size", "num_attention_heads", + "num_key_value_heads", "max_position_embeddings", + "image_tokens", "max_dynamic_patch", "required_gpu_count", +] + + +def load_metadata(path: Path = METADATA_JSON) -> dict: + """Load the metadata file as a dict keyed by model name.""" + with open(path, encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, dict): + raise ValueError(f"{path} must contain a JSON object keyed by model name.") + return data + + +def write_metadata(data: dict, path: Path = METADATA_JSON) -> None: + """Write the metadata file with one model per line (compact, diff-friendly).""" + names = sorted(data) + lines = ["{"] + for i, name in enumerate(names): + fields = data[name] + ordered = {k: fields[k] for k in FIELD_ORDER if k in fields} + # Preserve any unexpected extra keys so they are never silently dropped. + ordered.update({k: v for k, v in fields.items() if k not in ordered}) + tail = "," if i < len(names) - 1 else "" + lines.append(f" {json.dumps(name, ensure_ascii=False)}: " + f"{json.dumps(ordered, ensure_ascii=False)}{tail}") + lines.append("}") + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + + +def check_against_registry(data: dict) -> None: + """Warn about models that differ between the metadata file and registry.py.""" + try: + from mmeval.registry import series_mapping + except Exception as exc: # pragma: no cover - registry import is optional here + print(f"WARN: could not import mmeval.registry for cross-check: {exc}") + return + registry_models = {m for models in series_mapping.values() for m in models} + meta_models = set(data) + missing = sorted(registry_models - meta_models) + stale = sorted(meta_models - registry_models) + if missing: + print(f"NOTE: {len(missing)} model(s) in registry.py but not in the metadata file " + f"(add them with curated metadata): {', '.join(missing)}") + if stale: + print(f"NOTE: {len(stale)} model(s) in the metadata file but not in registry.py " + f"(run.py cannot resolve these): {', '.join(stale)}") + if not missing and not stale: + print(f"Registry cross-check OK: {len(meta_models)} models in sync with registry.py.") + + +def _int_or_none(value) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None + + +def extract_parameter_count(info) -> int | None: + st = getattr(info, "safetensors", None) + total = getattr(st, "total", None) if st is not None else None + if isinstance(total, (int, float)): + return int(total) + card = getattr(info, "cardData", None) or {} + for key in ("parameters", "parameter_count", "num_parameters"): + if isinstance(card.get(key), (int, float)): + return int(card[key]) + return None + + +def fetch_architecture(hf_path: str, token: str | None) -> dict: + from huggingface_hub import hf_hub_download + config_path = hf_hub_download(repo_id=hf_path, filename="config.json", token=token) + with open(config_path, encoding="utf-8") as f: + config = json.load(f) + + llm = config.get("llm_config") or {} + text = config.get("text_config") or {} + text_cfg = llm or text or config + + vision = config.get("vision_config") + image_tokens = None + if vision: + size = _int_or_none(vision.get("image_size")) or 448 + patch = _int_or_none(vision.get("patch_size")) or 14 + image_tokens = (size // patch) ** 2 + + return { + "num_hidden_layers": _int_or_none(text_cfg.get("num_hidden_layers")), + "hidden_size": _int_or_none(text_cfg.get("hidden_size")), + "num_attention_heads": _int_or_none(text_cfg.get("num_attention_heads")), + "num_key_value_heads": _int_or_none(text_cfg.get("num_key_value_heads") + or text_cfg.get("num_kv_heads")), + "max_position_embeddings": _int_or_none(text_cfg.get("max_position_embeddings")), + "torch_dtype": llm.get("torch_dtype") or text.get("torch_dtype") or config.get("torch_dtype"), + "image_tokens": image_tokens, + "max_dynamic_patch": _int_or_none(config.get("max_dynamic_patch")), + } + + +def fetch_metadata(hf_path: str, api, token: str | None, retries: int = 3) -> tuple[dict | None, str | None]: + from huggingface_hub.utils import RepositoryNotFoundError + for attempt in range(retries): + try: + info = api.model_info(hf_path) + meta: dict = {} + params = extract_parameter_count(info) + if params is not None: + meta["parameter_count"] = params + created = getattr(info, "created_at", None) + if created: + meta["created_at"] = created.date().isoformat() + try: + meta.update({k: v for k, v in fetch_architecture(hf_path, token).items() if v is not None}) + except Exception as exc: + print(f"(config.json failed: {exc}) ", end="") + return meta, None + except RepositoryNotFoundError: + return None, "repository not found" + except Exception as exc: + if attempt < retries - 1: + time.sleep(2 ** (attempt + 1)) + continue + return None, f"{type(exc).__name__}: {exc}" + return None, "max retries exceeded" + + +def main() -> None: + try: + from huggingface_hub import HfApi + except ImportError: + sys.exit("huggingface_hub not installed. Run: pip install huggingface_hub") + + data = load_metadata() + check_against_registry(data) + + token = os.environ.get("HF_TOKEN") + api = HfApi(token=token) + updated = 0 + failures: list[str] = [] + + for i, (name, model) in enumerate(sorted(data.items()), 1): + hf_path = (model.get("hf_path") or "").strip() + if not hf_path: + continue # API models and local-only entries have no HF repo. + print(f"[{i}/{len(data)}] {hf_path} ... ", end="", flush=True) + meta, error = fetch_metadata(hf_path, api, token) + if error: + print(f"FAILED ({error})") + failures.append(f"{name} ({error})") + continue + changed = False + for field in AUTO_FIELDS: + new = meta.get(field) + if new is not None and new != model.get(field): + model[field] = new + changed = True + print("UPDATED" if changed else "OK") + updated += changed + time.sleep(0.5) + + write_metadata(data) + print(f"\nDone. Updated {updated} model(s) in {METADATA_JSON}") + if failures: + print(f"\nSkipped/failed ({len(failures)}):") + for item in failures: + print(f" - {item}") + + +if __name__ == "__main__": + main()