From 533cd5caa609eae91e4c131da6fb6bdb3b0c3dfa Mon Sep 17 00:00:00 2001 From: bohaotu Date: Thu, 19 Nov 2026 15:21:39 +0800 Subject: [PATCH 1/8] fix(medusa): bind Target vocabulary at training startup --- aspec/training/medusa_trainer.py | 19 +++++++++++++++++++ tests/cpu/test_medusa_integration.py | 14 +++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/aspec/training/medusa_trainer.py b/aspec/training/medusa_trainer.py index fa25c61..262ec32 100644 --- a/aspec/training/medusa_trainer.py +++ b/aspec/training/medusa_trainer.py @@ -17,6 +17,25 @@ def _create_training_model(self, draft_model): gradient_checkpointing=getattr(self.args, "gradient_checkpointing", False), ) + def _load_algorithm_weights(self, draft_model, target_model_path): + # Plan can only bind the vocabulary when the Target config is local. + # Resolve the effective config here as well so a Hub ID cannot skip it. + from transformers import AutoConfig + + config = AutoConfig.from_pretrained( + target_model_path, + trust_remote_code=getattr(self.args, "trust_remote_code", False), + ) + config = getattr(config, "text_config", config) + target_vocab_size = getattr(config, "vocab_size", None) + if not isinstance(target_vocab_size, int) or isinstance(target_vocab_size, bool): + raise ValueError("Medusa requires the Target config to declare vocab_size") + if draft_model.vocab_size != target_vocab_size: + raise ValueError( + "Medusa heads project the full Target vocabulary: " + f"draft vocab_size {draft_model.vocab_size} != " + f"Target vocab_size {target_vocab_size}") + def _prepare_algorithm_training(self, draft_model): if getattr(self.args, "freeze_lm_head", False): raise ValueError("Medusa heads own trainable vocabulary projections") diff --git a/tests/cpu/test_medusa_integration.py b/tests/cpu/test_medusa_integration.py index 21a08ca..87d8eb4 100644 --- a/tests/cpu/test_medusa_integration.py +++ b/tests/cpu/test_medusa_integration.py @@ -64,7 +64,7 @@ def _tiny_config(**overrides): def _fake_qwen_target(tmp_path, vocab_size=151936): target = tmp_path / "Qwen3-8B-test" - target.mkdir() + target.mkdir(parents=True, exist_ok=True) (target / "config.json").write_text(json.dumps({ "model_type": "qwen3", "architectures": ["Qwen3ForCausalLM"], @@ -326,6 +326,18 @@ def test_head_count_drift_is_rejected(): trainer._prepare_algorithm_training(MedusaDraftModel(_tiny_config())) +def test_target_vocabulary_is_bound_at_startup(tmp_path): + from aspec.training.medusa_trainer import MedusaTrainer + + trainer = object.__new__(MedusaTrainer) + trainer.args = SimpleNamespace(trust_remote_code=False) + draft = MedusaDraftModel(_tiny_config()) + trainer._load_algorithm_weights(draft, str(_fake_qwen_target(tmp_path, vocab_size=32))) + mismatched = _fake_qwen_target(tmp_path / "mismatch", vocab_size=31) + with pytest.raises(ValueError, match="full Target vocabulary"): + trainer._load_algorithm_weights(draft, str(mismatched)) + + def test_metric_semantics_drop_unmeasured_acceptance(): from aspec.training.medusa_trainer import MedusaTrainer From c21469978e3bba2fc1d35b2a2ccfbee44cf04042 Mon Sep 17 00:00:00 2001 From: bohaotu Date: Thu, 19 Nov 2026 15:24:20 +0800 Subject: [PATCH 2/8] feat(medusa): export heads as a native artifact with strict reload Medusa checkpoints export as aspec-native assets only; SGLang format is rejected until a candidate-tree loader exists. The native loader now handles heads-only state without token embeddings, and --verify reloads every tensor strictly and compares it to the exported bytes. --- aspec/commands/convert_to_hf.py | 35 ++++++++++++-- aspec/draft_artifact.py | 20 +++++++- tests/cpu/test_medusa_export.py | 84 +++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 tests/cpu/test_medusa_export.py diff --git a/aspec/commands/convert_to_hf.py b/aspec/commands/convert_to_hf.py index d71b58f..c2072ff 100644 --- a/aspec/commands/convert_to_hf.py +++ b/aspec/commands/convert_to_hf.py @@ -166,6 +166,11 @@ def _serving_config(raw_config: dict) -> dict: if raw_config.get("mtp_conditioning"): raise NotImplementedError("MTP-conditioned Drafts require native --format aspec export") + if raw_config.get("architectures") == ["MedusaDraftModel"]: + raise NotImplementedError( + "Medusa heads require native --format aspec export; " + "SGLang has no verified candidate-tree Medusa loader") + if raw_config.get("architectures") == ["NativeMTPDraftModel"]: native = json.loads(json.dumps(raw_config["native_config"])) native["architectures"] = ["Qwen3_5ForCausalLMMTP"] @@ -343,6 +348,7 @@ def convert_checkpoint( output_dtype: torch.dtype | None = None, force: bool = False, artifact_format: str = "sglang", + verify: bool = False, ) -> dict: input_dir = input_dir.expanduser().resolve() model_dir = _detect_model_dir(input_dir) @@ -358,7 +364,7 @@ def convert_checkpoint( raw_config = json.loads(config_path.read_text(encoding="utf-8")) if artifact_format not in {"sglang", "aspec"}: raise ValueError("artifact_format must be sglang or aspec") - if resolve_draft_algorithm(raw_config) == "mtp": + if resolve_draft_algorithm(raw_config) in {"mtp", "medusa"}: raw_config = AutoDraftModelConfig.from_dict(raw_config).to_dict() training_contract = _checkpoint_training_contract(model_dir, raw_config) export_config = (_serving_config(raw_config) if artifact_format == "sglang" @@ -469,7 +475,9 @@ def convert_checkpoint( manifest["mtp_conditioning"] = mtp_conditioning_semantics(raw_config) if artifact_format == "aspec": - manifest["compute_dtype"] = str(model.embed_tokens.weight.dtype) + embedding = getattr(model, "embed_tokens", None) + compute = embedding.weight if embedding is not None else next(iter(tensors.values())) + manifest["compute_dtype"] = str(compute.dtype) manifest["tensor_dtypes"] = dict(Counter(str(t.dtype) for t in tensors.values())) if len(manifest["tensor_dtypes"]) > 1: manifest["dtype"] = "mixed" @@ -488,14 +496,28 @@ def convert_checkpoint( "runtime": "aspec", "entrypoint": "aspec.draft_artifact.load_native_draft", "proposal_entrypoint": proposal, "validation": "required_per_artifact", - "scope": ("asset and proposal; caller owns verifier features/cache/verification" - if proposal else "asset loading and weights-only warmstart; no native proposal component"), + "scope": ( + "asset and proposal; caller owns verifier features/cache/verification" + if proposal else + "heads-only asset; strict reload and per-head held-out metrics; " + "no candidate-tree acceptance" if algorithm == "medusa" else + "asset loading and weights-only warmstart; no native proposal component"), "borrowed_tensors": ["Target output head"] if algorithm in {"dspark", "dflash"} else [], } (output_dir / "export_manifest.json").write_text( json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) + if verify: + if artifact_format != "aspec": + raise ValueError("--verify requires --format aspec") + from aspec.draft_artifact import verify_native_export + + manifest["verified"] = verify_native_export(output_dir) + (output_dir / "export_manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) return manifest @@ -511,6 +533,10 @@ def _parse_args() -> argparse.Namespace: "--dtype", choices=("float16", "bfloat16", "float32"), default=None ) parser.add_argument("-f", "--force", action="store_true") + parser.add_argument( + "--verify", action="store_true", + help="Strictly reload the native artifact and compare every tensor", + ) return parser.parse_args() @@ -527,6 +553,7 @@ def main() -> None: output_dtype=dtype, force=args.force, artifact_format=args.artifact_format, + verify=args.verify, ) print(json.dumps(manifest, indent=2, sort_keys=True)) diff --git a/aspec/draft_artifact.py b/aspec/draft_artifact.py index 10e5f17..928c04f 100644 --- a/aspec/draft_artifact.py +++ b/aspec/draft_artifact.py @@ -117,7 +117,9 @@ def load_native_draft(path, *, device="cpu", dtype=None): state = load_file(str(path / "model.safetensors"), device="cpu") if len(state) != manifest.get("tensor_count"): raise ValueError("Native artifact tensor count mismatch") - checkpoint_dtype = state["embed_tokens.weight"].dtype + checkpoint_dtype = ( + state["embed_tokens.weight"].dtype if "embed_tokens.weight" in state + else next(iter(state.values())).dtype) model = AutoEagle3DraftModel.from_config(config, torch_dtype=dtype or checkpoint_dtype) # Establish mixed precision BEFORE load; casting FP32 compressors through # BF16 and back would silently destroy the checkpoint's retained precision. @@ -134,6 +136,22 @@ def load_native_draft(path, *, device="cpu", dtype=None): return model +def verify_native_export(path): + """Strictly reload a native artifact and prove every tensor roundtripped.""" + path = Path(path) + model = load_native_draft(path) + imported = load_file(str(path / "model.safetensors")) + reloaded = model.state_dict() + if set(reloaded) != set(imported): + raise ValueError("Native reload changed the exported tensor set") + for key, value in reloaded.items(): + if value.shape != imported[key].shape or value.dtype != imported[key].dtype: + raise ValueError(f"Native reload changed {key} shape/dtype") + if not torch.equal(value, imported[key]): + raise ValueError(f"Native reload changed {key} values") + return {"method": "strict_load_and_tensor_equality", "tensors": len(reloaded)} + + class BlockDraftConsumer: """One DSpark/DFlash block from verifier features and its frozen output head. diff --git a/tests/cpu/test_medusa_export.py b/tests/cpu/test_medusa_export.py new file mode 100644 index 0000000..fcf0580 --- /dev/null +++ b/tests/cpu/test_medusa_export.py @@ -0,0 +1,84 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Native Medusa export, strict reload and manifest contracts.""" + +import json + +import pytest +import torch + +from aspec.draft_artifact import load_native_draft, verify_native_export +from aspec.models.draft import AutoDraftModelConfig, AutoEagle3DraftModel +from tools import convert_to_hf + +_RAW = { + "architectures": ["MedusaDraftModel"], "model_type": "aspec_medusa", + "hidden_size": 16, "vocab_size": 32, "target_hidden_size": 16, + "target_num_hidden_layers": 4, "medusa_num_heads": 2, "medusa_num_layers": 1, + "tie_word_embeddings": False, +} + + +def _checkpoint(tmp_path, monkeypatch): + config = AutoDraftModelConfig.from_dict(_RAW) + torch.manual_seed(0) + source = AutoEagle3DraftModel.from_config(config, torch_dtype=torch.float32) + checkpoint = tmp_path / "iter_0000002" + (checkpoint / "model").mkdir(parents=True) + (checkpoint / "model" / ".metadata").write_bytes(b"stub") + (checkpoint / "draft_config.json").write_text( + json.dumps(config.to_dict()), encoding="utf-8") + wrapped = { + f"model_state.model.draft_model.{key}": value.detach().clone() + for key, value in source.state_dict().items() + } + monkeypatch.setattr(convert_to_hf, "_load_fsdp_state_dict", lambda _: wrapped) + return source, checkpoint + + +def test_native_export_roundtrips_every_head(tmp_path, monkeypatch): + source, checkpoint = _checkpoint(tmp_path, monkeypatch) + output = tmp_path / "asset" + manifest = convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=output, artifact_format="aspec", verify=True) + + assert manifest["algorithm"] == "medusa" + assert manifest["tensor_count"] == len(source.state_dict()) + assert manifest["verified"]["tensors"] == len(source.state_dict()) + assert manifest["draft_structure"]["loader_contract"]["runtime"] == "aspec" + assert "candidate-tree acceptance" in manifest["draft_structure"]["loader_contract"]["scope"] + + exported_config = json.loads((output / "config.json").read_text()) + assert exported_config["architectures"] == ["MedusaDraftModel"] + assert exported_config["target_layer_ids"] == [3] + assert exported_config["num_aux_hidden_states"] == 1 + assert exported_config["draft_vocab_size"] == 32 + + imported = load_native_draft(output) + assert len(imported.medusa_head) == 2 + for key, value in source.state_dict().items(): + assert torch.equal(imported.state_dict()[key], value) + + +def test_verify_detects_a_changed_tensor(tmp_path, monkeypatch): + _, checkpoint = _checkpoint(tmp_path, monkeypatch) + output = tmp_path / "asset" + convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=output, artifact_format="aspec") + + from safetensors.torch import load_file, save_file + + tensors = load_file(str(output / "model.safetensors")) + first = next(iter(tensors)) + tensors[first] = tensors[first] + 1 + save_file(tensors, str(output / "model.safetensors")) + with pytest.raises(ValueError, match="checksum mismatch"): + verify_native_export(output) + + +def test_sglang_format_is_rejected_for_medusa(tmp_path, monkeypatch): + _, checkpoint = _checkpoint(tmp_path, monkeypatch) + with pytest.raises(NotImplementedError, match="format aspec"): + convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=tmp_path / "serving", artifact_format="sglang") From d576289a1703475de59fcb5f9e296e247b7742a6 Mon Sep 17 00:00:00 2001 From: bohaotu Date: Thu, 19 Nov 2026 15:25:42 +0800 Subject: [PATCH 3/8] feat(medusa): report per-head held-out observations Add evaluate_medusa, which reloads a native heads-only asset and aggregates token-weighted per-head CE/accuracy over an offline eval split, recording the artifact and dataset identities with the report. --- aspec/commands/evaluate_medusa.py | 118 ++++++++++++++++++++++++++++ tests/cpu/test_medusa_evaluation.py | 92 ++++++++++++++++++++++ tools/evaluate_medusa.py | 13 +++ 3 files changed, 223 insertions(+) create mode 100644 aspec/commands/evaluate_medusa.py create mode 100644 tests/cpu/test_medusa_evaluation.py create mode 100644 tools/evaluate_medusa.py diff --git a/aspec/commands/evaluate_medusa.py b/aspec/commands/evaluate_medusa.py new file mode 100644 index 0000000..9619bc9 --- /dev/null +++ b/aspec/commands/evaluate_medusa.py @@ -0,0 +1,118 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Report per-head held-out loss and token accuracy for an exported Medusa Draft.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import torch + +from aspec.data.utils import unpack_loss_mask +from aspec.draft_artifact import load_native_draft +from aspec.models.medusa import MedusaModel + + +def _record_batch(record): + ids = record["input_ids"].reshape(-1).long() + hidden = record["hidden_states"] + hidden = hidden.reshape(-1, hidden.shape[-1]) + packed = record.get("packed_loss_mask") + if packed is None: + raise ValueError("Held-out Medusa records require packed_loss_mask") + mask = packed.reshape(-1) if isinstance(packed, torch.Tensor) else unpack_loss_mask(packed) + if ids.numel() == 0 or hidden.shape[0] != ids.numel() or mask.numel() != ids.numel(): + raise ValueError("Held-out record tensors disagree on sequence length") + return ids.unsqueeze(0), hidden.unsqueeze(0), mask.unsqueeze(0) + + +def evaluate_records(draft, records, *, device="cpu"): + """Aggregate each head's token-weighted CE and accuracy over held-out records.""" + model = MedusaModel(draft).to(device).eval() + dtype = next(model.parameters()).dtype + heads = draft.medusa_num_heads + loss_sums = torch.zeros(heads, dtype=torch.float64) + correct = torch.zeros(heads, dtype=torch.float64) + counts = torch.zeros(heads, dtype=torch.long) + samples = 0 + with torch.no_grad(): + for record in records: + ids, hidden, mask = _record_batch(record) + ids, hidden, mask = ids.to(device), hidden.to(device, dtype), mask.to(device) + losses, _, accuracy, head_counts, _ = model( + ids, hidden, mask, torch.ones_like(ids)) + for head in range(heads): + tokens = int(head_counts[head]) + if tokens == 0: + continue + loss_sums[head] += float(losses[head]) * tokens + correct[head] += float(accuracy[head]) * tokens + counts[head] += tokens + samples += 1 + if samples == 0: + raise ValueError("No held-out records were evaluated") + per_head = [] + for head in range(heads): + tokens = int(counts[head]) + per_head.append({ + "head": head, + "prediction_offset": head + 2, + "tokens": tokens, + "loss": float(loss_sums[head]) / tokens if tokens else None, + "accuracy": float(correct[head]) / tokens if tokens else None, + }) + measured = [item for item in per_head if item["tokens"]] + return { + "samples": samples, + "per_head": per_head, + "aggregate": { + "sum_of_head_mean_ce": ( + sum(item["loss"] for item in measured) if measured else None), + "measured_heads": len(measured), + }, + } + + +def _parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--draft", type=Path, required=True) + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--split", default="eval") + parser.add_argument("--limit", type=int, default=None) + parser.add_argument("--device", default="cpu") + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def main(): + args = _parse_args() + from aspec.offline.dataset import OfflineDataset + + dataset = OfflineDataset(args.dataset) + rows = dataset.rows(args.split) + if args.limit is not None: + if args.limit < 1: + raise ValueError("--limit must be positive") + rows = rows[: args.limit] + draft = load_native_draft(args.draft, device=args.device) + report = evaluate_records( + draft, (dataset.load(row) for row in rows), device=args.device) + report.update({ + "draft_manifest_sha256": hashlib.sha256( + (args.draft / "export_manifest.json").read_bytes()).hexdigest(), + "dataset_identity": dataset.identity, + "split": args.split, + "medusa_num_heads": draft.medusa_num_heads, + "medusa_num_layers": draft.medusa_num_layers, + }) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + print(json.dumps(report, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tests/cpu/test_medusa_evaluation.py b/tests/cpu/test_medusa_evaluation.py new file mode 100644 index 0000000..d547989 --- /dev/null +++ b/tests/cpu/test_medusa_evaluation.py @@ -0,0 +1,92 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Per-head held-out evaluation for exported Medusa heads.""" + +import math + +import pytest +import torch +from torch.nn import functional as F + +from aspec.commands.evaluate_medusa import _record_batch, evaluate_records +from aspec.models.draft import AutoDraftModelConfig, AutoEagle3DraftModel + + +def _draft(): + config = AutoDraftModelConfig.from_dict({ + "architectures": ["MedusaDraftModel"], "model_type": "aspec_medusa", + "hidden_size": 8, "vocab_size": 12, "target_hidden_size": 8, + "target_num_hidden_layers": 4, "medusa_num_heads": 2, "medusa_num_layers": 1, + }) + torch.manual_seed(1) + return AutoEagle3DraftModel.from_config(config, torch_dtype=torch.float32) + + +def _record(ids, hidden, packed): + return {"input_ids": ids, "hidden_states": hidden, "packed_loss_mask": packed} + + +def _expected(draft, ids, hidden, mask): + rows = [] + for head in range(draft.medusa_num_heads): + offset = head + 2 + keep = mask[offset:].bool() + if not keep.any(): + rows.append((0, None, None)) + continue + logits = draft.medusa_head[head](hidden[:-offset].float())[keep] + targets = ids[offset:][keep] + rows.append(( + int(keep.sum()), + F.cross_entropy(logits, targets, reduction="mean").item(), + (logits.argmax(-1) == targets).float().mean().item(), + )) + return rows + + +def _assert_report(report, expected_rows): + assert report["samples"] == len(expected_rows) + measured = [] + for item, rows in zip(report["per_head"], zip(*expected_rows), strict=True): + tokens = sum(row[0] for row in rows) + assert item["tokens"] == tokens + if tokens == 0: + assert item["loss"] is None and item["accuracy"] is None + continue + loss = sum(count * value for count, value, _ in rows) / tokens + accuracy = sum(count * value for count, _, value in rows) / tokens + assert math.isclose(item["loss"], loss, rel_tol=1e-6, abs_tol=1e-7) + assert math.isclose(item["accuracy"], accuracy, rel_tol=1e-6, abs_tol=1e-7) + measured.append(item["loss"]) + assert math.isclose( + report["aggregate"]["sum_of_head_mean_ce"], sum(measured), rel_tol=1e-6) + + +def test_per_head_metrics_are_token_weighted_across_records(): + draft = _draft() + torch.manual_seed(2) + ids_a = torch.randint(0, 12, (7,)) + hidden_a = torch.randn(7, 8) + mask_a = torch.tensor([0, 1, 1, 1, 1, 0, 1]) + ids_b = torch.randint(0, 12, (6,)) + hidden_b = torch.randn(6, 8) + records = [ + _record(ids_a, hidden_a, mask_a), + _record(ids_b, hidden_b, [1, 4, 1]), # packed: 0,1,1,1,1,0 + ] + report = evaluate_records(draft, records) + _assert_report(report, [ + _expected(draft, ids_a, hidden_a, mask_a), + _expected(draft, ids_b, hidden_b, torch.tensor([0, 1, 1, 1, 1, 0])), + ]) + + +def test_record_validation_rejects_missing_mask_and_length_drift(): + with pytest.raises(ValueError, match="packed_loss_mask"): + _record_batch({"input_ids": torch.arange(4), "hidden_states": torch.randn(4, 8)}) + with pytest.raises(ValueError, match="sequence length"): + _record_batch({ + "input_ids": torch.arange(4), "hidden_states": torch.randn(5, 8), + "packed_loss_mask": torch.ones(4), + }) diff --git a/tools/evaluate_medusa.py b/tools/evaluate_medusa.py new file mode 100644 index 0000000..4366a28 --- /dev/null +++ b/tools/evaluate_medusa.py @@ -0,0 +1,13 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Compatibility entry point for the installed evaluate_medusa command.""" + +import sys +from importlib import import_module + +_command = import_module("aspec.commands.evaluate_medusa") +if __name__ == "__main__": + _command.main() +else: + sys.modules[__name__] = _command From e7e6bb2a7b4ababe6199c8827634616849d54736 Mon Sep 17 00:00:00 2001 From: bohaotu Date: Thu, 19 Nov 2026 15:29:04 +0800 Subject: [PATCH 4/8] feat(medusa): add maintained recipe and export/eval docs Register the single-node 1T+1D Medusa recipe, document native export with strict reload plus per-head held-out reporting, and state the remaining candidate-tree acceptance boundary in the support matrix. --- configs/recipes/medusa_qwen3_8b.yaml | 77 ++++++++++++++++++++++++++++ docs/support.md | 2 +- docs/user-guide/algorithms.md | 6 +-- docs/user-guide/medusa.md | 42 +++++++++++++-- docs/user-guide/recipes.md | 10 ++-- tests/cpu/test_release_recipes.py | 2 +- tools/recipe_catalog.json | 11 ++++ 7 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 configs/recipes/medusa_qwen3_8b.yaml diff --git a/configs/recipes/medusa_qwen3_8b.yaml b/configs/recipes/medusa_qwen3_8b.yaml new file mode 100644 index 0000000..8302ba8 --- /dev/null +++ b/configs/recipes/medusa_qwen3_8b.yaml @@ -0,0 +1,77 @@ +# Release training starting point: three passes, held-out eval and checkpoints. +# Hyperparameters are not a convergence/quality claim; see docs/user-guide/recipes.md. +# Formal single-node heads-only Medusa training, resume and native export recipe. +model: + target_model_path: ${oc.env:ASPEC_TARGET_MODEL,Qwen/Qwen3-8B} + native_context_length: 32768 + trust_remote_code: false + draft_model_config: aspec/config/medusa_qwen3_8b_draft_config.json +dataset: + eval_interval: 100 + eval_micro_batch_size: 1 + train_data_path: ${oc.env:ASPEC_TRAIN_DATA} + eval_data_path: ${oc.env:ASPEC_EVAL_DATA} + chat_template: qwen3-instruct + prompt_key: conversations + min_loss_tokens: 1 + shuffle_dataset: true + dataset_seed: 42 +training: + attention_backend: sdpa + draft_context_parallel_size: 1 + micro_batch_size: 1 + draft_accumulation_steps: 4 + learning_rate: 6e-4 + min_lr: 6e-5 + weight_decay: 0.0 + max_grad_norm: 1.0 + max_seq_length: 4096 + num_epochs: 3 + warmup_ratio: 0.04 + seed: 42 + continual_training: false + training_num_gpus_per_node: 1 + training_num_nodes: 1 + placement_strategy: inference_first + fsdp_strategy: REPLICATE + fsdp_reduce_dtype: bfloat16 + compile_model: false + max_concurrent_batches: 1 + streaming_dispatch_with_training: true + pipeline_dispatch_with_training: true + prefetch_depth: 1 + save_interval: 100 + save_per_epoch: false + save_final_checkpoint: true + max_checkpoints: 2 + gradient_checkpointing: true + dflash_packing: false +inference: + inference_engine_type: sgl + store_last_hidden_states: false + target_prefill_memory_optimized: false + inference_num_gpus: 1 + inference_num_gpus_per_engine: 1 + inference_num_gpus_per_node: 2 + inference_fetch_batch: 8 + inference_buffer_threshold: 2 + max_sample_pool_size: 12 + inference_batch_size: 2 + sglang: + tp_size: 1 + mem_fraction_static: 0.5 +mooncake: + data_plane: store + protocol: tcp + require_nvlink: false + allow_host_fallback: false + max_seq_len: 4096 + hidden_dim: 4096 + global_segment_size: 32GB + local_buffer_size: 2GB + host_buffer_size: 2GB + async_put_pool_size: 8 +logging: + report_to: none +output_dir: outputs/qwen3-8b-medusa +cache_dir: cache/qwen3-8b-medusa \ No newline at end of file diff --git a/docs/support.md b/docs/support.md index c5f82d0..6ee9d35 100644 --- a/docs/support.md +++ b/docs/support.md @@ -13,7 +13,7 @@ and tests do not imply CPU training support. | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | | Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training, `lambda_base` curriculum, held-out corrected/backbone observations and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; native export and consumer acceptance pending; see [scope](user-guide/domino.md) | -| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery; SDPA CP1, MBS1, non-packing, REPLICATE | No packing/CP, warmstart, export or consumer acceptance yet; see [scope](user-guide/medusa.md) | +| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery and native export; SDPA CP1, MBS1, non-packing, REPLICATE, one node | No packing/CP, warmstart or candidate-tree acceptance yet; see [scope](user-guide/medusa.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | | JetSpec core | Qwen3-style causal parallel Draft head and temperature-scaled forward-KL objective, reused DFlash block machinery; chunked full-vocabulary KL and explicit causal visibility | Model/loss prerequisite only; no trainer selection, tree builder, recovery, export or consumer validation; CPU causal packing and direct unmasked Draft calls fail closed; see [scope](user-guide/jetspec.md) | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index 9416f91..40281c9 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -164,6 +164,6 @@ post-norm final state; head `i` at position `t` is supervised by token `t+2+i`. the sum of per-head mean cross-entropies (assistant tokens only): each head's CE is averaged within a microbatch, and microbatches are weighted equally by gradient accumulation rather than pooled by token count. The heads own trainable full-vocabulary projections. Start from -`configs/examples/medusa_qwen3_8b_4k_tcp.yaml` (SDPA, CP1, MBS1, non-packing, REPLICATE, -Store/TCP). Export, a maintained recipe, per-head held-out observations and candidate-tree -acceptance remain future work; see [details](medusa.md). +`configs/recipes/medusa_qwen3_8b.yaml` (SDPA, CP1, MBS1, non-packing, REPLICATE, Store/TCP). +Native export, strict reload and per-head held-out reporting are implemented; candidate-tree +acceptance remains future work; see [details](medusa.md). diff --git a/docs/user-guide/medusa.md b/docs/user-guide/medusa.md index 8e4c1f0..915041a 100644 --- a/docs/user-guide/medusa.md +++ b/docs/user-guide/medusa.md @@ -8,6 +8,8 @@ loss landed first; this page describes the current training scope. ## Reference and configuration +- Recipe: `configs/recipes/medusa_qwen3_8b.yaml` (three passes, held-out eval, checkpoints); use + `aspec recipes plan medusa` / `aspec recipes train medusa`. - Draft config: `aspec/config/medusa_qwen3_8b_draft_config.json` (Qwen3-8B Target, `K=5` heads). - Example: `configs/examples/medusa_qwen3_8b_4k_tcp.yaml` (one Target GPU + one Draft GPU, 4K, SDPA, CP1, non-packing, REPLICATE, Store/TCP transport; no NVLink required). Set @@ -15,7 +17,9 @@ loss landed first; this page describes the current training scope. `model.target_model_path`) and run from the repository root so the relative draft-config path resolves. - The heads are the only trainable parameters; the Target is frozen and its post-norm final - state is the only feature the Draft consumes. + state is the only feature the Draft consumes. At startup the trainer resolves the Target + config and rejects a draft projection width that differs from its `vocab_size`, including for + Hub IDs that Plan could not read locally. ## Inputs and prediction semantics @@ -44,5 +48,37 @@ The current path fails closed outside the following: - `fsdp_strategy: REPLICATE`, one node. Anchored training, MTP conditioning and whole-model compile are rejected. -Export, a maintained recipe, per-head held-out observations and candidate-tree acceptance are not -part of this scope. +## Export and strict reload + +Export a trained checkpoint as an ASpec-native asset: + +```bash +python -m tools.convert_to_hf \ + --input-dir outputs/qwen3-8b-medusa/checkpoints/iter_0000001 \ + --output-dir outputs/medusa-export --format aspec --verify +``` + +`--format aspec` is required: the SGLang serving format is rejected because no verified +candidate-tree Medusa loader exists. The artifact contains `config.json`, +`model.safetensors` and `export_manifest.json`; `--verify` reloads it through +`aspec.draft_artifact.load_native_draft` (strict, no dropped tensors) and compares every tensor +with the exported bytes. + +## Held-out per-head observations + +The training eval loop reports `eval/ploss_{i}` and `eval/acc_{i}` when `dataset.eval_data_path` +is set. To report the same observations from an exported asset against a materialized offline +feature dataset: + +```bash +python -m tools.evaluate_medusa \ + --draft outputs/medusa-export --dataset /path/to/offline-dataset \ + --split eval --output outputs/medusa-eval.json +``` + +The report records the artifact and dataset identities, per-head token counts, token-weighted +mean CE and token accuracy, and the sum of the per-head means. These are held-out training +observations, not candidate-tree acceptance. + +Candidate-tree acceptance and decode-equality under a Medusa evaluator are not part of this +scope; they require a verified consumer adapter. diff --git a/docs/user-guide/recipes.md b/docs/user-guide/recipes.md index 68cb315..0a1c3bc 100644 --- a/docs/user-guide/recipes.md +++ b/docs/user-guide/recipes.md @@ -3,10 +3,11 @@ Choose a recipe for your Target, Draft algorithm and available resources. EAGLE4 uses two successive stages. See [supported combinations](../support.md) before combining algorithms and structures. -Qwen and Ling DSpark/DFlash paths are the main optimization and recipe priorities. EAGLE3/4 and -native MTP have their own documented training and asset contracts; DFlash2 remains an experimental -example outside this recipe catalog. A listed recipe is a maintained starting configuration, not -a claim that every algorithm/structure combination has the same quality or performance evidence. +Qwen and Ling DSpark/DFlash paths are the main optimization and recipe priorities. EAGLE3/4, +native MTP and Medusa heads have their own documented training and asset contracts; DFlash2 +remains an experimental example outside this recipe catalog. A listed recipe is a maintained +starting configuration, not a claim that every algorithm/structure combination has the same +quality or performance evidence. The table shows each recipe's configured starting allocation, not a minimum requirement or optimal Target/Draft split. T = Target GPU; D = Draft GPU. Check memory fit and the required node-local NVLink @@ -17,6 +18,7 @@ connections on your hardware before training. | `quickstart` | Qwen3-8B / DSpark | 1 node, 2 GPUs: 1T + 1D; 1K | Try training, recovery and export with bundled data | | `dspark` | Qwen3-8B / Dense DSpark | 1 node, 8 GPUs: 4T + 4D; 4K | Train on your data with packing and NVLink | | `dflash` | Qwen3-8B / DFlash | 1 node, 3 GPUs: 1T + 2D; 4K | Train a block-parallel DFlash Draft | +| `medusa` | Qwen3-8B / heads-only Medusa | 1 node, 2 GPUs: 1T + 1D; 4K | Train parallel future-token heads and export a native heads asset | | `eagle3` | Qwen3-8B / EAGLE3 | 1 node, 3 GPUs: 1T + 2D; 4K | Train a recurrent EAGLE3 Draft | | `eagle4` | Qwen3-8B / EAGLE4 | 1 node, 3 GPUs: 1T + 2D; 4K; sequential stages | Train a Draft, then candidate retrieval projections | | `dmtp` | Qwen3.5-9B / MTP-conditioned DSpark | 1 node, 8 GPUs: 4T + 4D; 4K, CP1 | Train a Draft conditioned on the frozen native predictor; native assets | diff --git a/tests/cpu/test_release_recipes.py b/tests/cpu/test_release_recipes.py index 59674f6..4deb160 100644 --- a/tests/cpu/test_release_recipes.py +++ b/tests/cpu/test_release_recipes.py @@ -83,7 +83,7 @@ def test_qwen8b_recipes_plan_without_target_override(name, monkeypatch): def test_catalog_exactly_covers_formal_files(): entries = recipes.catalog() - assert len(entries) == 13 + assert len(entries) == 14 declared = {p for entry in entries.values() for p in entry["configs"]} assert declared == { str(p.relative_to(recipes.ROOT)) for p in (recipes.ROOT / "configs/recipes").glob("*.yaml") diff --git a/tools/recipe_catalog.json b/tools/recipe_catalog.json index 6c0873f..877e2e1 100644 --- a/tools/recipe_catalog.json +++ b/tools/recipe_catalog.json @@ -146,5 +146,16 @@ "target_gpus": 4, "draft_gpus": 4, "native_only_export": true + }, + "medusa": { + "configs": [ + "configs/recipes/medusa_qwen3_8b.yaml" + ], + "model": "Qwen3-8B", + "nodes": 1, + "gpus": 2, + "target_gpus": 1, + "draft_gpus": 1, + "native_only_export": true } } From 0370592e0c61c2da6d8672db97364c6d76baa6aa Mon Sep 17 00:00:00 2001 From: bohaotu Date: Thu, 19 Nov 2026 21:45:10 +0800 Subject: [PATCH 5/8] fix(medusa): verify native exports against the exported tensors verify_native_export re-read the artifact it had just loaded, so a loader-side cast or drop could never fail --verify and an artifact whose bytes and manifest agreed but whose values differed still reported success. Compare the reload with the tensors the export wrote, reject --verify before a SGLang export writes files, and cover both with tests. Also correct the documented checkpoint name to the recipe's save interval. --- aspec/commands/convert_to_hf.py | 6 ++-- aspec/draft_artifact.py | 19 ++++++++---- docs/user-guide/medusa.md | 6 ++-- tests/cpu/test_medusa_export.py | 51 +++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 12 deletions(-) diff --git a/aspec/commands/convert_to_hf.py b/aspec/commands/convert_to_hf.py index c2072ff..aed4360 100644 --- a/aspec/commands/convert_to_hf.py +++ b/aspec/commands/convert_to_hf.py @@ -364,6 +364,8 @@ def convert_checkpoint( raw_config = json.loads(config_path.read_text(encoding="utf-8")) if artifact_format not in {"sglang", "aspec"}: raise ValueError("artifact_format must be sglang or aspec") + if verify and artifact_format != "aspec": + raise ValueError("--verify requires --format aspec") if resolve_draft_algorithm(raw_config) in {"mtp", "medusa"}: raw_config = AutoDraftModelConfig.from_dict(raw_config).to_dict() training_contract = _checkpoint_training_contract(model_dir, raw_config) @@ -509,11 +511,9 @@ def convert_checkpoint( encoding="utf-8", ) if verify: - if artifact_format != "aspec": - raise ValueError("--verify requires --format aspec") from aspec.draft_artifact import verify_native_export - manifest["verified"] = verify_native_export(output_dir) + manifest["verified"] = verify_native_export(output_dir, expected=tensors) (output_dir / "export_manifest.json").write_text( json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8", diff --git a/aspec/draft_artifact.py b/aspec/draft_artifact.py index 928c04f..5e52659 100644 --- a/aspec/draft_artifact.py +++ b/aspec/draft_artifact.py @@ -136,18 +136,25 @@ def load_native_draft(path, *, device="cpu", dtype=None): return model -def verify_native_export(path): - """Strictly reload a native artifact and prove every tensor roundtripped.""" +def verify_native_export(path, *, expected=None): + """Strictly reload a native artifact and compare it with the exported values. + + ``expected`` is the state that was written; comparing the reloaded model with + it catches loader-side casts or drops. Without it the artifact's own bytes are + re-read, which only proves the loader is idempotent, not that it roundtripped + the exported tensors. + """ path = Path(path) model = load_native_draft(path) - imported = load_file(str(path / "model.safetensors")) reloaded = model.state_dict() - if set(reloaded) != set(imported): + if expected is None: + expected = load_file(str(path / "model.safetensors")) + if set(reloaded) != set(expected): raise ValueError("Native reload changed the exported tensor set") for key, value in reloaded.items(): - if value.shape != imported[key].shape or value.dtype != imported[key].dtype: + if value.shape != expected[key].shape or value.dtype != expected[key].dtype: raise ValueError(f"Native reload changed {key} shape/dtype") - if not torch.equal(value, imported[key]): + if not torch.equal(value, expected[key]): raise ValueError(f"Native reload changed {key} values") return {"method": "strict_load_and_tensor_equality", "tensors": len(reloaded)} diff --git a/docs/user-guide/medusa.md b/docs/user-guide/medusa.md index 915041a..186a6c9 100644 --- a/docs/user-guide/medusa.md +++ b/docs/user-guide/medusa.md @@ -54,15 +54,15 @@ Export a trained checkpoint as an ASpec-native asset: ```bash python -m tools.convert_to_hf \ - --input-dir outputs/qwen3-8b-medusa/checkpoints/iter_0000001 \ + --input-dir outputs/qwen3-8b-medusa/checkpoints/iter_00000100 \ --output-dir outputs/medusa-export --format aspec --verify ``` `--format aspec` is required: the SGLang serving format is rejected because no verified candidate-tree Medusa loader exists. The artifact contains `config.json`, `model.safetensors` and `export_manifest.json`; `--verify` reloads it through -`aspec.draft_artifact.load_native_draft` (strict, no dropped tensors) and compares every tensor -with the exported bytes. +`aspec.draft_artifact.load_native_draft` (strict, no dropped tensors) and compares every +reloaded tensor with the values the export wrote. ## Held-out per-head observations diff --git a/tests/cpu/test_medusa_export.py b/tests/cpu/test_medusa_export.py index fcf0580..e361c5c 100644 --- a/tests/cpu/test_medusa_export.py +++ b/tests/cpu/test_medusa_export.py @@ -77,6 +77,57 @@ def test_verify_detects_a_changed_tensor(tmp_path, monkeypatch): verify_native_export(output) +def test_verify_compares_reload_with_the_exported_tensors(tmp_path, monkeypatch): + source, checkpoint = _checkpoint(tmp_path, monkeypatch) + output = tmp_path / "asset" + convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=output, artifact_format="aspec") + + from safetensors.torch import load_file, save_file + + tensors = load_file(str(output / "model.safetensors")) + first = next(iter(tensors)) + tensors[first] = tensors[first] + 1 + save_file(tensors, str(output / "model.safetensors")) + # Keep the artifact internally consistent so only the exported values differ. + manifest_path = output / "export_manifest.json" + recorded = json.loads(manifest_path.read_text()) + recorded["model_sha256"] = convert_to_hf._sha256_file(output / "model.safetensors") + manifest_path.write_text( + json.dumps(recorded, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="changed .* values"): + verify_native_export(output, expected=source.state_dict()) + + +def test_verify_requires_native_format_before_writing(tmp_path, monkeypatch): + _, checkpoint = _checkpoint(tmp_path, monkeypatch) + output = tmp_path / "serving" + with pytest.raises(ValueError, match="--verify requires --format aspec"): + convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=output, artifact_format="sglang", verify=True) + assert not output.exists() + + +def test_export_verify_compares_with_the_tensors_it_wrote(tmp_path, monkeypatch): + source, checkpoint = _checkpoint(tmp_path, monkeypatch) + from aspec import draft_artifact + + captured = {} + + def fake_verify(path, *, expected=None): + captured["expected"] = expected + return {"method": "stub", "tensors": len(expected or {})} + + monkeypatch.setattr(draft_artifact, "verify_native_export", fake_verify) + manifest = convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=tmp_path / "asset", artifact_format="aspec", + verify=True) + + assert set(captured["expected"]) == set(source.state_dict()) + assert manifest["verified"] == {"method": "stub", "tensors": len(source.state_dict())} + + def test_sglang_format_is_rejected_for_medusa(tmp_path, monkeypatch): _, checkpoint = _checkpoint(tmp_path, monkeypatch) with pytest.raises(NotImplementedError, match="format aspec"): From da2f31dfd00907ff3a6e3c48bc3378f92d86f30e Mon Sep 17 00:00:00 2001 From: bohaotu Date: Thu, 19 Nov 2026 21:47:47 +0800 Subject: [PATCH 6/8] docs(medusa): use the produced checkpoint path in the export example --- docs/user-guide/medusa.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/medusa.md b/docs/user-guide/medusa.md index 186a6c9..eaff9e0 100644 --- a/docs/user-guide/medusa.md +++ b/docs/user-guide/medusa.md @@ -54,7 +54,7 @@ Export a trained checkpoint as an ASpec-native asset: ```bash python -m tools.convert_to_hf \ - --input-dir outputs/qwen3-8b-medusa/checkpoints/iter_00000100 \ + --input-dir outputs/qwen3-8b-medusa/checkpoints/iter_0000101 \ --output-dir outputs/medusa-export --format aspec --verify ``` From 20abaf9b5f737c8ce1a8b416b1acb71e09ed40c8 Mon Sep 17 00:00:00 2001 From: bohaotu Date: Thu, 19 Nov 2026 22:31:53 +0800 Subject: [PATCH 7/8] fix(medusa): harden evaluation binding and depth-shared export Require the offline dataset binding to carry the final Target post-norm state, reject non-finite report metrics, and clone repeated residual-block storage so safetensors accepts depth>1 exports. Document the recipe output directory. --- aspec/commands/convert_to_hf.py | 22 +++++++++-- aspec/commands/evaluate_medusa.py | 60 ++++++++++++++++++++++++++--- docs/user-guide/medusa.md | 2 +- tests/cpu/test_medusa_evaluation.py | 41 +++++++++++++++++++- tests/cpu/test_medusa_export.py | 24 +++++++++++- 5 files changed, 135 insertions(+), 14 deletions(-) diff --git a/aspec/commands/convert_to_hf.py b/aspec/commands/convert_to_hf.py index aed4360..4059cef 100644 --- a/aspec/commands/convert_to_hf.py +++ b/aspec/commands/convert_to_hf.py @@ -105,6 +105,23 @@ def _aspec_version() -> str: return "unknown" +def _export_tensors(serving_state): + """Detach state tensors, cloning repeated storage so safetensors accepts it.""" + tensors = {} + seen_storage = set() + for key, value in serving_state.items(): + tensor = value.detach().cpu().contiguous() + storage = tensor.untyped_storage().data_ptr() + if storage in seen_storage: + # Shared residual-block depths repeat one module, so several state + # keys alias one storage. Keep every key but give the file real copies. + tensor = tensor.clone() + else: + seen_storage.add(storage) + tensors[key] = tensor + return tensors + + def _detect_model_dir(input_dir: Path) -> Path: model_dir = input_dir / "model" return model_dir if model_dir.is_dir() else input_dir @@ -426,10 +443,7 @@ def convert_checkpoint( model.state_dict_for_serving() if hasattr(model, "state_dict_for_serving") else to_export_keys(model.state_dict()) ) - tensors = { - key: value.detach().cpu().contiguous() - for key, value in serving_state.items() - } + tensors = _export_tensors(serving_state) eagle4_asset = None if resolve_draft_algorithm(raw_config) == "eagle4": diff --git a/aspec/commands/evaluate_medusa.py b/aspec/commands/evaluate_medusa.py index 9619bc9..ad0d848 100644 --- a/aspec/commands/evaluate_medusa.py +++ b/aspec/commands/evaluate_medusa.py @@ -8,19 +8,25 @@ import argparse import hashlib import json +from collections.abc import Mapping from pathlib import Path import torch from aspec.data.utils import unpack_loss_mask from aspec.draft_artifact import load_native_draft +from aspec.feature_contract import validate_feature_contract from aspec.models.medusa import MedusaModel -def _record_batch(record): +def _record_batch(record, *, expected_hidden_size=None): ids = record["input_ids"].reshape(-1).long() hidden = record["hidden_states"] hidden = hidden.reshape(-1, hidden.shape[-1]) + if expected_hidden_size is not None and hidden.shape[-1] != expected_hidden_size: + raise ValueError( + "Held-out hidden width differs from the dataset feature contract: " + f"{hidden.shape[-1]} != {expected_hidden_size}") packed = record.get("packed_loss_mask") if packed is None: raise ValueError("Held-out Medusa records require packed_loss_mask") @@ -30,7 +36,35 @@ def _record_batch(record): return ids.unsqueeze(0), hidden.unsqueeze(0), mask.unsqueeze(0) -def evaluate_records(draft, records, *, device="cpu"): +def _validate_binding(binding): + """Medusa consumes only the Target's final post-norm state from the dataset binding.""" + if not isinstance(binding, Mapping): + raise ValueError("Offline dataset has no Target feature binding") + contract = binding.get("feature_contract") + try: + validate_feature_contract(contract) + except Exception as exc: + raise ValueError("Offline dataset feature contract is invalid") from exc + tensors = contract["tensors"] + hidden = tensors.get("hidden_states") or {} + if (hidden.get("semantic") != "target_final_hidden_state" + or hidden.get("value_stage") != "post_norm"): + raise ValueError( + "Medusa requires the Target's final post-norm hidden state") + depth = contract.get("target_num_hidden_layers") + hidden_size = contract.get("target_hidden_size") + layers = contract.get("aux_hidden_state_layers") + if (not isinstance(depth, int) or depth < 1 + or not isinstance(hidden_size, int) or hidden_size < 1 + or not isinstance(layers, list) or layers != [depth - 1]): + raise ValueError( + "Medusa features must come from the final Target decoder layer") + if "last_hidden_states" in tensors or "mtp_hidden_states" in tensors: + raise ValueError("Medusa does not consume duplicate teacher tensors") + return contract + + +def evaluate_records(draft, records, *, device="cpu", expected_hidden_size=None): """Aggregate each head's token-weighted CE and accuracy over held-out records.""" model = MedusaModel(draft).to(device).eval() dtype = next(model.parameters()).dtype @@ -41,7 +75,8 @@ def evaluate_records(draft, records, *, device="cpu"): samples = 0 with torch.no_grad(): for record in records: - ids, hidden, mask = _record_batch(record) + ids, hidden, mask = _record_batch( + record, expected_hidden_size=expected_hidden_size) ids, hidden, mask = ids.to(device), hidden.to(device, dtype), mask.to(device) losses, _, accuracy, head_counts, _ = model( ids, hidden, mask, torch.ones_like(ids)) @@ -77,6 +112,13 @@ def evaluate_records(draft, records, *, device="cpu"): } +def _serialize_report(report): + try: + return json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + except ValueError as exc: + raise ValueError("Held-out report contains non-finite metrics") from exc + + def _parse_args(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--draft", type=Path, required=True) @@ -93,6 +135,8 @@ def main(): from aspec.offline.dataset import OfflineDataset dataset = OfflineDataset(args.dataset) + binding = dataset.metadata.get("binding") or {} + contract = _validate_binding(binding) rows = dataset.rows(args.split) if args.limit is not None: if args.limit < 1: @@ -100,18 +144,22 @@ def main(): rows = rows[: args.limit] draft = load_native_draft(args.draft, device=args.device) report = evaluate_records( - draft, (dataset.load(row) for row in rows), device=args.device) + draft, (dataset.load(row) for row in rows), device=args.device, + expected_hidden_size=int(contract["target_hidden_size"])) report.update({ "draft_manifest_sha256": hashlib.sha256( (args.draft / "export_manifest.json").read_bytes()).hexdigest(), "dataset_identity": dataset.identity, + "feature_contract_id": contract["contract_id"], + "target_identity": binding.get("target_identity"), "split": args.split, "medusa_num_heads": draft.medusa_num_heads, "medusa_num_layers": draft.medusa_num_layers, }) + serialized = _serialize_report(report) args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") - print(json.dumps(report, indent=2, sort_keys=True)) + args.output.write_text(serialized + "\n") + print(serialized) if __name__ == "__main__": diff --git a/docs/user-guide/medusa.md b/docs/user-guide/medusa.md index eaff9e0..d1d6ef5 100644 --- a/docs/user-guide/medusa.md +++ b/docs/user-guide/medusa.md @@ -9,7 +9,7 @@ loss landed first; this page describes the current training scope. ## Reference and configuration - Recipe: `configs/recipes/medusa_qwen3_8b.yaml` (three passes, held-out eval, checkpoints); use - `aspec recipes plan medusa` / `aspec recipes train medusa`. + `aspec recipes plan medusa` / `aspec recipes train medusa --output outputs/qwen3-8b-medusa`. - Draft config: `aspec/config/medusa_qwen3_8b_draft_config.json` (Qwen3-8B Target, `K=5` heads). - Example: `configs/examples/medusa_qwen3_8b_4k_tcp.yaml` (one Target GPU + one Draft GPU, 4K, SDPA, CP1, non-packing, REPLICATE, Store/TCP transport; no NVLink required). Set diff --git a/tests/cpu/test_medusa_evaluation.py b/tests/cpu/test_medusa_evaluation.py index d547989..d69326b 100644 --- a/tests/cpu/test_medusa_evaluation.py +++ b/tests/cpu/test_medusa_evaluation.py @@ -9,7 +9,13 @@ import torch from torch.nn import functional as F -from aspec.commands.evaluate_medusa import _record_batch, evaluate_records +from aspec.commands.evaluate_medusa import ( + _record_batch, + _serialize_report, + _validate_binding, + evaluate_records, +) +from aspec.feature_contract import build_target_feature_contract from aspec.models.draft import AutoDraftModelConfig, AutoEagle3DraftModel @@ -90,3 +96,36 @@ def test_record_validation_rejects_missing_mask_and_length_drift(): "input_ids": torch.arange(4), "hidden_states": torch.randn(5, 8), "packed_loss_mask": torch.ones(4), }) + + +def _medusa_contract(): + return build_target_feature_contract( + drafter_family="medusa", target_hidden_size=8, target_num_hidden_layers=4, + aux_hidden_state_layers=[3], store_last_hidden_states=False) + + +def test_binding_requires_the_final_post_norm_state(): + contract = _validate_binding({ + "feature_contract": _medusa_contract(), "target_identity": {"config.json": "x"}}) + assert contract["tensors"]["hidden_states"]["value_stage"] == "post_norm" + eagle = build_target_feature_contract( + drafter_family="eagle3", target_hidden_size=8, target_num_hidden_layers=4, + aux_hidden_state_layers=[1, 3], store_last_hidden_states=True) + with pytest.raises(ValueError, match="final post-norm"): + _validate_binding({"feature_contract": eagle}) + + +def test_record_hidden_width_must_match_the_contract(): + with pytest.raises(ValueError, match="hidden width"): + _record_batch({ + "input_ids": torch.arange(4), "hidden_states": torch.randn(4, 8), + "packed_loss_mask": torch.ones(4), + }, expected_hidden_size=16) + + +def test_report_serialization_rejects_non_finite_metrics(): + assert '"loss": null' in _serialize_report({"loss": None}) + with pytest.raises(ValueError, match="non-finite"): + _serialize_report({"loss": float("nan")}) + with pytest.raises(ValueError, match="non-finite"): + _serialize_report({"loss": float("inf")}) diff --git a/tests/cpu/test_medusa_export.py b/tests/cpu/test_medusa_export.py index e361c5c..f46ec09 100644 --- a/tests/cpu/test_medusa_export.py +++ b/tests/cpu/test_medusa_export.py @@ -20,8 +20,8 @@ } -def _checkpoint(tmp_path, monkeypatch): - config = AutoDraftModelConfig.from_dict(_RAW) +def _checkpoint(tmp_path, monkeypatch, raw=_RAW): + config = AutoDraftModelConfig.from_dict(raw) torch.manual_seed(0) source = AutoEagle3DraftModel.from_config(config, torch_dtype=torch.float32) checkpoint = tmp_path / "iter_0000002" @@ -100,6 +100,26 @@ def test_verify_compares_reload_with_the_exported_tensors(tmp_path, monkeypatch) verify_native_export(output, expected=source.state_dict()) +def test_depth_two_shared_blocks_export_and_reload(tmp_path, monkeypatch): + source, checkpoint = _checkpoint( + tmp_path, monkeypatch, {**_RAW, "medusa_num_layers": 2}) + output = tmp_path / "asset" + manifest = convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=output, artifact_format="aspec", verify=True) + assert manifest["verified"]["tensors"] == len(source.state_dict()) + + from safetensors.torch import load_file + + tensors = load_file(str(output / "model.safetensors")) + for head in range(2): + first = tensors[f"medusa_head.{head}.0.linear.weight"] + second = tensors[f"medusa_head.{head}.1.linear.weight"] + assert torch.equal(first, second) + imported = load_native_draft(output) + for key, value in source.state_dict().items(): + assert torch.equal(imported.state_dict()[key], value) + + def test_verify_requires_native_format_before_writing(tmp_path, monkeypatch): _, checkpoint = _checkpoint(tmp_path, monkeypatch) output = tmp_path / "serving" From 301ad905dc377322a6cd401452cfc6cb6e7e1cc7 Mon Sep 17 00:00:00 2001 From: bohaotu Date: Fri, 20 Nov 2026 13:41:26 +0800 Subject: [PATCH 8/8] fix(medusa): bind evaluation features to the trained Target Reject offline datasets whose Target depth or hidden size differs from the draft config, and apply SGLang json_model_override_args in the startup vocabulary check so Plan and trainer agree on the effective Target. --- aspec/commands/evaluate_medusa.py | 14 ++++++++++++++ aspec/training/medusa_trainer.py | 26 +++++++++++++++++++++++--- docs/user-guide/medusa.md | 3 ++- tests/cpu/test_medusa_evaluation.py | 19 +++++++++++++++++++ tests/cpu/test_medusa_integration.py | 17 +++++++++++++++++ 5 files changed, 75 insertions(+), 4 deletions(-) diff --git a/aspec/commands/evaluate_medusa.py b/aspec/commands/evaluate_medusa.py index ad0d848..92f91c7 100644 --- a/aspec/commands/evaluate_medusa.py +++ b/aspec/commands/evaluate_medusa.py @@ -64,6 +64,19 @@ def _validate_binding(binding): return contract +def _validate_target_binding(contract, draft_config): + """The dataset features must come from the Target the Draft was trained against.""" + if contract["target_num_hidden_layers"] != draft_config.target_num_hidden_layers: + raise ValueError( + "Offline dataset Target depth differs from the Draft's training Target: " + f"{contract['target_num_hidden_layers']} != " + f"{draft_config.target_num_hidden_layers}") + if contract["target_hidden_size"] != draft_config.target_hidden_size: + raise ValueError( + "Offline dataset Target hidden size differs from the Draft's training Target: " + f"{contract['target_hidden_size']} != {draft_config.target_hidden_size}") + + def evaluate_records(draft, records, *, device="cpu", expected_hidden_size=None): """Aggregate each head's token-weighted CE and accuracy over held-out records.""" model = MedusaModel(draft).to(device).eval() @@ -143,6 +156,7 @@ def main(): raise ValueError("--limit must be positive") rows = rows[: args.limit] draft = load_native_draft(args.draft, device=args.device) + _validate_target_binding(contract, draft.config) report = evaluate_records( draft, (dataset.load(row) for row in rows), device=args.device, expected_hidden_size=int(contract["target_hidden_size"])) diff --git a/aspec/training/medusa_trainer.py b/aspec/training/medusa_trainer.py index 262ec32..124edf6 100644 --- a/aspec/training/medusa_trainer.py +++ b/aspec/training/medusa_trainer.py @@ -3,6 +3,9 @@ """Heads-only Medusa uses the existing optimizer/checkpoint/ownership training path.""" +import copy +import json +from collections.abc import Mapping from types import SimpleNamespace from aspec.models.medusa import MedusaModel @@ -25,9 +28,26 @@ def _load_algorithm_weights(self, draft_model, target_model_path): config = AutoConfig.from_pretrained( target_model_path, trust_remote_code=getattr(self.args, "trust_remote_code", False), - ) - config = getattr(config, "text_config", config) - target_vocab_size = getattr(config, "vocab_size", None) + ).to_dict() + extra_args = getattr(self.args, "sglang_extra_args", None) or {} + overrides = extra_args.get("json_model_override_args") + if overrides is not None: + if isinstance(overrides, str): + try: + overrides = json.loads(overrides) + except json.JSONDecodeError as exc: + raise ValueError( + "inference.sglang.extra_args.json_model_override_args must be " + f"valid JSON: {exc}") from exc + if not isinstance(overrides, Mapping): + raise ValueError( + "inference.sglang.extra_args.json_model_override_args must contain " + "a JSON object") + config.update(copy.deepcopy(dict(overrides))) + target_vocab_size = config.get("vocab_size") + text_config = config.get("text_config") + if target_vocab_size is None and isinstance(text_config, Mapping): + target_vocab_size = text_config.get("vocab_size") if not isinstance(target_vocab_size, int) or isinstance(target_vocab_size, bool): raise ValueError("Medusa requires the Target config to declare vocab_size") if draft_model.vocab_size != target_vocab_size: diff --git a/docs/user-guide/medusa.md b/docs/user-guide/medusa.md index d1d6ef5..4608b29 100644 --- a/docs/user-guide/medusa.md +++ b/docs/user-guide/medusa.md @@ -19,7 +19,8 @@ loss landed first; this page describes the current training scope. - The heads are the only trainable parameters; the Target is frozen and its post-norm final state is the only feature the Draft consumes. At startup the trainer resolves the Target config and rejects a draft projection width that differs from its `vocab_size`, including for - Hub IDs that Plan could not read locally. + Hub IDs that Plan could not read locally. It applies + `inference.sglang.extra_args.json_model_override_args` exactly as Plan does. ## Inputs and prediction semantics diff --git a/tests/cpu/test_medusa_evaluation.py b/tests/cpu/test_medusa_evaluation.py index d69326b..21b2179 100644 --- a/tests/cpu/test_medusa_evaluation.py +++ b/tests/cpu/test_medusa_evaluation.py @@ -13,6 +13,7 @@ _record_batch, _serialize_report, _validate_binding, + _validate_target_binding, evaluate_records, ) from aspec.feature_contract import build_target_feature_contract @@ -129,3 +130,21 @@ def test_report_serialization_rejects_non_finite_metrics(): _serialize_report({"loss": float("nan")}) with pytest.raises(ValueError, match="non-finite"): _serialize_report({"loss": float("inf")}) + + +def test_binding_must_match_the_draft_target_binding(): + draft = _draft() + contract = _validate_binding({"feature_contract": _medusa_contract()}) + _validate_target_binding(contract, draft.config) + + deeper = build_target_feature_contract( + drafter_family="medusa", target_hidden_size=8, target_num_hidden_layers=6, + aux_hidden_state_layers=[5], store_last_hidden_states=False) + with pytest.raises(ValueError, match="Target depth"): + _validate_target_binding(deeper, draft.config) + + wider = build_target_feature_contract( + drafter_family="medusa", target_hidden_size=16, target_num_hidden_layers=4, + aux_hidden_state_layers=[3], store_last_hidden_states=False) + with pytest.raises(ValueError, match="hidden size"): + _validate_target_binding(wider, draft.config) diff --git a/tests/cpu/test_medusa_integration.py b/tests/cpu/test_medusa_integration.py index 87d8eb4..37a4be2 100644 --- a/tests/cpu/test_medusa_integration.py +++ b/tests/cpu/test_medusa_integration.py @@ -338,6 +338,23 @@ def test_target_vocabulary_is_bound_at_startup(tmp_path): trainer._load_algorithm_weights(draft, str(mismatched)) +def test_startup_vocabulary_honors_sglang_overrides(tmp_path): + from aspec.training.medusa_trainer import MedusaTrainer + + trainer = object.__new__(MedusaTrainer) + trainer.args = SimpleNamespace(trust_remote_code=False, sglang_extra_args={}) + draft = MedusaDraftModel(_tiny_config()) + target = str(_fake_qwen_target(tmp_path, vocab_size=33)) + with pytest.raises(ValueError, match="full Target vocabulary"): + trainer._load_algorithm_weights(draft, target) + for overrides in ('{"vocab_size": 32}', {"vocab_size": 32}): + trainer.args.sglang_extra_args = {"json_model_override_args": overrides} + trainer._load_algorithm_weights(draft, target) + trainer.args.sglang_extra_args = {"json_model_override_args": {"vocab_size": 34}} + with pytest.raises(ValueError, match="full Target vocabulary"): + trainer._load_algorithm_weights(draft, target) + + def test_metric_semantics_drop_unmeasured_acceptance(): from aspec.training.medusa_trainer import MedusaTrainer