diff --git a/aspec/commands/convert_to_hf.py b/aspec/commands/convert_to_hf.py index d71b58f..4059cef 100644 --- a/aspec/commands/convert_to_hf.py +++ b/aspec/commands/convert_to_hf.py @@ -105,6 +105,23 @@ def _aspec_version() -> str: return "unknown" +def _export_tensors(serving_state): + """Detach state tensors, cloning repeated storage so safetensors accepts it.""" + tensors = {} + seen_storage = set() + for key, value in serving_state.items(): + tensor = value.detach().cpu().contiguous() + storage = tensor.untyped_storage().data_ptr() + if storage in seen_storage: + # Shared residual-block depths repeat one module, so several state + # keys alias one storage. Keep every key but give the file real copies. + tensor = tensor.clone() + else: + seen_storage.add(storage) + tensors[key] = tensor + return tensors + + def _detect_model_dir(input_dir: Path) -> Path: model_dir = input_dir / "model" return model_dir if model_dir.is_dir() else input_dir @@ -166,6 +183,11 @@ def _serving_config(raw_config: dict) -> dict: if raw_config.get("mtp_conditioning"): raise NotImplementedError("MTP-conditioned Drafts require native --format aspec export") + if raw_config.get("architectures") == ["MedusaDraftModel"]: + raise NotImplementedError( + "Medusa heads require native --format aspec export; " + "SGLang has no verified candidate-tree Medusa loader") + if raw_config.get("architectures") == ["NativeMTPDraftModel"]: native = json.loads(json.dumps(raw_config["native_config"])) native["architectures"] = ["Qwen3_5ForCausalLMMTP"] @@ -343,6 +365,7 @@ def convert_checkpoint( output_dtype: torch.dtype | None = None, force: bool = False, artifact_format: str = "sglang", + verify: bool = False, ) -> dict: input_dir = input_dir.expanduser().resolve() model_dir = _detect_model_dir(input_dir) @@ -358,7 +381,9 @@ def convert_checkpoint( raw_config = json.loads(config_path.read_text(encoding="utf-8")) if artifact_format not in {"sglang", "aspec"}: raise ValueError("artifact_format must be sglang or aspec") - if resolve_draft_algorithm(raw_config) == "mtp": + if verify and artifact_format != "aspec": + raise ValueError("--verify requires --format aspec") + if resolve_draft_algorithm(raw_config) in {"mtp", "medusa"}: raw_config = AutoDraftModelConfig.from_dict(raw_config).to_dict() training_contract = _checkpoint_training_contract(model_dir, raw_config) export_config = (_serving_config(raw_config) if artifact_format == "sglang" @@ -418,10 +443,7 @@ def convert_checkpoint( model.state_dict_for_serving() if hasattr(model, "state_dict_for_serving") else to_export_keys(model.state_dict()) ) - tensors = { - key: value.detach().cpu().contiguous() - for key, value in serving_state.items() - } + tensors = _export_tensors(serving_state) eagle4_asset = None if resolve_draft_algorithm(raw_config) == "eagle4": @@ -469,7 +491,9 @@ def convert_checkpoint( manifest["mtp_conditioning"] = mtp_conditioning_semantics(raw_config) if artifact_format == "aspec": - manifest["compute_dtype"] = str(model.embed_tokens.weight.dtype) + embedding = getattr(model, "embed_tokens", None) + compute = embedding.weight if embedding is not None else next(iter(tensors.values())) + manifest["compute_dtype"] = str(compute.dtype) manifest["tensor_dtypes"] = dict(Counter(str(t.dtype) for t in tensors.values())) if len(manifest["tensor_dtypes"]) > 1: manifest["dtype"] = "mixed" @@ -488,14 +512,26 @@ def convert_checkpoint( "runtime": "aspec", "entrypoint": "aspec.draft_artifact.load_native_draft", "proposal_entrypoint": proposal, "validation": "required_per_artifact", - "scope": ("asset and proposal; caller owns verifier features/cache/verification" - if proposal else "asset loading and weights-only warmstart; no native proposal component"), + "scope": ( + "asset and proposal; caller owns verifier features/cache/verification" + if proposal else + "heads-only asset; strict reload and per-head held-out metrics; " + "no candidate-tree acceptance" if algorithm == "medusa" else + "asset loading and weights-only warmstart; no native proposal component"), "borrowed_tensors": ["Target output head"] if algorithm in {"dspark", "dflash"} else [], } (output_dir / "export_manifest.json").write_text( json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) + if verify: + from aspec.draft_artifact import verify_native_export + + manifest["verified"] = verify_native_export(output_dir, expected=tensors) + (output_dir / "export_manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) return manifest @@ -511,6 +547,10 @@ def _parse_args() -> argparse.Namespace: "--dtype", choices=("float16", "bfloat16", "float32"), default=None ) parser.add_argument("-f", "--force", action="store_true") + parser.add_argument( + "--verify", action="store_true", + help="Strictly reload the native artifact and compare every tensor", + ) return parser.parse_args() @@ -527,6 +567,7 @@ def main() -> None: output_dtype=dtype, force=args.force, artifact_format=args.artifact_format, + verify=args.verify, ) print(json.dumps(manifest, indent=2, sort_keys=True)) diff --git a/aspec/commands/evaluate_medusa.py b/aspec/commands/evaluate_medusa.py new file mode 100644 index 0000000..92f91c7 --- /dev/null +++ b/aspec/commands/evaluate_medusa.py @@ -0,0 +1,180 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Report per-head held-out loss and token accuracy for an exported Medusa Draft.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections.abc import Mapping +from pathlib import Path + +import torch + +from aspec.data.utils import unpack_loss_mask +from aspec.draft_artifact import load_native_draft +from aspec.feature_contract import validate_feature_contract +from aspec.models.medusa import MedusaModel + + +def _record_batch(record, *, expected_hidden_size=None): + ids = record["input_ids"].reshape(-1).long() + hidden = record["hidden_states"] + hidden = hidden.reshape(-1, hidden.shape[-1]) + if expected_hidden_size is not None and hidden.shape[-1] != expected_hidden_size: + raise ValueError( + "Held-out hidden width differs from the dataset feature contract: " + f"{hidden.shape[-1]} != {expected_hidden_size}") + packed = record.get("packed_loss_mask") + if packed is None: + raise ValueError("Held-out Medusa records require packed_loss_mask") + mask = packed.reshape(-1) if isinstance(packed, torch.Tensor) else unpack_loss_mask(packed) + if ids.numel() == 0 or hidden.shape[0] != ids.numel() or mask.numel() != ids.numel(): + raise ValueError("Held-out record tensors disagree on sequence length") + return ids.unsqueeze(0), hidden.unsqueeze(0), mask.unsqueeze(0) + + +def _validate_binding(binding): + """Medusa consumes only the Target's final post-norm state from the dataset binding.""" + if not isinstance(binding, Mapping): + raise ValueError("Offline dataset has no Target feature binding") + contract = binding.get("feature_contract") + try: + validate_feature_contract(contract) + except Exception as exc: + raise ValueError("Offline dataset feature contract is invalid") from exc + tensors = contract["tensors"] + hidden = tensors.get("hidden_states") or {} + if (hidden.get("semantic") != "target_final_hidden_state" + or hidden.get("value_stage") != "post_norm"): + raise ValueError( + "Medusa requires the Target's final post-norm hidden state") + depth = contract.get("target_num_hidden_layers") + hidden_size = contract.get("target_hidden_size") + layers = contract.get("aux_hidden_state_layers") + if (not isinstance(depth, int) or depth < 1 + or not isinstance(hidden_size, int) or hidden_size < 1 + or not isinstance(layers, list) or layers != [depth - 1]): + raise ValueError( + "Medusa features must come from the final Target decoder layer") + if "last_hidden_states" in tensors or "mtp_hidden_states" in tensors: + raise ValueError("Medusa does not consume duplicate teacher tensors") + return contract + + +def _validate_target_binding(contract, draft_config): + """The dataset features must come from the Target the Draft was trained against.""" + if contract["target_num_hidden_layers"] != draft_config.target_num_hidden_layers: + raise ValueError( + "Offline dataset Target depth differs from the Draft's training Target: " + f"{contract['target_num_hidden_layers']} != " + f"{draft_config.target_num_hidden_layers}") + if contract["target_hidden_size"] != draft_config.target_hidden_size: + raise ValueError( + "Offline dataset Target hidden size differs from the Draft's training Target: " + f"{contract['target_hidden_size']} != {draft_config.target_hidden_size}") + + +def evaluate_records(draft, records, *, device="cpu", expected_hidden_size=None): + """Aggregate each head's token-weighted CE and accuracy over held-out records.""" + model = MedusaModel(draft).to(device).eval() + dtype = next(model.parameters()).dtype + heads = draft.medusa_num_heads + loss_sums = torch.zeros(heads, dtype=torch.float64) + correct = torch.zeros(heads, dtype=torch.float64) + counts = torch.zeros(heads, dtype=torch.long) + samples = 0 + with torch.no_grad(): + for record in records: + ids, hidden, mask = _record_batch( + record, expected_hidden_size=expected_hidden_size) + ids, hidden, mask = ids.to(device), hidden.to(device, dtype), mask.to(device) + losses, _, accuracy, head_counts, _ = model( + ids, hidden, mask, torch.ones_like(ids)) + for head in range(heads): + tokens = int(head_counts[head]) + if tokens == 0: + continue + loss_sums[head] += float(losses[head]) * tokens + correct[head] += float(accuracy[head]) * tokens + counts[head] += tokens + samples += 1 + if samples == 0: + raise ValueError("No held-out records were evaluated") + per_head = [] + for head in range(heads): + tokens = int(counts[head]) + per_head.append({ + "head": head, + "prediction_offset": head + 2, + "tokens": tokens, + "loss": float(loss_sums[head]) / tokens if tokens else None, + "accuracy": float(correct[head]) / tokens if tokens else None, + }) + measured = [item for item in per_head if item["tokens"]] + return { + "samples": samples, + "per_head": per_head, + "aggregate": { + "sum_of_head_mean_ce": ( + sum(item["loss"] for item in measured) if measured else None), + "measured_heads": len(measured), + }, + } + + +def _serialize_report(report): + try: + return json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + except ValueError as exc: + raise ValueError("Held-out report contains non-finite metrics") from exc + + +def _parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--draft", type=Path, required=True) + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--split", default="eval") + parser.add_argument("--limit", type=int, default=None) + parser.add_argument("--device", default="cpu") + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def main(): + args = _parse_args() + from aspec.offline.dataset import OfflineDataset + + dataset = OfflineDataset(args.dataset) + binding = dataset.metadata.get("binding") or {} + contract = _validate_binding(binding) + rows = dataset.rows(args.split) + if args.limit is not None: + if args.limit < 1: + raise ValueError("--limit must be positive") + rows = rows[: args.limit] + draft = load_native_draft(args.draft, device=args.device) + _validate_target_binding(contract, draft.config) + report = evaluate_records( + draft, (dataset.load(row) for row in rows), device=args.device, + expected_hidden_size=int(contract["target_hidden_size"])) + report.update({ + "draft_manifest_sha256": hashlib.sha256( + (args.draft / "export_manifest.json").read_bytes()).hexdigest(), + "dataset_identity": dataset.identity, + "feature_contract_id": contract["contract_id"], + "target_identity": binding.get("target_identity"), + "split": args.split, + "medusa_num_heads": draft.medusa_num_heads, + "medusa_num_layers": draft.medusa_num_layers, + }) + serialized = _serialize_report(report) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(serialized + "\n") + print(serialized) + + +if __name__ == "__main__": + main() diff --git a/aspec/draft_artifact.py b/aspec/draft_artifact.py index 10e5f17..5e52659 100644 --- a/aspec/draft_artifact.py +++ b/aspec/draft_artifact.py @@ -117,7 +117,9 @@ def load_native_draft(path, *, device="cpu", dtype=None): state = load_file(str(path / "model.safetensors"), device="cpu") if len(state) != manifest.get("tensor_count"): raise ValueError("Native artifact tensor count mismatch") - checkpoint_dtype = state["embed_tokens.weight"].dtype + checkpoint_dtype = ( + state["embed_tokens.weight"].dtype if "embed_tokens.weight" in state + else next(iter(state.values())).dtype) model = AutoEagle3DraftModel.from_config(config, torch_dtype=dtype or checkpoint_dtype) # Establish mixed precision BEFORE load; casting FP32 compressors through # BF16 and back would silently destroy the checkpoint's retained precision. @@ -134,6 +136,29 @@ def load_native_draft(path, *, device="cpu", dtype=None): return model +def verify_native_export(path, *, expected=None): + """Strictly reload a native artifact and compare it with the exported values. + + ``expected`` is the state that was written; comparing the reloaded model with + it catches loader-side casts or drops. Without it the artifact's own bytes are + re-read, which only proves the loader is idempotent, not that it roundtripped + the exported tensors. + """ + path = Path(path) + model = load_native_draft(path) + reloaded = model.state_dict() + if expected is None: + expected = load_file(str(path / "model.safetensors")) + if set(reloaded) != set(expected): + raise ValueError("Native reload changed the exported tensor set") + for key, value in reloaded.items(): + if value.shape != expected[key].shape or value.dtype != expected[key].dtype: + raise ValueError(f"Native reload changed {key} shape/dtype") + if not torch.equal(value, expected[key]): + raise ValueError(f"Native reload changed {key} values") + return {"method": "strict_load_and_tensor_equality", "tensors": len(reloaded)} + + class BlockDraftConsumer: """One DSpark/DFlash block from verifier features and its frozen output head. diff --git a/aspec/training/medusa_trainer.py b/aspec/training/medusa_trainer.py index fa25c61..124edf6 100644 --- a/aspec/training/medusa_trainer.py +++ b/aspec/training/medusa_trainer.py @@ -3,6 +3,9 @@ """Heads-only Medusa uses the existing optimizer/checkpoint/ownership training path.""" +import copy +import json +from collections.abc import Mapping from types import SimpleNamespace from aspec.models.medusa import MedusaModel @@ -17,6 +20,42 @@ def _create_training_model(self, draft_model): gradient_checkpointing=getattr(self.args, "gradient_checkpointing", False), ) + def _load_algorithm_weights(self, draft_model, target_model_path): + # Plan can only bind the vocabulary when the Target config is local. + # Resolve the effective config here as well so a Hub ID cannot skip it. + from transformers import AutoConfig + + config = AutoConfig.from_pretrained( + target_model_path, + trust_remote_code=getattr(self.args, "trust_remote_code", False), + ).to_dict() + extra_args = getattr(self.args, "sglang_extra_args", None) or {} + overrides = extra_args.get("json_model_override_args") + if overrides is not None: + if isinstance(overrides, str): + try: + overrides = json.loads(overrides) + except json.JSONDecodeError as exc: + raise ValueError( + "inference.sglang.extra_args.json_model_override_args must be " + f"valid JSON: {exc}") from exc + if not isinstance(overrides, Mapping): + raise ValueError( + "inference.sglang.extra_args.json_model_override_args must contain " + "a JSON object") + config.update(copy.deepcopy(dict(overrides))) + target_vocab_size = config.get("vocab_size") + text_config = config.get("text_config") + if target_vocab_size is None and isinstance(text_config, Mapping): + target_vocab_size = text_config.get("vocab_size") + if not isinstance(target_vocab_size, int) or isinstance(target_vocab_size, bool): + raise ValueError("Medusa requires the Target config to declare vocab_size") + if draft_model.vocab_size != target_vocab_size: + raise ValueError( + "Medusa heads project the full Target vocabulary: " + f"draft vocab_size {draft_model.vocab_size} != " + f"Target vocab_size {target_vocab_size}") + def _prepare_algorithm_training(self, draft_model): if getattr(self.args, "freeze_lm_head", False): raise ValueError("Medusa heads own trainable vocabulary projections") diff --git a/configs/recipes/medusa_qwen3_8b.yaml b/configs/recipes/medusa_qwen3_8b.yaml new file mode 100644 index 0000000..8302ba8 --- /dev/null +++ b/configs/recipes/medusa_qwen3_8b.yaml @@ -0,0 +1,77 @@ +# Release training starting point: three passes, held-out eval and checkpoints. +# Hyperparameters are not a convergence/quality claim; see docs/user-guide/recipes.md. +# Formal single-node heads-only Medusa training, resume and native export recipe. +model: + target_model_path: ${oc.env:ASPEC_TARGET_MODEL,Qwen/Qwen3-8B} + native_context_length: 32768 + trust_remote_code: false + draft_model_config: aspec/config/medusa_qwen3_8b_draft_config.json +dataset: + eval_interval: 100 + eval_micro_batch_size: 1 + train_data_path: ${oc.env:ASPEC_TRAIN_DATA} + eval_data_path: ${oc.env:ASPEC_EVAL_DATA} + chat_template: qwen3-instruct + prompt_key: conversations + min_loss_tokens: 1 + shuffle_dataset: true + dataset_seed: 42 +training: + attention_backend: sdpa + draft_context_parallel_size: 1 + micro_batch_size: 1 + draft_accumulation_steps: 4 + learning_rate: 6e-4 + min_lr: 6e-5 + weight_decay: 0.0 + max_grad_norm: 1.0 + max_seq_length: 4096 + num_epochs: 3 + warmup_ratio: 0.04 + seed: 42 + continual_training: false + training_num_gpus_per_node: 1 + training_num_nodes: 1 + placement_strategy: inference_first + fsdp_strategy: REPLICATE + fsdp_reduce_dtype: bfloat16 + compile_model: false + max_concurrent_batches: 1 + streaming_dispatch_with_training: true + pipeline_dispatch_with_training: true + prefetch_depth: 1 + save_interval: 100 + save_per_epoch: false + save_final_checkpoint: true + max_checkpoints: 2 + gradient_checkpointing: true + dflash_packing: false +inference: + inference_engine_type: sgl + store_last_hidden_states: false + target_prefill_memory_optimized: false + inference_num_gpus: 1 + inference_num_gpus_per_engine: 1 + inference_num_gpus_per_node: 2 + inference_fetch_batch: 8 + inference_buffer_threshold: 2 + max_sample_pool_size: 12 + inference_batch_size: 2 + sglang: + tp_size: 1 + mem_fraction_static: 0.5 +mooncake: + data_plane: store + protocol: tcp + require_nvlink: false + allow_host_fallback: false + max_seq_len: 4096 + hidden_dim: 4096 + global_segment_size: 32GB + local_buffer_size: 2GB + host_buffer_size: 2GB + async_put_pool_size: 8 +logging: + report_to: none +output_dir: outputs/qwen3-8b-medusa +cache_dir: cache/qwen3-8b-medusa \ No newline at end of file diff --git a/docs/support.md b/docs/support.md index c5f82d0..6ee9d35 100644 --- a/docs/support.md +++ b/docs/support.md @@ -13,7 +13,7 @@ and tests do not imply CPU training support. | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | | Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training, `lambda_base` curriculum, held-out corrected/backbone observations and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; native export and consumer acceptance pending; see [scope](user-guide/domino.md) | -| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery; SDPA CP1, MBS1, non-packing, REPLICATE | No packing/CP, warmstart, export or consumer acceptance yet; see [scope](user-guide/medusa.md) | +| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery and native export; SDPA CP1, MBS1, non-packing, REPLICATE, one node | No packing/CP, warmstart or candidate-tree acceptance yet; see [scope](user-guide/medusa.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | | JetSpec core | Qwen3-style causal parallel Draft head and temperature-scaled forward-KL objective, reused DFlash block machinery; chunked full-vocabulary KL and explicit causal visibility | Model/loss prerequisite only; no trainer selection, tree builder, recovery, export or consumer validation; CPU causal packing and direct unmasked Draft calls fail closed; see [scope](user-guide/jetspec.md) | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index 9416f91..40281c9 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -164,6 +164,6 @@ post-norm final state; head `i` at position `t` is supervised by token `t+2+i`. the sum of per-head mean cross-entropies (assistant tokens only): each head's CE is averaged within a microbatch, and microbatches are weighted equally by gradient accumulation rather than pooled by token count. The heads own trainable full-vocabulary projections. Start from -`configs/examples/medusa_qwen3_8b_4k_tcp.yaml` (SDPA, CP1, MBS1, non-packing, REPLICATE, -Store/TCP). Export, a maintained recipe, per-head held-out observations and candidate-tree -acceptance remain future work; see [details](medusa.md). +`configs/recipes/medusa_qwen3_8b.yaml` (SDPA, CP1, MBS1, non-packing, REPLICATE, Store/TCP). +Native export, strict reload and per-head held-out reporting are implemented; candidate-tree +acceptance remains future work; see [details](medusa.md). diff --git a/docs/user-guide/medusa.md b/docs/user-guide/medusa.md index 8e4c1f0..4608b29 100644 --- a/docs/user-guide/medusa.md +++ b/docs/user-guide/medusa.md @@ -8,6 +8,8 @@ loss landed first; this page describes the current training scope. ## Reference and configuration +- Recipe: `configs/recipes/medusa_qwen3_8b.yaml` (three passes, held-out eval, checkpoints); use + `aspec recipes plan medusa` / `aspec recipes train medusa --output outputs/qwen3-8b-medusa`. - Draft config: `aspec/config/medusa_qwen3_8b_draft_config.json` (Qwen3-8B Target, `K=5` heads). - Example: `configs/examples/medusa_qwen3_8b_4k_tcp.yaml` (one Target GPU + one Draft GPU, 4K, SDPA, CP1, non-packing, REPLICATE, Store/TCP transport; no NVLink required). Set @@ -15,7 +17,10 @@ loss landed first; this page describes the current training scope. `model.target_model_path`) and run from the repository root so the relative draft-config path resolves. - The heads are the only trainable parameters; the Target is frozen and its post-norm final - state is the only feature the Draft consumes. + state is the only feature the Draft consumes. At startup the trainer resolves the Target + config and rejects a draft projection width that differs from its `vocab_size`, including for + Hub IDs that Plan could not read locally. It applies + `inference.sglang.extra_args.json_model_override_args` exactly as Plan does. ## Inputs and prediction semantics @@ -44,5 +49,37 @@ The current path fails closed outside the following: - `fsdp_strategy: REPLICATE`, one node. Anchored training, MTP conditioning and whole-model compile are rejected. -Export, a maintained recipe, per-head held-out observations and candidate-tree acceptance are not -part of this scope. +## Export and strict reload + +Export a trained checkpoint as an ASpec-native asset: + +```bash +python -m tools.convert_to_hf \ + --input-dir outputs/qwen3-8b-medusa/checkpoints/iter_0000101 \ + --output-dir outputs/medusa-export --format aspec --verify +``` + +`--format aspec` is required: the SGLang serving format is rejected because no verified +candidate-tree Medusa loader exists. The artifact contains `config.json`, +`model.safetensors` and `export_manifest.json`; `--verify` reloads it through +`aspec.draft_artifact.load_native_draft` (strict, no dropped tensors) and compares every +reloaded tensor with the values the export wrote. + +## Held-out per-head observations + +The training eval loop reports `eval/ploss_{i}` and `eval/acc_{i}` when `dataset.eval_data_path` +is set. To report the same observations from an exported asset against a materialized offline +feature dataset: + +```bash +python -m tools.evaluate_medusa \ + --draft outputs/medusa-export --dataset /path/to/offline-dataset \ + --split eval --output outputs/medusa-eval.json +``` + +The report records the artifact and dataset identities, per-head token counts, token-weighted +mean CE and token accuracy, and the sum of the per-head means. These are held-out training +observations, not candidate-tree acceptance. + +Candidate-tree acceptance and decode-equality under a Medusa evaluator are not part of this +scope; they require a verified consumer adapter. diff --git a/docs/user-guide/recipes.md b/docs/user-guide/recipes.md index 68cb315..0a1c3bc 100644 --- a/docs/user-guide/recipes.md +++ b/docs/user-guide/recipes.md @@ -3,10 +3,11 @@ Choose a recipe for your Target, Draft algorithm and available resources. EAGLE4 uses two successive stages. See [supported combinations](../support.md) before combining algorithms and structures. -Qwen and Ling DSpark/DFlash paths are the main optimization and recipe priorities. EAGLE3/4 and -native MTP have their own documented training and asset contracts; DFlash2 remains an experimental -example outside this recipe catalog. A listed recipe is a maintained starting configuration, not -a claim that every algorithm/structure combination has the same quality or performance evidence. +Qwen and Ling DSpark/DFlash paths are the main optimization and recipe priorities. EAGLE3/4, +native MTP and Medusa heads have their own documented training and asset contracts; DFlash2 +remains an experimental example outside this recipe catalog. A listed recipe is a maintained +starting configuration, not a claim that every algorithm/structure combination has the same +quality or performance evidence. The table shows each recipe's configured starting allocation, not a minimum requirement or optimal Target/Draft split. T = Target GPU; D = Draft GPU. Check memory fit and the required node-local NVLink @@ -17,6 +18,7 @@ connections on your hardware before training. | `quickstart` | Qwen3-8B / DSpark | 1 node, 2 GPUs: 1T + 1D; 1K | Try training, recovery and export with bundled data | | `dspark` | Qwen3-8B / Dense DSpark | 1 node, 8 GPUs: 4T + 4D; 4K | Train on your data with packing and NVLink | | `dflash` | Qwen3-8B / DFlash | 1 node, 3 GPUs: 1T + 2D; 4K | Train a block-parallel DFlash Draft | +| `medusa` | Qwen3-8B / heads-only Medusa | 1 node, 2 GPUs: 1T + 1D; 4K | Train parallel future-token heads and export a native heads asset | | `eagle3` | Qwen3-8B / EAGLE3 | 1 node, 3 GPUs: 1T + 2D; 4K | Train a recurrent EAGLE3 Draft | | `eagle4` | Qwen3-8B / EAGLE4 | 1 node, 3 GPUs: 1T + 2D; 4K; sequential stages | Train a Draft, then candidate retrieval projections | | `dmtp` | Qwen3.5-9B / MTP-conditioned DSpark | 1 node, 8 GPUs: 4T + 4D; 4K, CP1 | Train a Draft conditioned on the frozen native predictor; native assets | diff --git a/tests/cpu/test_medusa_evaluation.py b/tests/cpu/test_medusa_evaluation.py new file mode 100644 index 0000000..21b2179 --- /dev/null +++ b/tests/cpu/test_medusa_evaluation.py @@ -0,0 +1,150 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Per-head held-out evaluation for exported Medusa heads.""" + +import math + +import pytest +import torch +from torch.nn import functional as F + +from aspec.commands.evaluate_medusa import ( + _record_batch, + _serialize_report, + _validate_binding, + _validate_target_binding, + evaluate_records, +) +from aspec.feature_contract import build_target_feature_contract +from aspec.models.draft import AutoDraftModelConfig, AutoEagle3DraftModel + + +def _draft(): + config = AutoDraftModelConfig.from_dict({ + "architectures": ["MedusaDraftModel"], "model_type": "aspec_medusa", + "hidden_size": 8, "vocab_size": 12, "target_hidden_size": 8, + "target_num_hidden_layers": 4, "medusa_num_heads": 2, "medusa_num_layers": 1, + }) + torch.manual_seed(1) + return AutoEagle3DraftModel.from_config(config, torch_dtype=torch.float32) + + +def _record(ids, hidden, packed): + return {"input_ids": ids, "hidden_states": hidden, "packed_loss_mask": packed} + + +def _expected(draft, ids, hidden, mask): + rows = [] + for head in range(draft.medusa_num_heads): + offset = head + 2 + keep = mask[offset:].bool() + if not keep.any(): + rows.append((0, None, None)) + continue + logits = draft.medusa_head[head](hidden[:-offset].float())[keep] + targets = ids[offset:][keep] + rows.append(( + int(keep.sum()), + F.cross_entropy(logits, targets, reduction="mean").item(), + (logits.argmax(-1) == targets).float().mean().item(), + )) + return rows + + +def _assert_report(report, expected_rows): + assert report["samples"] == len(expected_rows) + measured = [] + for item, rows in zip(report["per_head"], zip(*expected_rows), strict=True): + tokens = sum(row[0] for row in rows) + assert item["tokens"] == tokens + if tokens == 0: + assert item["loss"] is None and item["accuracy"] is None + continue + loss = sum(count * value for count, value, _ in rows) / tokens + accuracy = sum(count * value for count, _, value in rows) / tokens + assert math.isclose(item["loss"], loss, rel_tol=1e-6, abs_tol=1e-7) + assert math.isclose(item["accuracy"], accuracy, rel_tol=1e-6, abs_tol=1e-7) + measured.append(item["loss"]) + assert math.isclose( + report["aggregate"]["sum_of_head_mean_ce"], sum(measured), rel_tol=1e-6) + + +def test_per_head_metrics_are_token_weighted_across_records(): + draft = _draft() + torch.manual_seed(2) + ids_a = torch.randint(0, 12, (7,)) + hidden_a = torch.randn(7, 8) + mask_a = torch.tensor([0, 1, 1, 1, 1, 0, 1]) + ids_b = torch.randint(0, 12, (6,)) + hidden_b = torch.randn(6, 8) + records = [ + _record(ids_a, hidden_a, mask_a), + _record(ids_b, hidden_b, [1, 4, 1]), # packed: 0,1,1,1,1,0 + ] + report = evaluate_records(draft, records) + _assert_report(report, [ + _expected(draft, ids_a, hidden_a, mask_a), + _expected(draft, ids_b, hidden_b, torch.tensor([0, 1, 1, 1, 1, 0])), + ]) + + +def test_record_validation_rejects_missing_mask_and_length_drift(): + with pytest.raises(ValueError, match="packed_loss_mask"): + _record_batch({"input_ids": torch.arange(4), "hidden_states": torch.randn(4, 8)}) + with pytest.raises(ValueError, match="sequence length"): + _record_batch({ + "input_ids": torch.arange(4), "hidden_states": torch.randn(5, 8), + "packed_loss_mask": torch.ones(4), + }) + + +def _medusa_contract(): + return build_target_feature_contract( + drafter_family="medusa", target_hidden_size=8, target_num_hidden_layers=4, + aux_hidden_state_layers=[3], store_last_hidden_states=False) + + +def test_binding_requires_the_final_post_norm_state(): + contract = _validate_binding({ + "feature_contract": _medusa_contract(), "target_identity": {"config.json": "x"}}) + assert contract["tensors"]["hidden_states"]["value_stage"] == "post_norm" + eagle = build_target_feature_contract( + drafter_family="eagle3", target_hidden_size=8, target_num_hidden_layers=4, + aux_hidden_state_layers=[1, 3], store_last_hidden_states=True) + with pytest.raises(ValueError, match="final post-norm"): + _validate_binding({"feature_contract": eagle}) + + +def test_record_hidden_width_must_match_the_contract(): + with pytest.raises(ValueError, match="hidden width"): + _record_batch({ + "input_ids": torch.arange(4), "hidden_states": torch.randn(4, 8), + "packed_loss_mask": torch.ones(4), + }, expected_hidden_size=16) + + +def test_report_serialization_rejects_non_finite_metrics(): + assert '"loss": null' in _serialize_report({"loss": None}) + with pytest.raises(ValueError, match="non-finite"): + _serialize_report({"loss": float("nan")}) + with pytest.raises(ValueError, match="non-finite"): + _serialize_report({"loss": float("inf")}) + + +def test_binding_must_match_the_draft_target_binding(): + draft = _draft() + contract = _validate_binding({"feature_contract": _medusa_contract()}) + _validate_target_binding(contract, draft.config) + + deeper = build_target_feature_contract( + drafter_family="medusa", target_hidden_size=8, target_num_hidden_layers=6, + aux_hidden_state_layers=[5], store_last_hidden_states=False) + with pytest.raises(ValueError, match="Target depth"): + _validate_target_binding(deeper, draft.config) + + wider = build_target_feature_contract( + drafter_family="medusa", target_hidden_size=16, target_num_hidden_layers=4, + aux_hidden_state_layers=[3], store_last_hidden_states=False) + with pytest.raises(ValueError, match="hidden size"): + _validate_target_binding(wider, draft.config) diff --git a/tests/cpu/test_medusa_export.py b/tests/cpu/test_medusa_export.py new file mode 100644 index 0000000..f46ec09 --- /dev/null +++ b/tests/cpu/test_medusa_export.py @@ -0,0 +1,155 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Native Medusa export, strict reload and manifest contracts.""" + +import json + +import pytest +import torch + +from aspec.draft_artifact import load_native_draft, verify_native_export +from aspec.models.draft import AutoDraftModelConfig, AutoEagle3DraftModel +from tools import convert_to_hf + +_RAW = { + "architectures": ["MedusaDraftModel"], "model_type": "aspec_medusa", + "hidden_size": 16, "vocab_size": 32, "target_hidden_size": 16, + "target_num_hidden_layers": 4, "medusa_num_heads": 2, "medusa_num_layers": 1, + "tie_word_embeddings": False, +} + + +def _checkpoint(tmp_path, monkeypatch, raw=_RAW): + config = AutoDraftModelConfig.from_dict(raw) + torch.manual_seed(0) + source = AutoEagle3DraftModel.from_config(config, torch_dtype=torch.float32) + checkpoint = tmp_path / "iter_0000002" + (checkpoint / "model").mkdir(parents=True) + (checkpoint / "model" / ".metadata").write_bytes(b"stub") + (checkpoint / "draft_config.json").write_text( + json.dumps(config.to_dict()), encoding="utf-8") + wrapped = { + f"model_state.model.draft_model.{key}": value.detach().clone() + for key, value in source.state_dict().items() + } + monkeypatch.setattr(convert_to_hf, "_load_fsdp_state_dict", lambda _: wrapped) + return source, checkpoint + + +def test_native_export_roundtrips_every_head(tmp_path, monkeypatch): + source, checkpoint = _checkpoint(tmp_path, monkeypatch) + output = tmp_path / "asset" + manifest = convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=output, artifact_format="aspec", verify=True) + + assert manifest["algorithm"] == "medusa" + assert manifest["tensor_count"] == len(source.state_dict()) + assert manifest["verified"]["tensors"] == len(source.state_dict()) + assert manifest["draft_structure"]["loader_contract"]["runtime"] == "aspec" + assert "candidate-tree acceptance" in manifest["draft_structure"]["loader_contract"]["scope"] + + exported_config = json.loads((output / "config.json").read_text()) + assert exported_config["architectures"] == ["MedusaDraftModel"] + assert exported_config["target_layer_ids"] == [3] + assert exported_config["num_aux_hidden_states"] == 1 + assert exported_config["draft_vocab_size"] == 32 + + imported = load_native_draft(output) + assert len(imported.medusa_head) == 2 + for key, value in source.state_dict().items(): + assert torch.equal(imported.state_dict()[key], value) + + +def test_verify_detects_a_changed_tensor(tmp_path, monkeypatch): + _, checkpoint = _checkpoint(tmp_path, monkeypatch) + output = tmp_path / "asset" + convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=output, artifact_format="aspec") + + from safetensors.torch import load_file, save_file + + tensors = load_file(str(output / "model.safetensors")) + first = next(iter(tensors)) + tensors[first] = tensors[first] + 1 + save_file(tensors, str(output / "model.safetensors")) + with pytest.raises(ValueError, match="checksum mismatch"): + verify_native_export(output) + + +def test_verify_compares_reload_with_the_exported_tensors(tmp_path, monkeypatch): + source, checkpoint = _checkpoint(tmp_path, monkeypatch) + output = tmp_path / "asset" + convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=output, artifact_format="aspec") + + from safetensors.torch import load_file, save_file + + tensors = load_file(str(output / "model.safetensors")) + first = next(iter(tensors)) + tensors[first] = tensors[first] + 1 + save_file(tensors, str(output / "model.safetensors")) + # Keep the artifact internally consistent so only the exported values differ. + manifest_path = output / "export_manifest.json" + recorded = json.loads(manifest_path.read_text()) + recorded["model_sha256"] = convert_to_hf._sha256_file(output / "model.safetensors") + manifest_path.write_text( + json.dumps(recorded, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="changed .* values"): + verify_native_export(output, expected=source.state_dict()) + + +def test_depth_two_shared_blocks_export_and_reload(tmp_path, monkeypatch): + source, checkpoint = _checkpoint( + tmp_path, monkeypatch, {**_RAW, "medusa_num_layers": 2}) + output = tmp_path / "asset" + manifest = convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=output, artifact_format="aspec", verify=True) + assert manifest["verified"]["tensors"] == len(source.state_dict()) + + from safetensors.torch import load_file + + tensors = load_file(str(output / "model.safetensors")) + for head in range(2): + first = tensors[f"medusa_head.{head}.0.linear.weight"] + second = tensors[f"medusa_head.{head}.1.linear.weight"] + assert torch.equal(first, second) + imported = load_native_draft(output) + for key, value in source.state_dict().items(): + assert torch.equal(imported.state_dict()[key], value) + + +def test_verify_requires_native_format_before_writing(tmp_path, monkeypatch): + _, checkpoint = _checkpoint(tmp_path, monkeypatch) + output = tmp_path / "serving" + with pytest.raises(ValueError, match="--verify requires --format aspec"): + convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=output, artifact_format="sglang", verify=True) + assert not output.exists() + + +def test_export_verify_compares_with_the_tensors_it_wrote(tmp_path, monkeypatch): + source, checkpoint = _checkpoint(tmp_path, monkeypatch) + from aspec import draft_artifact + + captured = {} + + def fake_verify(path, *, expected=None): + captured["expected"] = expected + return {"method": "stub", "tensors": len(expected or {})} + + monkeypatch.setattr(draft_artifact, "verify_native_export", fake_verify) + manifest = convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=tmp_path / "asset", artifact_format="aspec", + verify=True) + + assert set(captured["expected"]) == set(source.state_dict()) + assert manifest["verified"] == {"method": "stub", "tensors": len(source.state_dict())} + + +def test_sglang_format_is_rejected_for_medusa(tmp_path, monkeypatch): + _, checkpoint = _checkpoint(tmp_path, monkeypatch) + with pytest.raises(NotImplementedError, match="format aspec"): + convert_to_hf.convert_checkpoint( + input_dir=checkpoint, output_dir=tmp_path / "serving", artifact_format="sglang") diff --git a/tests/cpu/test_medusa_integration.py b/tests/cpu/test_medusa_integration.py index 21a08ca..37a4be2 100644 --- a/tests/cpu/test_medusa_integration.py +++ b/tests/cpu/test_medusa_integration.py @@ -64,7 +64,7 @@ def _tiny_config(**overrides): def _fake_qwen_target(tmp_path, vocab_size=151936): target = tmp_path / "Qwen3-8B-test" - target.mkdir() + target.mkdir(parents=True, exist_ok=True) (target / "config.json").write_text(json.dumps({ "model_type": "qwen3", "architectures": ["Qwen3ForCausalLM"], @@ -326,6 +326,35 @@ def test_head_count_drift_is_rejected(): trainer._prepare_algorithm_training(MedusaDraftModel(_tiny_config())) +def test_target_vocabulary_is_bound_at_startup(tmp_path): + from aspec.training.medusa_trainer import MedusaTrainer + + trainer = object.__new__(MedusaTrainer) + trainer.args = SimpleNamespace(trust_remote_code=False) + draft = MedusaDraftModel(_tiny_config()) + trainer._load_algorithm_weights(draft, str(_fake_qwen_target(tmp_path, vocab_size=32))) + mismatched = _fake_qwen_target(tmp_path / "mismatch", vocab_size=31) + with pytest.raises(ValueError, match="full Target vocabulary"): + trainer._load_algorithm_weights(draft, str(mismatched)) + + +def test_startup_vocabulary_honors_sglang_overrides(tmp_path): + from aspec.training.medusa_trainer import MedusaTrainer + + trainer = object.__new__(MedusaTrainer) + trainer.args = SimpleNamespace(trust_remote_code=False, sglang_extra_args={}) + draft = MedusaDraftModel(_tiny_config()) + target = str(_fake_qwen_target(tmp_path, vocab_size=33)) + with pytest.raises(ValueError, match="full Target vocabulary"): + trainer._load_algorithm_weights(draft, target) + for overrides in ('{"vocab_size": 32}', {"vocab_size": 32}): + trainer.args.sglang_extra_args = {"json_model_override_args": overrides} + trainer._load_algorithm_weights(draft, target) + trainer.args.sglang_extra_args = {"json_model_override_args": {"vocab_size": 34}} + with pytest.raises(ValueError, match="full Target vocabulary"): + trainer._load_algorithm_weights(draft, target) + + def test_metric_semantics_drop_unmeasured_acceptance(): from aspec.training.medusa_trainer import MedusaTrainer diff --git a/tests/cpu/test_release_recipes.py b/tests/cpu/test_release_recipes.py index 59674f6..4deb160 100644 --- a/tests/cpu/test_release_recipes.py +++ b/tests/cpu/test_release_recipes.py @@ -83,7 +83,7 @@ def test_qwen8b_recipes_plan_without_target_override(name, monkeypatch): def test_catalog_exactly_covers_formal_files(): entries = recipes.catalog() - assert len(entries) == 13 + assert len(entries) == 14 declared = {p for entry in entries.values() for p in entry["configs"]} assert declared == { str(p.relative_to(recipes.ROOT)) for p in (recipes.ROOT / "configs/recipes").glob("*.yaml") diff --git a/tools/evaluate_medusa.py b/tools/evaluate_medusa.py new file mode 100644 index 0000000..4366a28 --- /dev/null +++ b/tools/evaluate_medusa.py @@ -0,0 +1,13 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Compatibility entry point for the installed evaluate_medusa command.""" + +import sys +from importlib import import_module + +_command = import_module("aspec.commands.evaluate_medusa") +if __name__ == "__main__": + _command.main() +else: + sys.modules[__name__] = _command diff --git a/tools/recipe_catalog.json b/tools/recipe_catalog.json index 6c0873f..877e2e1 100644 --- a/tools/recipe_catalog.json +++ b/tools/recipe_catalog.json @@ -146,5 +146,16 @@ "target_gpus": 4, "draft_gpus": 4, "native_only_export": true + }, + "medusa": { + "configs": [ + "configs/recipes/medusa_qwen3_8b.yaml" + ], + "model": "Qwen3-8B", + "nodes": 1, + "gpus": 2, + "target_gpus": 1, + "draft_gpus": 1, + "native_only_export": true } }