Skip to content
57 changes: 49 additions & 8 deletions aspec/commands/convert_to_hf.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,23 @@ def _aspec_version() -> str:
return "unknown"


def _export_tensors(serving_state):
"""Detach state tensors, cloning repeated storage so safetensors accepts it."""
tensors = {}
seen_storage = set()
for key, value in serving_state.items():
tensor = value.detach().cpu().contiguous()
storage = tensor.untyped_storage().data_ptr()
if storage in seen_storage:
# Shared residual-block depths repeat one module, so several state
# keys alias one storage. Keep every key but give the file real copies.
tensor = tensor.clone()
else:
seen_storage.add(storage)
tensors[key] = tensor
return tensors


def _detect_model_dir(input_dir: Path) -> Path:
model_dir = input_dir / "model"
return model_dir if model_dir.is_dir() else input_dir
Expand Down Expand Up @@ -166,6 +183,11 @@ def _serving_config(raw_config: dict) -> dict:
if raw_config.get("mtp_conditioning"):
raise NotImplementedError("MTP-conditioned Drafts require native --format aspec export")

if raw_config.get("architectures") == ["MedusaDraftModel"]:
raise NotImplementedError(
"Medusa heads require native --format aspec export; "
"SGLang has no verified candidate-tree Medusa loader")

if raw_config.get("architectures") == ["NativeMTPDraftModel"]:
native = json.loads(json.dumps(raw_config["native_config"]))
native["architectures"] = ["Qwen3_5ForCausalLMMTP"]
Expand Down Expand Up @@ -343,6 +365,7 @@ def convert_checkpoint(
output_dtype: torch.dtype | None = None,
force: bool = False,
artifact_format: str = "sglang",
verify: bool = False,
) -> dict:
input_dir = input_dir.expanduser().resolve()
model_dir = _detect_model_dir(input_dir)
Expand All @@ -358,7 +381,9 @@ def convert_checkpoint(
raw_config = json.loads(config_path.read_text(encoding="utf-8"))
if artifact_format not in {"sglang", "aspec"}:
raise ValueError("artifact_format must be sglang or aspec")
if resolve_draft_algorithm(raw_config) == "mtp":
if verify and artifact_format != "aspec":
raise ValueError("--verify requires --format aspec")
if resolve_draft_algorithm(raw_config) in {"mtp", "medusa"}:
raw_config = AutoDraftModelConfig.from_dict(raw_config).to_dict()
training_contract = _checkpoint_training_contract(model_dir, raw_config)
export_config = (_serving_config(raw_config) if artifact_format == "sglang"
Expand Down Expand Up @@ -418,10 +443,7 @@ def convert_checkpoint(
model.state_dict_for_serving() if hasattr(model, "state_dict_for_serving")
else to_export_keys(model.state_dict())
)
tensors = {
key: value.detach().cpu().contiguous()
for key, value in serving_state.items()
}
tensors = _export_tensors(serving_state)

eagle4_asset = None
if resolve_draft_algorithm(raw_config) == "eagle4":
Expand Down Expand Up @@ -469,7 +491,9 @@ def convert_checkpoint(

manifest["mtp_conditioning"] = mtp_conditioning_semantics(raw_config)
if artifact_format == "aspec":
manifest["compute_dtype"] = str(model.embed_tokens.weight.dtype)
embedding = getattr(model, "embed_tokens", None)
compute = embedding.weight if embedding is not None else next(iter(tensors.values()))
manifest["compute_dtype"] = str(compute.dtype)
manifest["tensor_dtypes"] = dict(Counter(str(t.dtype) for t in tensors.values()))
if len(manifest["tensor_dtypes"]) > 1:
manifest["dtype"] = "mixed"
Expand All @@ -488,14 +512,26 @@ def convert_checkpoint(
"runtime": "aspec", "entrypoint": "aspec.draft_artifact.load_native_draft",
"proposal_entrypoint": proposal,
"validation": "required_per_artifact",
"scope": ("asset and proposal; caller owns verifier features/cache/verification"
if proposal else "asset loading and weights-only warmstart; no native proposal component"),
"scope": (
"asset and proposal; caller owns verifier features/cache/verification"
if proposal else
"heads-only asset; strict reload and per-head held-out metrics; "
"no candidate-tree acceptance" if algorithm == "medusa" else
"asset loading and weights-only warmstart; no native proposal component"),
"borrowed_tensors": ["Target output head"] if algorithm in {"dspark", "dflash"} else [],
}
(output_dir / "export_manifest.json").write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
if verify:
from aspec.draft_artifact import verify_native_export

manifest["verified"] = verify_native_export(output_dir, expected=tensors)
(output_dir / "export_manifest.json").write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return manifest


Expand All @@ -511,6 +547,10 @@ def _parse_args() -> argparse.Namespace:
"--dtype", choices=("float16", "bfloat16", "float32"), default=None
)
parser.add_argument("-f", "--force", action="store_true")
parser.add_argument(
"--verify", action="store_true",
help="Strictly reload the native artifact and compare every tensor",
)
return parser.parse_args()


Expand All @@ -527,6 +567,7 @@ def main() -> None:
output_dtype=dtype,
force=args.force,
artifact_format=args.artifact_format,
verify=args.verify,
)
print(json.dumps(manifest, indent=2, sort_keys=True))

Expand Down
180 changes: 180 additions & 0 deletions aspec/commands/evaluate_medusa.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
# Copyright 2026 Ant Group Co., Ltd.
# SPDX-License-Identifier: Apache-2.0

"""Report per-head held-out loss and token accuracy for an exported Medusa Draft."""

from __future__ import annotations

import argparse
import hashlib
import json
from collections.abc import Mapping
from pathlib import Path

import torch

from aspec.data.utils import unpack_loss_mask
from aspec.draft_artifact import load_native_draft
from aspec.feature_contract import validate_feature_contract
from aspec.models.medusa import MedusaModel


def _record_batch(record, *, expected_hidden_size=None):
ids = record["input_ids"].reshape(-1).long()
hidden = record["hidden_states"]
hidden = hidden.reshape(-1, hidden.shape[-1])
if expected_hidden_size is not None and hidden.shape[-1] != expected_hidden_size:
raise ValueError(
"Held-out hidden width differs from the dataset feature contract: "
f"{hidden.shape[-1]} != {expected_hidden_size}")
packed = record.get("packed_loss_mask")
if packed is None:
raise ValueError("Held-out Medusa records require packed_loss_mask")
mask = packed.reshape(-1) if isinstance(packed, torch.Tensor) else unpack_loss_mask(packed)
if ids.numel() == 0 or hidden.shape[0] != ids.numel() or mask.numel() != ids.numel():
raise ValueError("Held-out record tensors disagree on sequence length")
return ids.unsqueeze(0), hidden.unsqueeze(0), mask.unsqueeze(0)


def _validate_binding(binding):
"""Medusa consumes only the Target's final post-norm state from the dataset binding."""
if not isinstance(binding, Mapping):
raise ValueError("Offline dataset has no Target feature binding")
contract = binding.get("feature_contract")
try:
validate_feature_contract(contract)
except Exception as exc:
raise ValueError("Offline dataset feature contract is invalid") from exc
tensors = contract["tensors"]
hidden = tensors.get("hidden_states") or {}
if (hidden.get("semantic") != "target_final_hidden_state"
or hidden.get("value_stage") != "post_norm"):
raise ValueError(
"Medusa requires the Target's final post-norm hidden state")
depth = contract.get("target_num_hidden_layers")
hidden_size = contract.get("target_hidden_size")
layers = contract.get("aux_hidden_state_layers")
if (not isinstance(depth, int) or depth < 1
or not isinstance(hidden_size, int) or hidden_size < 1
or not isinstance(layers, list) or layers != [depth - 1]):
raise ValueError(
"Medusa features must come from the final Target decoder layer")
if "last_hidden_states" in tensors or "mtp_hidden_states" in tensors:
raise ValueError("Medusa does not consume duplicate teacher tensors")
return contract


def _validate_target_binding(contract, draft_config):
"""The dataset features must come from the Target the Draft was trained against."""
if contract["target_num_hidden_layers"] != draft_config.target_num_hidden_layers:
raise ValueError(
"Offline dataset Target depth differs from the Draft's training Target: "
f"{contract['target_num_hidden_layers']} != "
f"{draft_config.target_num_hidden_layers}")
if contract["target_hidden_size"] != draft_config.target_hidden_size:
raise ValueError(
"Offline dataset Target hidden size differs from the Draft's training Target: "
f"{contract['target_hidden_size']} != {draft_config.target_hidden_size}")


def evaluate_records(draft, records, *, device="cpu", expected_hidden_size=None):
"""Aggregate each head's token-weighted CE and accuracy over held-out records."""
model = MedusaModel(draft).to(device).eval()
dtype = next(model.parameters()).dtype
heads = draft.medusa_num_heads
loss_sums = torch.zeros(heads, dtype=torch.float64)
correct = torch.zeros(heads, dtype=torch.float64)
counts = torch.zeros(heads, dtype=torch.long)
samples = 0
with torch.no_grad():
for record in records:
ids, hidden, mask = _record_batch(
record, expected_hidden_size=expected_hidden_size)
ids, hidden, mask = ids.to(device), hidden.to(device, dtype), mask.to(device)
losses, _, accuracy, head_counts, _ = model(
ids, hidden, mask, torch.ones_like(ids))
for head in range(heads):
tokens = int(head_counts[head])
if tokens == 0:
continue
loss_sums[head] += float(losses[head]) * tokens
correct[head] += float(accuracy[head]) * tokens
counts[head] += tokens
samples += 1
if samples == 0:
raise ValueError("No held-out records were evaluated")
per_head = []
for head in range(heads):
tokens = int(counts[head])
per_head.append({
"head": head,
"prediction_offset": head + 2,
"tokens": tokens,
"loss": float(loss_sums[head]) / tokens if tokens else None,
"accuracy": float(correct[head]) / tokens if tokens else None,
})
measured = [item for item in per_head if item["tokens"]]
return {
"samples": samples,
"per_head": per_head,
"aggregate": {
"sum_of_head_mean_ce": (
sum(item["loss"] for item in measured) if measured else None),
"measured_heads": len(measured),
},
}


def _serialize_report(report):
try:
return json.dumps(report, indent=2, sort_keys=True, allow_nan=False)
except ValueError as exc:
raise ValueError("Held-out report contains non-finite metrics") from exc


def _parse_args():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--draft", type=Path, required=True)
parser.add_argument("--dataset", type=Path, required=True)
parser.add_argument("--split", default="eval")
parser.add_argument("--limit", type=int, default=None)
parser.add_argument("--device", default="cpu")
parser.add_argument("--output", type=Path, required=True)
return parser.parse_args()


def main():
args = _parse_args()
from aspec.offline.dataset import OfflineDataset

dataset = OfflineDataset(args.dataset)
binding = dataset.metadata.get("binding") or {}
contract = _validate_binding(binding)
rows = dataset.rows(args.split)
if args.limit is not None:
if args.limit < 1:
raise ValueError("--limit must be positive")
rows = rows[: args.limit]
draft = load_native_draft(args.draft, device=args.device)
_validate_target_binding(contract, draft.config)
report = evaluate_records(
draft, (dataset.load(row) for row in rows), device=args.device,
expected_hidden_size=int(contract["target_hidden_size"]))
report.update({
"draft_manifest_sha256": hashlib.sha256(
(args.draft / "export_manifest.json").read_bytes()).hexdigest(),
"dataset_identity": dataset.identity,
"feature_contract_id": contract["contract_id"],
"target_identity": binding.get("target_identity"),
"split": args.split,
"medusa_num_heads": draft.medusa_num_heads,
"medusa_num_layers": draft.medusa_num_layers,
})
serialized = _serialize_report(report)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(serialized + "\n")
print(serialized)


if __name__ == "__main__":
main()
27 changes: 26 additions & 1 deletion aspec/draft_artifact.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ def load_native_draft(path, *, device="cpu", dtype=None):
state = load_file(str(path / "model.safetensors"), device="cpu")
if len(state) != manifest.get("tensor_count"):
raise ValueError("Native artifact tensor count mismatch")
checkpoint_dtype = state["embed_tokens.weight"].dtype
checkpoint_dtype = (
state["embed_tokens.weight"].dtype if "embed_tokens.weight" in state
else next(iter(state.values())).dtype)
model = AutoEagle3DraftModel.from_config(config, torch_dtype=dtype or checkpoint_dtype)
# Establish mixed precision BEFORE load; casting FP32 compressors through
# BF16 and back would silently destroy the checkpoint's retained precision.
Expand All @@ -134,6 +136,29 @@ def load_native_draft(path, *, device="cpu", dtype=None):
return model


def verify_native_export(path, *, expected=None):
"""Strictly reload a native artifact and compare it with the exported values.

``expected`` is the state that was written; comparing the reloaded model with
it catches loader-side casts or drops. Without it the artifact's own bytes are
re-read, which only proves the loader is idempotent, not that it roundtripped
the exported tensors.
"""
path = Path(path)
model = load_native_draft(path)
reloaded = model.state_dict()
if expected is None:
expected = load_file(str(path / "model.safetensors"))
if set(reloaded) != set(expected):
raise ValueError("Native reload changed the exported tensor set")
for key, value in reloaded.items():
if value.shape != expected[key].shape or value.dtype != expected[key].dtype:
raise ValueError(f"Native reload changed {key} shape/dtype")
if not torch.equal(value, expected[key]):
raise ValueError(f"Native reload changed {key} values")
return {"method": "strict_load_and_tensor_equality", "tensors": len(reloaded)}


class BlockDraftConsumer:
"""One DSpark/DFlash block from verifier features and its frozen output head.

Expand Down
Loading