Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions aspec/commands/convert_to_hf.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ def _serving_config(raw_config: dict) -> dict:
native["architectures"] = ["Qwen3_5ForCausalLMMTP"]
native["mtp_num_hidden_layers"] = 1
return native
if raw_config.get("architectures") == ["JetSpecDraftModel"]:
raise NotImplementedError(
"JetSpec requires --format aspec; SGLang 0.5.21 has no JetSpec serving loader"
)
attention = resolve_draft_attention(raw_config)
if attention["csa"] is not None:
raise NotImplementedError(
Expand Down
1 change: 1 addition & 0 deletions aspec/config/draft_algorithm.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
"NativeMTPDraftModel": "mtp",
"MedusaDraftModel": "medusa",
"DominoDraftModel": "domino",
"JetSpecDraftModel": "jetspec",

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried the default aspec export on a JetSpec checkpoint. It succeeded and wrote JETSPEC as the speculative algorithm in the manifest, but SGLang 0.5.21 doesn’t recognize that name. Could we have --format sglang return a clear error for JetSpec until that export path is supported?

"DFlashDraftModel": "dflash",
"DFlash2DraftModel": "dflash2",
"DSparkDraftModel": "dspark",
Expand Down
21 changes: 21 additions & 0 deletions aspec/config/jetspec_qwen3_8b_draft_config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"architectures": ["JetSpecDraftModel"],
"model_type": "jetspec",
"hidden_size": 4096,
"intermediate_size": 12288,
"num_hidden_layers": 5,
"num_attention_heads": 32,
"num_key_value_heads": 8,
"vocab_size": 151936,
"rms_norm_eps": 1e-6,
"max_position_embeddings": 40960,
"rope_theta": 1000000.0,
"num_target_layers": 5,
"target_hidden_size": 4096,
"target_num_hidden_layers": 36,
"target_layer_ids": [1, 9, 17, 25, 33],
"mask_token_id": 151669,
"block_size": 16,
"is_causal": true,
"tie_word_embeddings": false
}
6 changes: 6 additions & 0 deletions aspec/config/train_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,12 @@ class TrainingConfig:
domino_lambda_base_decay_ratio: Optional[float] = None
# Report backbone-only per-position held-out metrics alongside corrected.
domino_eval_backbone: bool = False
# JetSpec distills the frozen Target distribution with T^2 * KL(p || q).
# Full-vocabulary KL intermediates are processed in bounded token chunks.
jetspec_distillation_temperature: float = 1.0
jetspec_kl_chunk_rows: int = 256
# Main JetSpec results use uniform per-position weights (gamma = 0).
jetspec_loss_decay_gamma: float = 0.0
# Pack multiple complete documents into one fixed row. The controller owns
# grouping; the trainer preserves document boundaries for attention/RoPE.
dflash_packing: bool = False
Expand Down
19 changes: 17 additions & 2 deletions aspec/evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -1055,9 +1055,24 @@ def attention_observed(item):
attention_policy_observed = bool(consumers) and all(
attention_observed(item) for item in consumers
)
if (
attention_decision = next(
(
planned_attention_backend in {"fa4", "cudnn_flex_attention"}
item
for item in self._plan.capabilities
if item.name == "draft_attention_execution"
),
None,
)
attention_planned = (
attention_decision is not None
and attention_decision.effective_state != "omitted"
)
if (
attention_planned
and (
planned_attention_backend in {
"fa4", "cudnn_flex_attention", "flex_attention",

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The JetSpec case looks good now. One small thing I noticed with plain DSpark at CP=1 is that the plan marks draft_attention_execution as omitted, then the recorder flips it to active while effective stays null. Could we skip activation when the plan omitted that check? This only affects the report, so it’s non-blocking from my side.

}
or selected_policy_requested
)
and attention_policy_observed
Expand Down
3 changes: 3 additions & 0 deletions aspec/models/draft/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
K3DSparkConfig,
K3DSparkModel,
)
from aspec.models.draft.jetspec import JetSpecConfig, JetSpecDraftModel

__all__ = [
"AutoDraftModelConfig",
Expand All @@ -38,6 +39,8 @@
"DFlash2DraftModel",
"DominoConfig",
"DominoDraftModel",
"JetSpecConfig",
"JetSpecDraftModel",
"DSparkConfig",
"DSparkDraftModel",
"K3DSparkConfig",
Expand Down
3 changes: 3 additions & 0 deletions aspec/models/draft/auto.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
K3DSparkModel,
)
from aspec.models.draft.eagle4 import Eagle4DraftModel
from aspec.models.draft.jetspec import JetSpecConfig, JetSpecDraftModel
from aspec.models.draft.llama3_eagle import LlamaForCausalLMEagle3
from aspec.models.draft.medusa import MedusaConfig, MedusaDraftModel
from aspec.models.draft.mtp import NativeMTPConfig, NativeMTPDraftModel
Expand All @@ -50,6 +51,7 @@ class AutoEagle3DraftModel(AutoModelForCausalLMBase):
NativeMTPConfig: NativeMTPDraftModel,
MedusaConfig: MedusaDraftModel,
DominoConfig: DominoDraftModel,
JetSpecConfig: JetSpecDraftModel,
DFlashConfig: DFlashDraftModel,
DFlash2Config: DFlash2DraftModel,
DSparkConfig: DSparkDraftModel,
Expand All @@ -74,6 +76,7 @@ class AutoDraftModelConfig:
"NativeMTPDraftModel": NativeMTPConfig,
"MedusaDraftModel": MedusaConfig,
"DominoDraftModel": DominoConfig,
"JetSpecDraftModel": JetSpecConfig,
"DFlashDraftModel": DFlashConfig,
"DFlash2DraftModel": DFlash2Config,
"Qwen3DSparkModel": DSparkConfig,
Expand Down
88 changes: 86 additions & 2 deletions aspec/planning.py
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,71 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]:
if not isinstance(_select(config, "training.domino_eval_backbone", False), bool):
raise PlanCompileError("domino_eval_backbone must be boolean")

if family == "jetspec":
from aspec.models.draft.jetspec import JetSpecConfig

try:
raw = JetSpecConfig.from_dict(raw).to_dict()
except (ValueError, TypeError) as exc:
raise PlanCompileError(str(exc)) from exc
if raw["block_size"] != _select(config, "training.dflash_block_size"):
raise PlanCompileError(
"JetSpec block_size must match training.dflash_block_size"
)
if raw.get("cudnn_gqa_mode", "auto") != "auto":
raise PlanCompileError(
"JetSpec requires cudnn_gqa_mode=auto; the FlexAttention mask "
"controls causal block visibility"
)
backend = _select(config, "training.attention_backend")
if backend != "flex_attention":
raise PlanCompileError(
"JetSpec requires training.attention_backend=flex_attention; "
"that is the masked block-attention path the CUDA dispatcher uses"
)
if _select(config, "training.draft_context_parallel_size", 1) != 1:
raise PlanCompileError("JetSpec currently supports draft_context_parallel_size=1")
if _select(config, "training.dflash_packing", False):
raise PlanCompileError(
"JetSpec does not support sequence packing; set "
"training.dflash_packing=false"
)
if _select(config, "training.compile_model", False):
raise PlanCompileError("JetSpec whole-model compile has not been enabled")
if _select(config, "training.dflash_loss_objective") != "decay":
raise PlanCompileError("JetSpec supports the decay loss objective only")
if (_select(config, "training.dflash_ce_loss_alpha") != 0.0
or _select(config, "training.dflash_l1_loss_alpha") != 0.0):
raise PlanCompileError(
"JetSpec replaces the CE/L1 mix; set dflash_ce_loss_alpha=0.0 "
"and dflash_l1_loss_alpha=0.0"
)
temperature = _select(config, "training.jetspec_distillation_temperature", 1.0)
min_temperature = 1e-6
max_temperature = math.sqrt(3.4028234663852886e38)
if (isinstance(temperature, bool)
or not isinstance(temperature, (int, float))
or not math.isfinite(temperature)
or not min_temperature <= temperature <= max_temperature):
raise PlanCompileError(
"jetspec_distillation_temperature must be finite and in "
f"[{min_temperature:g}, {max_temperature:g}]"
)
chunk_rows = _select(config, "training.jetspec_kl_chunk_rows", 256)
if (isinstance(chunk_rows, bool) or not isinstance(chunk_rows, int)
or chunk_rows < 1):
raise PlanCompileError("jetspec_kl_chunk_rows must be a positive integer")
gamma = _select(config, "training.jetspec_loss_decay_gamma", 0.0)
if (isinstance(gamma, bool) or not isinstance(gamma, (int, float))
or not math.isfinite(gamma) or gamma < 0):
raise PlanCompileError(
"jetspec_loss_decay_gamma must be a finite non-negative number"
)
if not _select(config, "inference.store_last_hidden_states", False):
raise PlanCompileError(
"JetSpec forward KL requires inference.store_last_hidden_states=true"
)

target_layer_ids = raw.get("target_layer_ids")
if not isinstance(target_layer_ids, list) or not target_layer_ids:
raise PlanCompileError(
Expand Down Expand Up @@ -1480,6 +1545,25 @@ def _workload(
"gru_hidden_dim": draft["domino_gru_hidden_dim"],
},
}
elif family == "jetspec":
objective = {
"family": "jetspec",
"loss": "temperature_scaled_forward_kl",
"block_size": _select(config, "training.dflash_block_size"),
"num_anchors": _select(config, "training.dflash_num_anchors"),
"num_target_layers": _select(config, "training.dflash_num_target_layers"),
"loss_objective": _select(config, "training.dflash_loss_objective"),
"loss_decay_gamma": _select(config, "training.jetspec_loss_decay_gamma", 0.0),
"ce_loss_alpha": 0.0,
"l1_loss_alpha": 0.0,
"distillation_temperature": _select(
config, "training.jetspec_distillation_temperature", 1.0
),
"kl_chunk_rows": _select(config, "training.jetspec_kl_chunk_rows", 256),
"prediction_start": 1,
"teacher_feature": "last_hidden_states",
"frozen_parts": ["target_lm_head"],
}
elif family == "dspark":
objective = {
"family": family,
Expand Down Expand Up @@ -1530,7 +1614,7 @@ def _workload(
objective["frozen_parts"] = (["draft_body", "embedding", "lm_head"]
if draft["eagle4"]["stage"] == "candidate_head" else
["embedding", "candidate_head"] + (["lm_head"] if _select(config, "model.freeze_lm_head", False) else []))
if family in {"dspark", "dflash", "dflash2", "domino"}:
if family in {"dspark", "dflash", "dflash2", "domino", "jetspec"}:
objective["anchor_sampling_scope"] = anchor_scope
if draft.get("mtp_conditioning"):
from aspec.config.mtp_conditioning import mtp_conditioning_semantics
Expand Down Expand Up @@ -2127,7 +2211,7 @@ def compile_execution_plan(
or draft["attention"]["csa"] is not None
or draft["attention"]["qsa"] is not None
or draft["attention"].get("dsa") is not None
or draft["family"] == "domino"
or draft["family"] in {"domino", "jetspec"}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
)
direct_target_output = bool(
_select(config, "inference.target_prefill_memory_optimized", False)
Expand Down
60 changes: 52 additions & 8 deletions aspec/training/checkpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]:
key: getattr(args, key, None)
for key in ("ttt_length", "loss_type", "lk_eta", "ploss_weights")
}
elif getattr(args, "draft_algorithm", None) in {"dflash", "dflash2", "dspark", "domino"}:
elif getattr(args, "draft_algorithm", None) in {"dflash", "dflash2", "dspark", "domino", "jetspec"}:
topology["training_objective"] = {
"block_size": int(args.dflash_block_size),
"anchor_sampling_scope": getattr(args, "anchor_sampling_scope", "row"),
Expand All @@ -101,7 +101,7 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]:
for name, default in (("temperature", 1.0), ("top_p", 1.0), ("top_k", -1)):
key = f"l1_{side}_{name}"
topology["training_objective"][key] = getattr(args, f"dspark_{key}", default)
elif args.draft_algorithm == "domino":
elif args.draft_algorithm in {"domino", "jetspec"}:
fields += ("loss_objective",)
else:
fields += ("loss_objective", "dpace_alpha")
Expand Down Expand Up @@ -131,6 +131,17 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]:
"lr_total_steps or num_train_steps horizon"
)
topology["training_objective"]["lambda_base_total_steps"] = horizon
if args.draft_algorithm == "jetspec":
topology["training_objective"]["loss"] = "temperature_scaled_forward_kl"
topology["training_objective"]["distillation_temperature"] = getattr(
args, "jetspec_distillation_temperature", 1.0
)
topology["training_objective"]["kl_chunk_rows"] = getattr(
args, "jetspec_kl_chunk_rows", 256
)
topology["training_objective"]["loss_decay_gamma"] = getattr(
args, "jetspec_loss_decay_gamma", 0.0
)
if "training_objective" in topology:
fields = ("max_seq_length", "micro_batch_size", "draft_accumulation_steps",
"max_grad_norm", "seed")
Expand Down Expand Up @@ -193,7 +204,7 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec
raise RuntimeError(f"Exact resume cannot change dspark_{key}; use weights-only initialization")
# Checkpoints predating this option always sampled per physical row.
# Missing semantic identity must not authorize a switch to per-sample training.
if (getattr(args, "draft_algorithm", None) in {"dspark", "dflash", "dflash2", "domino"}
if (getattr(args, "draft_algorithm", None) in {"dspark", "dflash", "dflash2", "domino", "jetspec"}
and ((saved_topology or {}).get("training_objective") or {}).get(
"anchor_sampling_scope", "row"
) != getattr(args, "anchor_sampling_scope", "row")):
Expand Down Expand Up @@ -225,9 +236,10 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec
):
raise RuntimeError("Exact resume cannot change a legacy full-attention checkpoint to SWA")
if saved_topology is None:
if getattr(args, "draft_algorithm", None) == "domino":
if getattr(args, "draft_algorithm", None) in {"domino", "jetspec"}:
name = "Domino" if args.draft_algorithm == "domino" else "JetSpec"
raise RuntimeError(
"Exact resume requires training_topology for Domino checkpoints; "
f"Exact resume requires training_topology for {name} checkpoints; "
"use continual_training=true for a weights-only new stage."
)
# Older complete checkpoints recorded world_size only. Preserve that
Expand Down Expand Up @@ -264,6 +276,26 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec
"constant-lambda checkpoint; use continual_training=true for a "
"new phase."
)
if getattr(args, "draft_algorithm", None) == "jetspec":
objective = saved_topology.get("training_objective") or {}
semantics = saved_topology.get("draft_model_semantics") or {}
missing = [
key for key in (
"distillation_temperature", "kl_chunk_rows", "block_size",
"loss", "loss_decay_gamma", "ce_loss_alpha", "l1_loss_alpha",
)
if key not in objective
]
missing += [
key for key in ("architectures", "block_size", "is_causal")
if key not in semantics
]
if missing:
raise RuntimeError(
"Exact resume requires complete JetSpec training_topology; "
f"missing {missing}. Use continual_training=true for a "
"weights-only new stage."
)
if current_policy.get("dsa") is not None:
# Chunk workspace is an execution bound, not the DSA objective. Preserve
# every recorded semantic field while allowing a different memory budget.
Expand Down Expand Up @@ -1138,6 +1170,18 @@ def finalize_load(actor: Any, checkpoint_payload: dict[str, Any] | None) -> None
dist.barrier()


def _checkpoint_writer(path: Path):
"""Use the serial CPU loader for deterministic CUDA-to-host copies.

PyTorch's overlapping pinned-copy loader can raise ``cudaErrorInvalidValue``
on some hosts after several optimizer steps. The serial loader keeps the
checkpoint contents unchanged while avoiding that runtime-only failure.
"""
from torch.distributed.checkpoint import FileSystemWriter

return FileSystemWriter(path, per_thread_copy_ahead=0)


def save(actor: Any, step: int) -> None:
"""Save checkpoint to disk.

Expand All @@ -1164,17 +1208,17 @@ def save(actor: Any, step: int) -> None:
# Save model weights
model_state = ModelState(actor.model)
state_dict = {"model_state": model_state}
dcp.save(state_dict, checkpoint_id=str(model_dir))
dcp.save(state_dict, storage_writer=_checkpoint_writer(model_dir))

if hasattr(actor, "optimizer") and actor.optimizer is not None:
optimizer_state = OptimizerState(actor.model, actor.optimizer)
optim_state_dict = {"optim_state": optimizer_state}
dcp.save(optim_state_dict, checkpoint_id=str(optimizer_dir))
dcp.save(optim_state_dict, storage_writer=_checkpoint_writer(optimizer_dir))

if hasattr(actor, "lr_scheduler") and actor.lr_scheduler is not None:
lr_scheduler_state = LRSchedulerState(actor.lr_scheduler)
lr_scheduler_state_dict = {"lr_scheduler_state": lr_scheduler_state}
dcp.save(lr_scheduler_state_dict, checkpoint_id=str(lr_scheduler_dir))
dcp.save(lr_scheduler_state_dict, storage_writer=_checkpoint_writer(lr_scheduler_dir))

_save_rank_rng_state(
checkpoint_dir,
Expand Down
Loading