diff --git a/aspec/commands/convert_to_hf.py b/aspec/commands/convert_to_hf.py index d71b58f..3d08f1a 100644 --- a/aspec/commands/convert_to_hf.py +++ b/aspec/commands/convert_to_hf.py @@ -171,6 +171,10 @@ def _serving_config(raw_config: dict) -> dict: native["architectures"] = ["Qwen3_5ForCausalLMMTP"] native["mtp_num_hidden_layers"] = 1 return native + if raw_config.get("architectures") == ["JetSpecDraftModel"]: + raise NotImplementedError( + "JetSpec requires --format aspec; SGLang 0.5.21 has no JetSpec serving loader" + ) attention = resolve_draft_attention(raw_config) if attention["csa"] is not None: raise NotImplementedError( diff --git a/aspec/config/draft_algorithm.py b/aspec/config/draft_algorithm.py index 1605840..bbceb49 100644 --- a/aspec/config/draft_algorithm.py +++ b/aspec/config/draft_algorithm.py @@ -20,6 +20,7 @@ "NativeMTPDraftModel": "mtp", "MedusaDraftModel": "medusa", "DominoDraftModel": "domino", + "JetSpecDraftModel": "jetspec", "DFlashDraftModel": "dflash", "DFlash2DraftModel": "dflash2", "DSparkDraftModel": "dspark", diff --git a/aspec/config/jetspec_qwen3_8b_draft_config.json b/aspec/config/jetspec_qwen3_8b_draft_config.json new file mode 100644 index 0000000..53846bc --- /dev/null +++ b/aspec/config/jetspec_qwen3_8b_draft_config.json @@ -0,0 +1,21 @@ +{ + "architectures": ["JetSpecDraftModel"], + "model_type": "jetspec", + "hidden_size": 4096, + "intermediate_size": 12288, + "num_hidden_layers": 5, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "vocab_size": 151936, + "rms_norm_eps": 1e-6, + "max_position_embeddings": 40960, + "rope_theta": 1000000.0, + "num_target_layers": 5, + "target_hidden_size": 4096, + "target_num_hidden_layers": 36, + "target_layer_ids": [1, 9, 17, 25, 33], + "mask_token_id": 151669, + "block_size": 16, + "is_causal": true, + "tie_word_embeddings": false +} diff --git a/aspec/config/train_config.py b/aspec/config/train_config.py index 4fb5a92..d2b858d 100644 --- a/aspec/config/train_config.py +++ b/aspec/config/train_config.py @@ -247,6 +247,12 @@ class TrainingConfig: domino_lambda_base_decay_ratio: Optional[float] = None # Report backbone-only per-position held-out metrics alongside corrected. domino_eval_backbone: bool = False + # JetSpec distills the frozen Target distribution with T^2 * KL(p || q). + # Full-vocabulary KL intermediates are processed in bounded token chunks. + jetspec_distillation_temperature: float = 1.0 + jetspec_kl_chunk_rows: int = 256 + # Main JetSpec results use uniform per-position weights (gamma = 0). + jetspec_loss_decay_gamma: float = 0.0 # Pack multiple complete documents into one fixed row. The controller owns # grouping; the trainer preserves document boundaries for attention/RoPE. dflash_packing: bool = False diff --git a/aspec/evidence.py b/aspec/evidence.py index ccfc32e..958270f 100644 --- a/aspec/evidence.py +++ b/aspec/evidence.py @@ -1055,9 +1055,24 @@ def attention_observed(item): attention_policy_observed = bool(consumers) and all( attention_observed(item) for item in consumers ) - if ( + attention_decision = next( ( - planned_attention_backend in {"fa4", "cudnn_flex_attention"} + item + for item in self._plan.capabilities + if item.name == "draft_attention_execution" + ), + None, + ) + attention_planned = ( + attention_decision is not None + and attention_decision.effective_state != "omitted" + ) + if ( + attention_planned + and ( + planned_attention_backend in { + "fa4", "cudnn_flex_attention", "flex_attention", + } or selected_policy_requested ) and attention_policy_observed diff --git a/aspec/models/draft/__init__.py b/aspec/models/draft/__init__.py index aa72ac3..773fc05 100644 --- a/aspec/models/draft/__init__.py +++ b/aspec/models/draft/__init__.py @@ -28,6 +28,7 @@ K3DSparkConfig, K3DSparkModel, ) +from aspec.models.draft.jetspec import JetSpecConfig, JetSpecDraftModel __all__ = [ "AutoDraftModelConfig", @@ -38,6 +39,8 @@ "DFlash2DraftModel", "DominoConfig", "DominoDraftModel", + "JetSpecConfig", + "JetSpecDraftModel", "DSparkConfig", "DSparkDraftModel", "K3DSparkConfig", diff --git a/aspec/models/draft/auto.py b/aspec/models/draft/auto.py index e9e802a..044bcca 100644 --- a/aspec/models/draft/auto.py +++ b/aspec/models/draft/auto.py @@ -36,6 +36,7 @@ K3DSparkModel, ) from aspec.models.draft.eagle4 import Eagle4DraftModel +from aspec.models.draft.jetspec import JetSpecConfig, JetSpecDraftModel from aspec.models.draft.llama3_eagle import LlamaForCausalLMEagle3 from aspec.models.draft.medusa import MedusaConfig, MedusaDraftModel from aspec.models.draft.mtp import NativeMTPConfig, NativeMTPDraftModel @@ -50,6 +51,7 @@ class AutoEagle3DraftModel(AutoModelForCausalLMBase): NativeMTPConfig: NativeMTPDraftModel, MedusaConfig: MedusaDraftModel, DominoConfig: DominoDraftModel, + JetSpecConfig: JetSpecDraftModel, DFlashConfig: DFlashDraftModel, DFlash2Config: DFlash2DraftModel, DSparkConfig: DSparkDraftModel, @@ -74,6 +76,7 @@ class AutoDraftModelConfig: "NativeMTPDraftModel": NativeMTPConfig, "MedusaDraftModel": MedusaConfig, "DominoDraftModel": DominoConfig, + "JetSpecDraftModel": JetSpecConfig, "DFlashDraftModel": DFlashConfig, "DFlash2DraftModel": DFlash2Config, "Qwen3DSparkModel": DSparkConfig, diff --git a/aspec/planning.py b/aspec/planning.py index b83445d..9d3856e 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -452,6 +452,71 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]: if not isinstance(_select(config, "training.domino_eval_backbone", False), bool): raise PlanCompileError("domino_eval_backbone must be boolean") + if family == "jetspec": + from aspec.models.draft.jetspec import JetSpecConfig + + try: + raw = JetSpecConfig.from_dict(raw).to_dict() + except (ValueError, TypeError) as exc: + raise PlanCompileError(str(exc)) from exc + if raw["block_size"] != _select(config, "training.dflash_block_size"): + raise PlanCompileError( + "JetSpec block_size must match training.dflash_block_size" + ) + if raw.get("cudnn_gqa_mode", "auto") != "auto": + raise PlanCompileError( + "JetSpec requires cudnn_gqa_mode=auto; the FlexAttention mask " + "controls causal block visibility" + ) + backend = _select(config, "training.attention_backend") + if backend != "flex_attention": + raise PlanCompileError( + "JetSpec requires training.attention_backend=flex_attention; " + "that is the masked block-attention path the CUDA dispatcher uses" + ) + if _select(config, "training.draft_context_parallel_size", 1) != 1: + raise PlanCompileError("JetSpec currently supports draft_context_parallel_size=1") + if _select(config, "training.dflash_packing", False): + raise PlanCompileError( + "JetSpec does not support sequence packing; set " + "training.dflash_packing=false" + ) + if _select(config, "training.compile_model", False): + raise PlanCompileError("JetSpec whole-model compile has not been enabled") + if _select(config, "training.dflash_loss_objective") != "decay": + raise PlanCompileError("JetSpec supports the decay loss objective only") + if (_select(config, "training.dflash_ce_loss_alpha") != 0.0 + or _select(config, "training.dflash_l1_loss_alpha") != 0.0): + raise PlanCompileError( + "JetSpec replaces the CE/L1 mix; set dflash_ce_loss_alpha=0.0 " + "and dflash_l1_loss_alpha=0.0" + ) + temperature = _select(config, "training.jetspec_distillation_temperature", 1.0) + min_temperature = 1e-6 + max_temperature = math.sqrt(3.4028234663852886e38) + if (isinstance(temperature, bool) + or not isinstance(temperature, (int, float)) + or not math.isfinite(temperature) + or not min_temperature <= temperature <= max_temperature): + raise PlanCompileError( + "jetspec_distillation_temperature must be finite and in " + f"[{min_temperature:g}, {max_temperature:g}]" + ) + chunk_rows = _select(config, "training.jetspec_kl_chunk_rows", 256) + if (isinstance(chunk_rows, bool) or not isinstance(chunk_rows, int) + or chunk_rows < 1): + raise PlanCompileError("jetspec_kl_chunk_rows must be a positive integer") + gamma = _select(config, "training.jetspec_loss_decay_gamma", 0.0) + if (isinstance(gamma, bool) or not isinstance(gamma, (int, float)) + or not math.isfinite(gamma) or gamma < 0): + raise PlanCompileError( + "jetspec_loss_decay_gamma must be a finite non-negative number" + ) + if not _select(config, "inference.store_last_hidden_states", False): + raise PlanCompileError( + "JetSpec forward KL requires inference.store_last_hidden_states=true" + ) + target_layer_ids = raw.get("target_layer_ids") if not isinstance(target_layer_ids, list) or not target_layer_ids: raise PlanCompileError( @@ -1480,6 +1545,25 @@ def _workload( "gru_hidden_dim": draft["domino_gru_hidden_dim"], }, } + elif family == "jetspec": + objective = { + "family": "jetspec", + "loss": "temperature_scaled_forward_kl", + "block_size": _select(config, "training.dflash_block_size"), + "num_anchors": _select(config, "training.dflash_num_anchors"), + "num_target_layers": _select(config, "training.dflash_num_target_layers"), + "loss_objective": _select(config, "training.dflash_loss_objective"), + "loss_decay_gamma": _select(config, "training.jetspec_loss_decay_gamma", 0.0), + "ce_loss_alpha": 0.0, + "l1_loss_alpha": 0.0, + "distillation_temperature": _select( + config, "training.jetspec_distillation_temperature", 1.0 + ), + "kl_chunk_rows": _select(config, "training.jetspec_kl_chunk_rows", 256), + "prediction_start": 1, + "teacher_feature": "last_hidden_states", + "frozen_parts": ["target_lm_head"], + } elif family == "dspark": objective = { "family": family, @@ -1530,7 +1614,7 @@ def _workload( objective["frozen_parts"] = (["draft_body", "embedding", "lm_head"] if draft["eagle4"]["stage"] == "candidate_head" else ["embedding", "candidate_head"] + (["lm_head"] if _select(config, "model.freeze_lm_head", False) else [])) - if family in {"dspark", "dflash", "dflash2", "domino"}: + if family in {"dspark", "dflash", "dflash2", "domino", "jetspec"}: objective["anchor_sampling_scope"] = anchor_scope if draft.get("mtp_conditioning"): from aspec.config.mtp_conditioning import mtp_conditioning_semantics @@ -2127,7 +2211,7 @@ def compile_execution_plan( or draft["attention"]["csa"] is not None or draft["attention"]["qsa"] is not None or draft["attention"].get("dsa") is not None - or draft["family"] == "domino" + or draft["family"] in {"domino", "jetspec"} ) direct_target_output = bool( _select(config, "inference.target_prefill_memory_optimized", False) diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index 3b9fab2..0cb0303 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -86,7 +86,7 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: key: getattr(args, key, None) for key in ("ttt_length", "loss_type", "lk_eta", "ploss_weights") } - elif getattr(args, "draft_algorithm", None) in {"dflash", "dflash2", "dspark", "domino"}: + elif getattr(args, "draft_algorithm", None) in {"dflash", "dflash2", "dspark", "domino", "jetspec"}: topology["training_objective"] = { "block_size": int(args.dflash_block_size), "anchor_sampling_scope": getattr(args, "anchor_sampling_scope", "row"), @@ -101,7 +101,7 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: for name, default in (("temperature", 1.0), ("top_p", 1.0), ("top_k", -1)): key = f"l1_{side}_{name}" topology["training_objective"][key] = getattr(args, f"dspark_{key}", default) - elif args.draft_algorithm == "domino": + elif args.draft_algorithm in {"domino", "jetspec"}: fields += ("loss_objective",) else: fields += ("loss_objective", "dpace_alpha") @@ -131,6 +131,17 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: "lr_total_steps or num_train_steps horizon" ) topology["training_objective"]["lambda_base_total_steps"] = horizon + if args.draft_algorithm == "jetspec": + topology["training_objective"]["loss"] = "temperature_scaled_forward_kl" + topology["training_objective"]["distillation_temperature"] = getattr( + args, "jetspec_distillation_temperature", 1.0 + ) + topology["training_objective"]["kl_chunk_rows"] = getattr( + args, "jetspec_kl_chunk_rows", 256 + ) + topology["training_objective"]["loss_decay_gamma"] = getattr( + args, "jetspec_loss_decay_gamma", 0.0 + ) if "training_objective" in topology: fields = ("max_seq_length", "micro_batch_size", "draft_accumulation_steps", "max_grad_norm", "seed") @@ -193,7 +204,7 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec raise RuntimeError(f"Exact resume cannot change dspark_{key}; use weights-only initialization") # Checkpoints predating this option always sampled per physical row. # Missing semantic identity must not authorize a switch to per-sample training. - if (getattr(args, "draft_algorithm", None) in {"dspark", "dflash", "dflash2", "domino"} + if (getattr(args, "draft_algorithm", None) in {"dspark", "dflash", "dflash2", "domino", "jetspec"} and ((saved_topology or {}).get("training_objective") or {}).get( "anchor_sampling_scope", "row" ) != getattr(args, "anchor_sampling_scope", "row")): @@ -225,9 +236,10 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec ): raise RuntimeError("Exact resume cannot change a legacy full-attention checkpoint to SWA") if saved_topology is None: - if getattr(args, "draft_algorithm", None) == "domino": + if getattr(args, "draft_algorithm", None) in {"domino", "jetspec"}: + name = "Domino" if args.draft_algorithm == "domino" else "JetSpec" raise RuntimeError( - "Exact resume requires training_topology for Domino checkpoints; " + f"Exact resume requires training_topology for {name} checkpoints; " "use continual_training=true for a weights-only new stage." ) # Older complete checkpoints recorded world_size only. Preserve that @@ -264,6 +276,26 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec "constant-lambda checkpoint; use continual_training=true for a " "new phase." ) + if getattr(args, "draft_algorithm", None) == "jetspec": + objective = saved_topology.get("training_objective") or {} + semantics = saved_topology.get("draft_model_semantics") or {} + missing = [ + key for key in ( + "distillation_temperature", "kl_chunk_rows", "block_size", + "loss", "loss_decay_gamma", "ce_loss_alpha", "l1_loss_alpha", + ) + if key not in objective + ] + missing += [ + key for key in ("architectures", "block_size", "is_causal") + if key not in semantics + ] + if missing: + raise RuntimeError( + "Exact resume requires complete JetSpec training_topology; " + f"missing {missing}. Use continual_training=true for a " + "weights-only new stage." + ) if current_policy.get("dsa") is not None: # Chunk workspace is an execution bound, not the DSA objective. Preserve # every recorded semantic field while allowing a different memory budget. @@ -1138,6 +1170,18 @@ def finalize_load(actor: Any, checkpoint_payload: dict[str, Any] | None) -> None dist.barrier() +def _checkpoint_writer(path: Path): + """Use the serial CPU loader for deterministic CUDA-to-host copies. + + PyTorch's overlapping pinned-copy loader can raise ``cudaErrorInvalidValue`` + on some hosts after several optimizer steps. The serial loader keeps the + checkpoint contents unchanged while avoiding that runtime-only failure. + """ + from torch.distributed.checkpoint import FileSystemWriter + + return FileSystemWriter(path, per_thread_copy_ahead=0) + + def save(actor: Any, step: int) -> None: """Save checkpoint to disk. @@ -1164,17 +1208,17 @@ def save(actor: Any, step: int) -> None: # Save model weights model_state = ModelState(actor.model) state_dict = {"model_state": model_state} - dcp.save(state_dict, checkpoint_id=str(model_dir)) + dcp.save(state_dict, storage_writer=_checkpoint_writer(model_dir)) if hasattr(actor, "optimizer") and actor.optimizer is not None: optimizer_state = OptimizerState(actor.model, actor.optimizer) optim_state_dict = {"optim_state": optimizer_state} - dcp.save(optim_state_dict, checkpoint_id=str(optimizer_dir)) + dcp.save(optim_state_dict, storage_writer=_checkpoint_writer(optimizer_dir)) if hasattr(actor, "lr_scheduler") and actor.lr_scheduler is not None: lr_scheduler_state = LRSchedulerState(actor.lr_scheduler) lr_scheduler_state_dict = {"lr_scheduler_state": lr_scheduler_state} - dcp.save(lr_scheduler_state_dict, checkpoint_id=str(lr_scheduler_dir)) + dcp.save(lr_scheduler_state_dict, storage_writer=_checkpoint_writer(lr_scheduler_dir)) _save_rank_rng_state( checkpoint_dir, diff --git a/aspec/training/jetspec_trainer.py b/aspec/training/jetspec_trainer.py new file mode 100644 index 0000000..a17d526 --- /dev/null +++ b/aspec/training/jetspec_trainer.py @@ -0,0 +1,94 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""JetSpec trainer — causal parallel Draft head with target forward KL.""" + +import math +from argparse import Namespace + +from aspec.models.draft.jetspec import JetSpecConfig, JetSpecDraftModel +from aspec.models.jetspec import JetSpecModel +from aspec.training.dflash_trainer import DFlashTrainer + +_MIN_TEMPERATURE = 1e-6 +_MAX_TEMPERATURE = math.sqrt(3.4028234663852886e38) + + +class JetSpecTrainer(DFlashTrainer): + """JetSpec-specific trainer; reuses the DFlash optimizer/checkpoint path.""" + + _draft_config_class = JetSpecConfig + + def __init__(self, args: Namespace): + super().__init__(args) + self.distillation_temperature = float( + getattr(args, "jetspec_distillation_temperature", 1.0) + ) + self.kl_chunk_rows = getattr(args, "jetspec_kl_chunk_rows", 256) + self.loss_decay_gamma = float( + getattr(args, "jetspec_loss_decay_gamma", 0.0) + ) + temperature = self.distillation_temperature + if ( + isinstance(getattr(args, "jetspec_distillation_temperature", 1.0), bool) + or not math.isfinite(temperature) + or not _MIN_TEMPERATURE <= temperature <= _MAX_TEMPERATURE + ): + raise ValueError( + "jetspec_distillation_temperature must be finite and in " + f"[{_MIN_TEMPERATURE:g}, {_MAX_TEMPERATURE:g}]" + ) + if ( + isinstance(self.kl_chunk_rows, bool) + or not isinstance(self.kl_chunk_rows, int) + or self.kl_chunk_rows < 1 + ): + raise ValueError("jetspec_kl_chunk_rows must be a positive integer") + if not math.isfinite(self.loss_decay_gamma) or self.loss_decay_gamma < 0: + raise ValueError("jetspec_loss_decay_gamma must be finite and non-negative") + if self.loss_objective != "decay": + raise ValueError("JetSpec supports the decay/uniform weighting objective only") + if self.ce_loss_alpha != 0.0 or self.l1_loss_alpha != 0.0: + raise ValueError( + "JetSpec replaces CE/L1 with forward KL; set " + "dflash_ce_loss_alpha=0.0 and dflash_l1_loss_alpha=0.0" + ) + + def _build_draft_model(self, config): + if config.block_size != self.block_size: + raise ValueError( + "training.dflash_block_size must match jetspec_config.block_size " + f"({self.block_size} != {config.block_size})" + ) + if config.num_target_layers != self.num_target_layers: + raise ValueError( + "training.dflash_num_target_layers must match the number of " + f"jetspec_config.target_layer_ids ({self.num_target_layers} != " + f"{config.num_target_layers})" + ) + return JetSpecDraftModel(config) + + def _build_training_wrapper(self, draft_model): + wrapper = JetSpecModel( + draft_model=draft_model, + block_size=self.block_size, + num_anchors=self.num_anchors, + loss_objective=self.loss_objective, + dpace_alpha=self.dpace_alpha, + loss_decay_gamma=self.loss_decay_gamma, + ce_loss_alpha=self.ce_loss_alpha, + l1_loss_alpha=self.l1_loss_alpha, + distillation_temperature=self.distillation_temperature, + kl_chunk_rows=self.kl_chunk_rows, + deterministic_anchor_sampling=getattr( + self.args, "deterministic_anchor_sampling", False + ), + anchor_seed=getattr(self.args, "seed", 0), + anchor_sampling_scope=getattr(self.args, "anchor_sampling_scope", "row"), + attention_backend=getattr(self.args, "attention_backend", "flex_attention"), + draft_context_parallel_size=getattr( + self.args, "draft_context_parallel_size", 1 + ), + ) + self._anchor_slot_offset = wrapper.prediction_start + return wrapper diff --git a/aspec/training/trainer_actor.py b/aspec/training/trainer_actor.py index 3077962..67fcdea 100644 --- a/aspec/training/trainer_actor.py +++ b/aspec/training/trainer_actor.py @@ -56,6 +56,10 @@ def _trainer_class_for_config(draft_model_config): from aspec.training.domino_trainer import DominoTrainer return DominoTrainer + if getattr(draft_model_config, "architectures", None) == ["JetSpecDraftModel"]: + from aspec.training.jetspec_trainer import JetSpecTrainer + + return JetSpecTrainer if isinstance(draft_model_config, DFlash2Config): from aspec.training.dflash2_trainer import DFlash2Trainer @@ -75,7 +79,7 @@ def _trainer_class_for_config(draft_model_config): return Eagle3Trainer raise ValueError( - "ASpec supports DSpark, DFlash, Domino, EAGLE3, native MTP and Medusa draft model configs; " + "ASpec supports DSpark, DFlash, Domino, JetSpec, EAGLE3, native MTP and Medusa draft model configs; " f"got {type(draft_model_config).__name__}" ) diff --git a/configs/examples/jetspec_qwen3_8b_4k_tcp.yaml b/configs/examples/jetspec_qwen3_8b_4k_tcp.yaml new file mode 100644 index 0000000..e69cd27 --- /dev/null +++ b/configs/examples/jetspec_qwen3_8b_4k_tcp.yaml @@ -0,0 +1,95 @@ +# Supplementary configuration example; see the support matrix for supported workflows. +# Single-node JetSpec on a frozen Qwen3-8B Target: one Target GPU plus two Draft GPUs, +# 4K, CP1, no packing, FlexAttention, Store/TCP transport (no NVLink). This is a +# functional smoke test, not evidence of convergence, acceptance or decoding quality. + +model: + target_model_path: ${oc.env:ASPEC_TARGET_MODEL} + native_context_length: 32768 + trust_remote_code: false + draft_model_config: aspec/config/jetspec_qwen3_8b_draft_config.json + +dataset: + train_data_path: ../../examples/data/sample_conversations.jsonl + eval_data_path: null + chat_template: qwen3-instruct + min_loss_tokens: 32 + shuffle_dataset: false + +training: + attention_backend: flex_attention + draft_context_parallel_size: 1 + micro_batch_size: 1 + draft_accumulation_steps: 4 + learning_rate: 6e-4 + min_lr: 6e-5 + weight_decay: 0.0 + max_concurrent_batches: 1 + streaming_dispatch_with_training: true + pipeline_dispatch_with_training: true + max_grad_norm: 1.0 + max_seq_length: 4096 + num_epochs: 100 + num_train_steps: 8 + lr_total_steps: 8 + seed: 42 + training_num_gpus_per_node: 2 + training_num_nodes: 1 + placement_strategy: inference_first + fsdp_strategy: REPLICATE + fsdp_reduce_dtype: bfloat16 + prefetch_depth: 1 + save_interval: 0 + save_per_epoch: false + save_final_checkpoint: true + warmup_ratio: 0.04 + dflash_packing: false + dflash_block_size: 16 + dflash_num_target_layers: 5 + dflash_num_anchors: 128 + dflash_loss_objective: decay + dflash_ce_loss_alpha: 0.0 + dflash_l1_loss_alpha: 0.0 + jetspec_distillation_temperature: 1.0 + jetspec_kl_chunk_rows: 256 + jetspec_loss_decay_gamma: 0.0 + gradient_checkpointing: true + +inference: + inference_engine_type: sgl + store_last_hidden_states: true + target_prefill_memory_optimized: false + inference_num_gpus: 1 + inference_num_gpus_per_engine: 1 + inference_num_gpus_per_node: 2 + inference_fetch_batch: 8 + inference_buffer_threshold: 2 + max_sample_pool_size: 12 + inference_batch_size: 2 + sglang: + tp_size: 1 + mem_fraction_static: 0.5 + +mooncake: + master_server_address: null + metadata_server: null + data_plane: store + protocol: tcp + enable_gpu_direct: false + allow_host_fallback: false + global_segment_size: 32GB + local_buffer_size: 8GB + host_buffer_size: 1GB + async_put_pool_size: 1 + enable_hard_pin: true + max_seq_len: 4096 + hidden_dim: 4096 + +logging: + report_to: none + +debug: + enable_perf_metrics: true + +output_dir: outputs/jetspec_qwen3_8b_4k_tcp +cache_dir: cache/jetspec_qwen3_8b_4k_tcp diff --git a/docs/support.md b/docs/support.md index c5f82d0..e762e3f 100644 --- a/docs/support.md +++ b/docs/support.md @@ -16,7 +16,7 @@ and tests do not imply CPU training support. | Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery; SDPA CP1, MBS1, non-packing, REPLICATE | No packing/CP, warmstart, export or consumer acceptance yet; see [scope](user-guide/medusa.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | -| JetSpec core | Qwen3-style causal parallel Draft head and temperature-scaled forward-KL objective, reused DFlash block machinery; chunked full-vocabulary KL and explicit causal visibility | Model/loss prerequisite only; no trainer selection, tree builder, recovery, export or consumer validation; CPU causal packing and direct unmasked Draft calls fail closed; see [scope](user-guide/jetspec.md) | +| JetSpec | Qwen3-style causal parallel Draft head and temperature-scaled forward-KL objective, reused DFlash block machinery; chunked full-vocabulary KL, explicit causal visibility and exact recovery with FlexAttention, CP1, padded/non-packing | No tree builder, native export or consumer acceptance yet; CPU causal packing and direct unmasked Draft calls fail closed; see [scope](user-guide/jetspec.md) | | MTP-conditioned DSpark / DFlash | Target-side frozen native Qwen3.5 features, token/hidden dual-anchor training, exact recovery and strict native assets; legacy Draft-side assets retained | Dense full GQA, CP1, MBS1, REPLICATE, one node, non-packing, decay objective; Target capture PP1/unquantized; no generic serving consumer | | Native MTP | Qwen3.5-9B native single predictor, same-Target frozen embedding/head, multi-step training, recovery and export/stitch | Text-only Dense MTP; SDPA CP1 or independently validated FA4 packing/CP; no generic multi-layer/MoE MTP | | Dense / Qwen MoE / Ling MoE | Structure-specific FFNs, routing state, checkpointing and strict assets | No promise of every algorithm × structure combination | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index 9416f91..b3ac79f 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -115,13 +115,16 @@ anchor-shared window. FA4 preserves convolution, selector, document boundaries and CP ownership. Functional results do not establish official-recipe reproduction or long-term quality. -## JetSpec core prerequisite - -The [JetSpec model/loss API](jetspec.md) combines the retained DFlash block head -with causal intra-block attention and a temperature-scaled forward-KL objective -against frozen Target features. `JetSpecConfig` forces that causal contract and -rejects incompatible structures. Public training, tree construction, recovery, -export and consumer validation remain pending. +## JetSpec + +The [JetSpec workflow](jetspec.md) combines the retained DFlash block head with +causal intra-block attention and a temperature-scaled forward-KL objective against +frozen Target features. `JetSpecConfig` forces that causal contract and rejects +incompatible structures. The ASpec training CLI selects the algorithm through a +shipped Qwen3-8B Draft config and a JetSpec trainer that reuses the DFlash +optimizer/checkpoint path. FlexAttention, CP1, padded/non-packing execution and +exact recovery are the initial scope. Tree construction, export and consumer +acceptance remain pending. ## Native MTP conditioning for DSpark/DFlash diff --git a/docs/user-guide/jetspec.md b/docs/user-guide/jetspec.md index 3df3b1f..0d1446e 100644 --- a/docs/user-guide/jetspec.md +++ b/docs/user-guide/jetspec.md @@ -1,10 +1,84 @@ -# JetSpec model and loss core +# JetSpec Draft training + +This implements [ASpec #15](https://github.com/inclusionAI/ASpec/issues/15) for the +model/loss and initial training path through +`aspec.models.jetspec.JetSpecModel`, +`aspec.models.draft.jetspec.JetSpecConfig`, +`aspec.models.draft.jetspec.JetSpecDraftModel` and +`aspec.training.jetspec_trainer.JetSpecTrainer`. The ASpec training CLI selects this +algorithm through `aspec/config/jetspec_qwen3_8b_draft_config.json`; the +[example workflow](../../configs/examples/jetspec_qwen3_8b_4k_tcp.yaml) runs one +Target GPU plus two Draft GPUs at 4K (three GPUs total). + +## Training selection + +- Draft config: `aspec/config/jetspec_qwen3_8b_draft_config.json` (Qwen3-8B, + `block_size=16`, `is_causal=true`). +- Example: `configs/examples/jetspec_qwen3_8b_4k_tcp.yaml` (FlexAttention, CP1, MBS1, + non-packing, REPLICATE, Store/TCP transport, + `inference.store_last_hidden_states=true`). +- The supported path fails closed outside dense full attention, padded inputs, + `attention_backend=flex_attention`, `draft_context_parallel_size=1` and + `dflash_packing=false`. MoE, sliding-window layers, MTP conditioning, packing, + CP>1 and whole-model compile are rejected during Plan compilation or + configuration. The CPU dense fallback builds the same causal block mask for + contract tests; the training path requires FlexAttention. +- Exact recovery records the architecture, causal block size, temperature, KL chunk + size and uniform/decay weighting. Changing any of them requires a weights-only + new stage. + +## Running + +Prepare the Target in a separate environment and keep it running on GPU 0: + +```bash +aspec patch-sglang +ASPEC_TARGET_MODEL=/path/to/Qwen3-8B CUDA_VISIBLE_DEVICES=0 \ + aspec target --config configs/examples/jetspec_qwen3_8b_4k_tcp.yaml +``` + +Compile the example, then train with two Draft GPUs (GPUs 1 and 2): + +```bash +aspec --config configs/examples/jetspec_qwen3_8b_4k_tcp.yaml --dry-run \ + model.target_model_path=/path/to/Qwen3-8B +CUDA_VISIBLE_DEVICES=1,2 RAY_ADDRESS=local \ + aspec --config configs/examples/jetspec_qwen3_8b_4k_tcp.yaml \ + model.target_model_path=/path/to/Qwen3-8B +``` + +For exact recovery, stop the first run at step 4 while keeping `lr_total_steps=8`, +then resume that checkpoint into a fresh directory: + +```bash +CUDA_VISIBLE_DEVICES=1,2 RAY_ADDRESS=local \ + aspec --config configs/examples/jetspec_qwen3_8b_4k_tcp.yaml \ + model.target_model_path=/path/to/Qwen3-8B \ + training.num_train_steps=4 training.lr_total_steps=8 \ + output_dir=outputs/jetspec_qwen3_8b_4k_tcp_step4 + +CUDA_VISIBLE_DEVICES=1,2 RAY_ADDRESS=local \ + aspec --config configs/examples/jetspec_qwen3_8b_4k_tcp.yaml \ + model.target_model_path=/path/to/Qwen3-8B \ + training.load_path=outputs/jetspec_qwen3_8b_4k_tcp_step4/checkpoints \ + training.num_train_steps=8 training.lr_total_steps=8 \ + training.continual_training=false \ + output_dir=outputs/jetspec_qwen3_8b_4k_tcp_resumed +``` + +The example is a short functional smoke-test recipe, not evidence of convergence, +throughput or decoding acceptance. Public CI is CPU-only; GPU end-to-end training +and exact resume still need a hardware run before making quality claims. + +## Validation status -This is the first prerequisite for [ASpec #15](https://github.com/inclusionAI/ASpec/issues/15). -It provides `aspec.models.jetspec.JetSpecModel`, -`aspec.models.draft.jetspec.JetSpecConfig` and -`aspec.models.draft.jetspec.JetSpecDraftModel`. It is a model/loss API; the ASpec -training CLI does not select this algorithm yet. +Registration, Plan compilation, trainer construction, checkpoint topology and the +CPU numerical contracts are covered by `tests/cpu`. The CPU checks cover the +configuration contract, causal mask construction, bounded forward KL and gradients +against independent equations, explicit-mask and packed fail-closed paths, +anchor/loss-mask exclusion, empty supervision and next-token label/teacher +alignment. GPU attention execution, convergence, acceptance length and serving +performance are not certified by those checks. ## Reference and selected method @@ -118,14 +192,10 @@ The return value follows `DFlashModel.forward`: scalar loss, accuracy, per-position loss/accuracy/count, extra components and the objective numerator / denominator. Per-position loss now reports KL rather than CE. -## Boundaries and validation +## Boundaries -- This core does not provide a recipe, trainer selection, exact resume, tree - construction, export, strict reloading or a serving consumer. Those remain for +- This integration does not provide tree construction, export, strict reloading, + held-out acceptance observations or a serving consumer. Those remain for follow-up work under the issue's expected outcomes. -- CPU checks cover the configuration contract, causal mask construction, bounded - forward KL and gradients against independent equations, explicit-mask and packed - fail-closed paths, anchor/loss-mask exclusion, empty supervision and next-token - label/teacher alignment on unpacked inputs. They do not certify GPU attention - execution, convergence, acceptance length or serving performance. -- The default tests are in `tests/cpu/test_jetspec.py`. +- The default tests are in `tests/cpu/test_jetspec.py` and + `tests/cpu/test_jetspec_integration.py`. diff --git a/tests/cpu/test_jetspec_integration.py b/tests/cpu/test_jetspec_integration.py new file mode 100644 index 0000000..0d8088a --- /dev/null +++ b/tests/cpu/test_jetspec_integration.py @@ -0,0 +1,282 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""JetSpec integration contracts: registration, Plan, dispatch and resume.""" + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from aspec.config.draft_algorithm import draft_model_semantics, resolve_draft_algorithm +from aspec.config.train_config import load_config +from aspec.models.draft.auto import AutoDraftModelConfig, AutoEagle3DraftModel +from aspec.models.draft.jetspec import JetSpecConfig, JetSpecDraftModel +from aspec.models.jetspec import JetSpecModel +from aspec.planning import ( + PlanCompileError, + _load_draft_descriptor, + _runtime_values, + compile_execution_plan, +) +from aspec.training.checkpoint import ( + _logical_training_topology, + _validate_exact_resume_topology, +) +from aspec.training.dflash_trainer import DFlashTrainer +from aspec.training.jetspec_trainer import JetSpecTrainer +from aspec.training.trainer_actor import _trainer_class_for_config +from tools.convert_to_hf import _serving_config + +_REPO_ROOT = Path(__file__).parents[2] +_EXAMPLE = _REPO_ROOT / "configs" / "examples" / "jetspec_qwen3_8b_4k_tcp.yaml" +_DRAFT_JSON = _REPO_ROOT / "aspec" / "config" / "jetspec_qwen3_8b_draft_config.json" + +_ILLEGAL_OVERRIDES = [ + ("training.dflash_packing=true", PlanCompileError), + ("training.draft_context_parallel_size=2", NotImplementedError), + ("training.attention_backend=sdpa", PlanCompileError), + ("training.attention_backend=fa4", PlanCompileError), + ("training.attention_backend=cudnn_flex_attention", PlanCompileError), + ("training.compile_model=true", PlanCompileError), + ("training.dflash_loss_objective=dpace", PlanCompileError), + ("training.dflash_ce_loss_alpha=0.5", PlanCompileError), + ("training.dflash_l1_loss_alpha=0.1", PlanCompileError), + ("training.dflash_block_size=8", PlanCompileError), + ("training.jetspec_distillation_temperature=0.0", PlanCompileError), + ("training.jetspec_distillation_temperature=2.0e19", PlanCompileError), + ("training.jetspec_kl_chunk_rows=0", PlanCompileError), + ("training.jetspec_loss_decay_gamma=-1.0", PlanCompileError), + ("inference.store_last_hidden_states=false", PlanCompileError), +] + + +def _tiny_config(**overrides): + raw = dict( + architectures=["JetSpecDraftModel"], model_type="jetspec", + hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, head_dim=8, vocab_size=32, + target_hidden_size=16, target_num_hidden_layers=4, target_layer_ids=[1], + num_target_layers=1, mask_token_id=31, max_position_embeddings=128, + block_size=5, is_causal=True, + ) + raw.update(overrides) + return AutoDraftModelConfig.from_dict(raw) + + +def _fake_qwen_target(tmp_path): + target = tmp_path / "Qwen3-8B-test" + target.mkdir() + (target / "config.json").write_text(json.dumps({ + "model_type": "qwen3", + "architectures": ["Qwen3ForCausalLM"], + "hidden_size": 4096, + "vocab_size": 151936, + "num_hidden_layers": 36, + "max_position_embeddings": 40960, + }), encoding="utf-8") + return target + + +def _config(tmp_path, *overrides): + args = [f"model.target_model_path={_fake_qwen_target(tmp_path)}", *overrides] + return load_config(config_path=str(_EXAMPLE), cli_args=args), args + + +def test_registration_resolves_jetspec(): + assert resolve_draft_algorithm({"architectures": ["JetSpecDraftModel"]}) == "jetspec" + shipped = AutoDraftModelConfig.from_file(str(_DRAFT_JSON)) + assert isinstance(shipped, JetSpecConfig) + assert shipped.block_size == 16 + assert shipped.is_causal is True + assert shipped.target_layer_ids == [1, 9, 17, 25, 33] + + tiny = _tiny_config() + model = AutoEagle3DraftModel.from_config(tiny) + assert isinstance(model, JetSpecDraftModel) + assert _trainer_class_for_config(tiny) is JetSpecTrainer + semantics = draft_model_semantics(tiny) + assert semantics["architectures"] == ["JetSpecDraftModel"] + assert semantics["is_causal"] is True + assert semantics["block_size"] == 5 + + +def test_sglang_serving_export_rejects_jetspec(): + raw = json.loads(_DRAFT_JSON.read_text()) + with pytest.raises(NotImplementedError, match="--format aspec"): + _serving_config(raw) + + +def test_plan_compiles_with_jetspec_objective(tmp_path): + config, args = _config(tmp_path) + plan = compile_execution_plan(config, config_path=str(_EXAMPLE), cli_args=args).to_dict() + draft = plan["workload"]["models"]["draft"] + assert draft["family"] == "jetspec" + objective = plan["workload"]["objective"] + assert objective["loss"] == "temperature_scaled_forward_kl" + assert objective["block_size"] == 16 + assert objective["prediction_start"] == 1 + assert objective["teacher_feature"] == "last_hidden_states" + assert objective["distillation_temperature"] == 1.0 + assert objective["kl_chunk_rows"] == 256 + assert objective["loss_decay_gamma"] == 0.0 + assert objective["ce_loss_alpha"] == 0.0 + assert objective["l1_loss_alpha"] == 0.0 + assert objective["frozen_parts"] == ["target_lm_head"] + features = plan["workload"]["features"] + assert features["aux_hidden_state_layers"] == [1, 9, 17, 25, 33] + assert "last_hidden_states" in features["tensors"] + attention = next( + item for item in plan["capabilities"] if item["name"] == "draft_attention_execution" + ) + assert attention["requirement"] == "required" + + runtime = _runtime_values(config, _load_draft_descriptor(config)) + assert runtime["draft_algorithm"] == "jetspec" + assert runtime["jetspec_distillation_temperature"] == 1.0 + assert runtime["jetspec_kl_chunk_rows"] == 256 + assert runtime["jetspec_loss_decay_gamma"] == 0.0 + + +@pytest.mark.parametrize("temperature, accepted", [(1.0, True), (1.8446744e19, False)]) +def test_plan_matches_trainer_temperature_limit(tmp_path, temperature, accepted): + override = f"training.jetspec_distillation_temperature={temperature}" + if accepted: + config, args = _config(tmp_path, override) + plan = compile_execution_plan( + config, config_path=str(_EXAMPLE), cli_args=args + ).to_dict() + assert plan["workload"]["objective"]["distillation_temperature"] == temperature + else: + with pytest.raises(PlanCompileError, match="jetspec_distillation_temperature"): + config, args = _config(tmp_path, override) + compile_execution_plan( + config, config_path=str(_EXAMPLE), cli_args=args + ) + + +@pytest.mark.parametrize("override, error", _ILLEGAL_OVERRIDES) +def test_plan_rejects_unsupported_jetspec_combinations(tmp_path, override, error): + target = f"model.target_model_path={_fake_qwen_target(tmp_path)}" + args = [target, override] + with pytest.raises(error): + config = load_config(config_path=str(_EXAMPLE), cli_args=args) + compile_execution_plan(config, config_path=str(_EXAMPLE), cli_args=args) + + +def _trainer_args(): + return SimpleNamespace( + draft_algorithm="jetspec", draft_model_config_obj=_tiny_config(), world_size=2, + dflash_block_size=5, dflash_num_anchors=2, dflash_num_target_layers=1, + dflash_loss_objective="decay", dflash_dpace_alpha=0.5, + dflash_loss_decay_gamma=7.0, dflash_ce_loss_alpha=0.0, dflash_l1_loss_alpha=0.0, + jetspec_distillation_temperature=1.0, jetspec_kl_chunk_rows=64, + jetspec_loss_decay_gamma=0.0, anchor_sampling_scope="row", + deterministic_anchor_sampling=False, max_seq_length=128, micro_batch_size=1, + draft_accumulation_steps=1, max_grad_norm=1.0, seed=42, + draft_context_parallel_size=1, training_num_nodes=1, training_num_gpus_per_node=2, + attention_backend="flex_attention", fsdp_strategy="REPLICATE", + sp_ulysses_size=1, sp_ring_size=1, + ) + + +def test_trainer_constructor_propagates_jetspec_objective( + monkeypatch, +): + def base_init(self, args): + self.args = args + self.block_size = args.dflash_block_size + self.num_anchors = args.dflash_num_anchors + self.num_target_layers = args.dflash_num_target_layers + self.loss_objective = args.dflash_loss_objective + self.dpace_alpha = args.dflash_dpace_alpha + self.loss_decay_gamma = args.dflash_loss_decay_gamma + self.ce_loss_alpha = args.dflash_ce_loss_alpha + self.l1_loss_alpha = args.dflash_l1_loss_alpha + self._anchor_slot_offset = 1 + + monkeypatch.setattr(DFlashTrainer, "__init__", base_init) + args = _trainer_args() + args.jetspec_distillation_temperature = 0.5 + args.jetspec_kl_chunk_rows = 32 + args.jetspec_loss_decay_gamma = 0.25 + trainer = JetSpecTrainer(args) + + assert trainer.loss_decay_gamma == 0.25 + assert trainer.loss_decay_gamma != args.dflash_loss_decay_gamma + wrapper = trainer._build_training_wrapper(JetSpecDraftModel(_tiny_config())) + assert isinstance(wrapper, JetSpecModel) + assert wrapper.distillation_temperature == 0.5 + assert wrapper.kl_chunk_rows == 32 + assert wrapper.loss_decay_gamma == 0.25 + assert wrapper.prediction_start == 1 + assert trainer._anchor_slot_offset == 1 + + objective = _logical_training_topology(args, 2)["training_objective"] + assert objective["distillation_temperature"] == wrapper.distillation_temperature + assert objective["kl_chunk_rows"] == wrapper.kl_chunk_rows + assert objective["loss_decay_gamma"] == wrapper.loss_decay_gamma + + trainer.block_size = 4 + with pytest.raises(ValueError, match="block_size"): + trainer._build_draft_model(_tiny_config()) + trainer.block_size = 5 + trainer.num_target_layers = 2 + with pytest.raises(ValueError, match="num_target_layers"): + trainer._build_draft_model(_tiny_config()) + + +@pytest.mark.parametrize( + "missing", + [ + "empty", "distillation_temperature", "kl_chunk_rows", "loss_decay_gamma", + "is_causal", + ], +) +def test_exact_resume_rejects_incomplete_jetspec_topology(missing): + args = _trainer_args() + topology = json.loads(json.dumps(_logical_training_topology(args, 2))) + if missing == "empty": + topology = {} + elif missing in { + "distillation_temperature", "kl_chunk_rows", "loss_decay_gamma", + }: + del topology["training_objective"][missing] + else: + del topology["draft_model_semantics"][missing] + with pytest.raises(RuntimeError, match="complete JetSpec training_topology"): + _validate_exact_resume_topology( + args, {"world_size": 2, "training_topology": topology} + ) + + +def test_exact_resume_requires_topology_for_jetspec(): + args = _trainer_args() + with pytest.raises(RuntimeError, match="training_topology for JetSpec"): + _validate_exact_resume_topology(args, {"world_size": 2}) + + +def test_checkpoint_topology_pins_jetspec_objective(): + args = _trainer_args() + current = _logical_training_topology(args, 2) + objective = current["training_objective"] + assert objective["loss"] == "temperature_scaled_forward_kl" + assert objective["distillation_temperature"] == 1.0 + assert objective["kl_chunk_rows"] == 64 + assert objective["block_size"] == 5 + assert objective["loss_decay_gamma"] == 0.0 + assert current["draft_model_semantics"]["is_causal"] is True + assert current["draft_model_semantics"]["block_size"] == 5 + + metadata = {"world_size": 2, "training_topology": json.loads(json.dumps(current))} + _validate_exact_resume_topology(args, metadata) + for key, value in ( + ("distillation_temperature", 0.5), + ("kl_chunk_rows", 32), + ("loss_decay_gamma", 1.0), + ): + changed = json.loads(json.dumps(metadata)) + changed["training_topology"]["training_objective"][key] = value + with pytest.raises(RuntimeError): + _validate_exact_resume_topology(args, changed) diff --git a/tests/test_evidence.py b/tests/test_evidence.py index 997f9ab..04337b0 100644 --- a/tests/test_evidence.py +++ b/tests/test_evidence.py @@ -853,6 +853,101 @@ def test_cudnn_flex_attention_execution_activates_from_runtime_evidence(tmp_path ] +def test_flex_attention_execution_activates_required_jetspec_capability(tmp_path): + target = tmp_path / "Qwen3-test" + target.mkdir() + (target / "config.json").write_text( + json.dumps( + { + "model_type": "qwen3", + "architectures": ["Qwen3ForCausalLM"], + "hidden_size": 4096, + "vocab_size": 151936, + "num_hidden_layers": 36, + "max_position_embeddings": 40960, + } + ), + encoding="utf-8", + ) + train_data = tmp_path / "train.jsonl" + train_data.write_text("{}\n", encoding="utf-8") + example = _REPO_ROOT / "configs" / "examples" / "jetspec_qwen3_8b_4k_tcp.yaml" + overrides = [ + f"model.target_model_path={target}", + f"dataset.train_data_path={train_data}", + "dataset.eval_data_path=null", + ] + config = load_config(config_path=str(example), cli_args=overrides) + plan = compile_execution_plan( + config, config_path=str(example), cli_args=overrides + ) + assert plan.workload.models["draft"]["family"] == "jetspec" + recorder = _recorder_for_plan(tmp_path, plan) + policy = plan.workload.models["draft"]["attention"] + + recorder.record_training( + { + "optimizer_steps_observed": 1, + "checkpoint": {"enabled": False, "exists": False}, + "feature_contract": _feature_observation(plan), + "execution": { + "attention_backend": "flex_attention", + "context_parallel_size": 1, + }, + } + ) + + capability = { + item["name"]: item for item in recorder.data["capabilities"] + }["draft_attention_execution"] + assert capability["requirement"] == "required" + assert capability["effective_state"] == "active" + assert capability["evidence"] == [ + { + "predicate": "optimizer_steps_consumed_planned_attention_backend_at_planned_cp", + "optimizer_steps": 1, + "head_layout": policy["head_layout"], + "attention_backend": "flex_attention", + "context_parallel_size": 1, + } + ] + + +def test_omitted_plan_capability_is_not_activated_by_runtime_evidence(tmp_path): + overrides = [ + "training.attention_backend=flex_attention", + "training.draft_context_parallel_size=1", + ] + config = load_config(config_path=str(_REFERENCE_CONFIG), cli_args=overrides) + plan = compile_execution_plan( + config, config_path=str(_REFERENCE_CONFIG), cli_args=overrides + ) + planned = next( + item for item in plan.capabilities if item.name == "draft_attention_execution" + ) + assert planned.effective_state == "omitted" + recorder = _recorder_for_plan(tmp_path, plan) + + recorder.record_training( + { + "optimizer_steps_observed": 1, + "checkpoint": {"enabled": False, "exists": False}, + "feature_contract": _feature_observation(plan), + "execution": { + "attention_backend": "flex_attention", + "context_parallel_size": 1, + }, + } + ) + + capability = { + item["name"]: item for item in recorder.data["capabilities"] + }["draft_attention_execution"] + assert capability["effective_state"] == "omitted" + assert capability["effective"] is None + assert capability["evidence"] == [] + + @pytest.mark.parametrize("observation", ["missing", "wrong_head_layout"]) def test_full_attention_execution_requires_actual_head_layout(tmp_path, observation): overrides = ["training.attention_backend=cudnn_flex_attention"]