From 3e75720b2276c904577f44bc03e1e0ea057dd862 Mon Sep 17 00:00:00 2001 From: zhouaihui Date: Tue, 17 Nov 2026 14:11:55 +0800 Subject: [PATCH 1/7] feat(dspark): add selectable losses and GLM Flash training Keep DeepSpec as the default and add the pinned Speculators Red Hat objective, including independent slot masks, row reductions, confidence target rounding and exact-resume identity. Add CuTeDSL CE/TV and Markov projection kernels for SM90/SM100. Preserve BF16 projection/add rounding and independent embedding gradient reductions; reuse shared teacher positions on Hopper. Add the official FP8 TP4 Target recipe, GLM text rendering, and verified published-Draft initialization with frozen Target bindings and native export. Retain upstream attribution and the QuACK license. --- aspec/config/draft_algorithm.py | 11 + aspec/config/draft_attention.py | 6 +- .../dspark_glm5_3_flash_draft_config.json | 36 ++ aspec/config/train_config.py | 1 + aspec/config/utils.py | 15 +- aspec/data/renderers/__init__.py | 3 + aspec/data/renderers/glm5.py | 67 +++ aspec/inference/engine/sgl_engine.py | 4 +- aspec/kernels/cutedsl/speculators_loss.py | 482 ++++++++++++++++++ aspec/models/dflash.py | 2 + aspec/models/dspark.py | 199 ++++++-- aspec/models/ops/loss.py | 40 ++ aspec/models/target/target_utils.py | 7 +- aspec/planning.py | 24 +- aspec/training/checkpoint.py | 43 +- aspec/training/dflash_trainer.py | 12 +- aspec/training/dspark_trainer.py | 5 +- configs/recipes/glm5_3_flash_dspark.yaml | 103 ++++ docs/framework/REFERENCES.md | 3 + docs/support.md | 1 + docs/user-guide/algorithms.md | 7 +- docs/user-guide/recipes.md | 57 +++ docs/user-guide/training.md | 39 +- licenses/QuACK-LICENSE | 201 ++++++++ tests/cpu/test_dspark_loss_recipes.py | 188 +++++++ tests/cpu/test_glm5_flash_draft.py | 125 +++++ tests/cpu/test_release_recipes.py | 4 +- tests/cpu/test_target_lm_head.py | 4 +- tests/test_dspark_speculators_cutedsl.py | 218 ++++++++ tools/recipe_catalog.json | 11 + 30 files changed, 1843 insertions(+), 75 deletions(-) create mode 100644 aspec/config/dspark_glm5_3_flash_draft_config.json create mode 100644 aspec/data/renderers/glm5.py create mode 100644 aspec/kernels/cutedsl/speculators_loss.py create mode 100644 configs/recipes/glm5_3_flash_dspark.yaml create mode 100644 licenses/QuACK-LICENSE create mode 100644 tests/cpu/test_dspark_loss_recipes.py create mode 100644 tests/cpu/test_glm5_flash_draft.py create mode 100644 tests/test_dspark_speculators_cutedsl.py diff --git a/aspec/config/draft_algorithm.py b/aspec/config/draft_algorithm.py index 1605840..70ca2e4 100644 --- a/aspec/config/draft_algorithm.py +++ b/aspec/config/draft_algorithm.py @@ -15,6 +15,17 @@ """Algorithm identity for the currently registered Draft architectures.""" from collections.abc import Mapping +from enum import StrEnum + + +class DSparkLossRecipe(StrEnum): + DEEPSPEC = "deepspec" + SPECULATORS_REDHAT_V1 = "speculators_redhat_v1" + + @property + def distribution_metric(self) -> str: + return "tv_loss" if self is self.SPECULATORS_REDHAT_V1 else "l1_loss" + DRAFT_ALGORITHMS = { "NativeMTPDraftModel": "mtp", diff --git a/aspec/config/draft_attention.py b/aspec/config/draft_attention.py index 52f3b04..ac82ffe 100644 --- a/aspec/config/draft_attention.py +++ b/aspec/config/draft_attention.py @@ -143,6 +143,9 @@ def resolve_draft_attention(raw: Mapping) -> dict: ): raise ValueError("Draft sliding_window must be a positive integer or null") schedule = raw.get("layer_types") + sliding_non_causal = raw.get("sliding_window_non_causal", True) + if not isinstance(sliding_non_causal, bool): + raise ValueError("sliding_window_non_causal must be boolean") ratios = raw.get("compress_ratios") if ratios is not None: if not isinstance(ratios, (list, tuple)) or len(ratios) != layers: @@ -421,7 +424,8 @@ def dsa_positive_integer(name: str, default: int) -> int: else kind == "sliding_attention") else "bidirectional" for kind in schedule ] if raw.get("architectures") == ["DFlash2DraftModel"] - else "bidirectional" + else ["causal" if kind == "sliding_attention" else "bidirectional" for kind in schedule] + if not sliding_non_causal else "bidirectional" ), } diff --git a/aspec/config/dspark_glm5_3_flash_draft_config.json b/aspec/config/dspark_glm5_3_flash_draft_config.json new file mode 100644 index 0000000..8bccc42 --- /dev/null +++ b/aspec/config/dspark_glm5_3_flash_draft_config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "DSparkDraftModel" + ], + "model_type": "qwen3_dspark", + "hidden_size": 4096, + "intermediate_size": 12288, + "head_dim": 64, + "num_hidden_layers": 5, + "num_attention_heads": 64, + "num_key_value_heads": 64, + "vocab_size": 154880, + "rms_norm_eps": 1e-05, + "max_position_embeddings": 1048576, + "rope_theta": 10000, + "num_target_layers": 6, + "target_hidden_size": 4096, + "target_num_hidden_layers": 45, + "target_layer_ids": [ + 19, + 27, + 31, + 35, + 39, + 43 + ], + "mask_token_id": 154856, + "block_size": 8, + "markov_rank": 256, + "markov_head_type": "vanilla", + "enable_confidence_head": true, + "confidence_head_with_markov": true, + "tie_word_embeddings": false, + "sliding_window": 2048, + "sliding_window_non_causal": false +} diff --git a/aspec/config/train_config.py b/aspec/config/train_config.py index 3fbb132..60e52bd 100644 --- a/aspec/config/train_config.py +++ b/aspec/config/train_config.py @@ -251,6 +251,7 @@ class TrainingConfig: # DSpark-specific parameters (used by DSpark trainer only) # Precision of the frozen Draft LM head during training (SM90 FP8, CP1). dspark_lm_head_precision: str = "bf16" + dspark_loss_recipe: str = "deepspec" dspark_num_anchors: int = 512 dspark_num_target_layers: int = 5 dspark_loss_decay_gamma: float = 4.0 diff --git a/aspec/config/utils.py b/aspec/config/utils.py index e5a596b..2e27186 100644 --- a/aspec/config/utils.py +++ b/aspec/config/utils.py @@ -25,11 +25,22 @@ from typing import Optional import torch -from transformers import AutoConfig, AutoTokenizer +from transformers import AutoConfig, AutoTokenizer, PretrainedConfig logger = logging.getLogger(__name__) +def load_target_config(model_path, **kwargs): + """Read GLM text geometry even in Draft environments without SGLang's registry.""" + try: + return AutoConfig.from_pretrained(model_path, **kwargs) + except ValueError: + raw, _ = PretrainedConfig.get_config_dict(model_path, **kwargs) + if raw.get("model_type") != "glm5_next": + raise + return PretrainedConfig.from_dict(raw["text_config"]) + + def _copy_config_value(value): if hasattr(value, "to_dict"): return value.to_dict() @@ -114,7 +125,7 @@ def generate_draft_model_config( Returns: dict: Generated draft model config dictionary """ - target_config = AutoConfig.from_pretrained( + target_config = load_target_config( target_model_path, cache_dir=cache_dir, trust_remote_code=trust_remote_code, diff --git a/aspec/data/renderers/__init__.py b/aspec/data/renderers/__init__.py index f0542df..87f3e1a 100644 --- a/aspec/data/renderers/__init__.py +++ b/aspec/data/renderers/__init__.py @@ -19,14 +19,17 @@ # SOFTWARE. from aspec.data.renderers.base import DEFAULT_CACHE_VERSION, ConversationRenderer +from aspec.data.renderers.glm5 import Glm5FlashRenderer from aspec.data.renderers.kimi_k3 import K3Renderer from aspec.data.renderers.registry import RENDERER_REGISTRY, RendererRegistry +RENDERER_REGISTRY.register("glm5_3_flash", Glm5FlashRenderer) RENDERER_REGISTRY.register("kimi-k3", K3Renderer) __all__ = [ "DEFAULT_CACHE_VERSION", "ConversationRenderer", + "Glm5FlashRenderer", "K3Renderer", "RENDERER_REGISTRY", "RendererRegistry", diff --git a/aspec/data/renderers/glm5.py b/aspec/data/renderers/glm5.py new file mode 100644 index 0000000..b6718c1 --- /dev/null +++ b/aspec/data/renderers/glm5.py @@ -0,0 +1,67 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Text-only conversations using the GLM-5.3-Flash checkpoint template.""" + + +class Glm5FlashRenderer: + CACHE_VERSION = "v1" + roles = ("system", "user", "assistant") + + def __init__(self, tokenizer): + self.tokenizer = tokenizer + self.role_ids = {} + for role in self.roles: + marker = f"<|{role}|>" + ids = tokenizer.encode(marker, add_special_tokens=False) + if len(ids) != 1 or tokenizer.convert_ids_to_tokens(ids[0]) != marker: + raise ValueError(f"GLM tokenizer lacks its {marker} role token") + self.role_ids[ids[0]] = role + + def get_assistant_token_ids(self): + ids = {role: token for token, role in self.role_ids.items()} + return [ids["assistant"]], [ids["user"]], 0 + + def render( + self, messages, tools=None, *, max_seq_length, last_turn_only=False, generation_config=None + ): + if tools: + raise ValueError("GLM text training requires conversations without tools") + for msg in messages: + if ( + msg.get("role") not in self.roles + or not isinstance(msg.get("content"), str) + or msg.get("tool_calls") + ): + raise ValueError("Expected GLM system/user/assistant text messages") + for field in ("content", "reasoning_content"): + value = msg.get(field) + if value is None: + continue + if not isinstance(value, str): + raise ValueError(f"GLM {field} must be text") + if any(f"<|{role}|>" in value for role in self.roles): + raise ValueError("A GLM message contains a reserved role token") + + kwargs = { + k: v + for k, v in (generation_config or {}).items() + if k in ("reasoning_effort", "clear_thinking") + } + ids = self.tokenizer.apply_chat_template( + messages, tokenize=True, add_generation_prompt=False, return_dict=False, **kwargs + ) + mask = [0] * len(ids) + last_assistant = 0 + supervise = False + for i, token in enumerate(ids): + role = self.role_ids.get(token) + if role is not None: + supervise = role == "assistant" + if supervise: + last_assistant = i + else: + mask[i] = int(supervise) + if last_turn_only: + mask[:last_assistant] = [0] * last_assistant + return ids[:max_seq_length], mask[:max_seq_length] diff --git a/aspec/inference/engine/sgl_engine.py b/aspec/inference/engine/sgl_engine.py index e564d87..b2d711b 100644 --- a/aspec/inference/engine/sgl_engine.py +++ b/aspec/inference/engine/sgl_engine.py @@ -984,9 +984,9 @@ def _get_hidden_size_from_engine(self) -> int: Returns: Hidden size dimension. """ - from transformers import AutoConfig + from aspec.config.utils import load_target_config - config = AutoConfig.from_pretrained( + config = load_target_config( self.args.target_model_path, trust_remote_code=getattr(self.args, "trust_remote_code", False), ) diff --git a/aspec/kernels/cutedsl/speculators_loss.py b/aspec/kernels/cutedsl/speculators_loss.py new file mode 100644 index 0000000..50a7ed6 --- /dev/null +++ b/aspec/kernels/cutedsl/speculators_loss.py @@ -0,0 +1,482 @@ +# Copyright 2026 Ant Group Co., Ltd. +# Copyright (c) 2025, Wentao Guo, Ted Zadouri, Tri Dao (QuACK tile layout). +# SPDX-License-Identifier: Apache-2.0 +"""Hopper/Blackwell joint hard CE and overlap TV. Reduction layout adapted from QuACK 0.6.5 +softmax; objective and BF16 gradient addition reimplement Speculators b9c51bc. +""" + +import math +from functools import lru_cache + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import torch +from cutlass import BFloat16, Float32, Int32, Int64, const_expr +from quack import copy_utils +from quack.compile_utils import make_fake_tensor +from quack.epilogue.frontend import F2, gemm_epilogue +from quack.reduce import row_reduce +from quack.reduction_base import ReductionBase + + +@cute.jit +def _lane_inputs(d, t, indices, tile, row, part, copy, vocab: cutlass.Constexpr): + teacher_row = row + if const_expr(indices is not None): + teacher_row = Int32(indices[row]) + ids = cute.make_identity_tensor(d.shape) + dx = copy.partition_S(cute.local_tile(d, tile, (row, part))) + tx = copy.partition_S(cute.local_tile(t, tile, (teacher_row, part))) + ix = copy.partition_S(cute.local_tile(ids, tile, (row, part))) + rd, rt = [cute.make_rmem_tensor_like(a) for a in (dx, tx)] + rd.fill(-BFloat16.inf) + rt.fill(-BFloat16.inf) + pred = copy_utils.predicate_k(ix, limit=vocab) + copy_utils.copy(dx, rd, pred=pred) + copy_utils.copy(tx, rt, pred=pred) + columns = cute.make_rmem_tensor(rd.shape, Int32) + for i in cutlass.range_constexpr(cute.size(columns)): + columns[i] = ix[i][1] + return rd.load().to(Float32), rt.load().to(Float32), columns.load(), pred + + +class _Grad: + def __init__(self, vocab): + self.vocab = vocab + + @cute.jit + def __call__( + self, + d: cute.Tensor, + t: cute.Tensor, + indices, + stats: cute.Tensor, + labels: cute.Tensor, + gc: cute.Tensor, + gt: cute.Tensor, + out: cute.Tensor, + stream: cuda.CUstream, + ): + vec = math.gcd(self.vocab, 8) + copy = copy_utils.tiled_copy_2d(BFloat16, 128, 128, vec) + tile = (1, 128 * vec) + self.kernel(d, t, indices, stats, labels, gc, gt, out, copy, tile).launch( + grid=(d.shape[0], cute.ceil_div(self.vocab, tile[1]), 1), + block=(128, 1, 1), + stream=stream, + ) + + @cute.kernel + def kernel( + self, + d: cute.Tensor, + t: cute.Tensor, + indices, + stats: cute.Tensor, + labels: cute.Tensor, + gc: cute.Tensor, + gt: cute.Tensor, + out: cute.Tensor, + copy: cute.TiledCopy, + tile: cute.Shape, + ): + row, part, _ = cute.arch.block_idx() + thr = copy.get_slice(cute.arch.thread_idx()[0]) + x, y, col, pred = _lane_inputs(d, t, indices, tile, row, part, thr, self.vocab) + q = cute.math.exp(x - stats[row, 0], fastmath=True) + p = cute.math.exp(y - stats[row, 1], fastmath=True) + dc = gc[row] * (q - (col == Int32(labels[row])).to(Float32)) + dt = gt[row] * (q * (stats[row, 2] - (q <= p).to(Float32))) + # Upstream writes two BF16 gradients before autograd adds them. + dest = thr.partition_D(cute.local_tile(out, tile, (row, part))) + reg = cute.make_rmem_tensor_like(dest) + reg.store((dc.to(BFloat16).to(Float32) + dt.to(BFloat16).to(Float32)).to(BFloat16)) + copy_utils.copy(reg, dest, pred=pred) + + +_TILE = 2048 + + +class _Overlap(ReductionBase): + """Stream independent vocabulary tiles, then combine FP32 row partials.""" + + def __init__(self, vocab, merge): + super().__init__(BFloat16, vocab, stage=4) + self.merge_partials = merge + self.cluster_n = 1 + self.width = _TILE + self.tiles = (vocab + self.width - 1) // self.width + + @cute.jit + def __call__( + self, + d: cute.Tensor, + t: cute.Tensor, + idx, + parts: cute.Tensor, + norm: cute.Tensor, + stats: cute.Tensor, + labels: cute.Tensor, + ce: cute.Tensor, + tv: cute.Tensor, + correct: cute.Tensor, + stream: cuda.CUstream, + ): + if const_expr(not self.merge_partials): + copy = copy_utils.tiled_copy_2d(BFloat16, 128, 128, math.gcd(self.N, 8)) + self.scan(d, t, idx, parts, norm, copy).launch( + grid=(d.shape[0] * self.tiles, 1, 1), block=(128, 1, 1), stream=stream + ) + else: + self.merge(d, parts, norm, stats, labels, ce, tv, correct).launch( + grid=(d.shape[0], 1, 1), block=(128, 1, 1), stream=stream + ) + + @cute.kernel + def scan( + self, + d: cute.Tensor, + t: cute.Tensor, + idx, + parts: cute.Tensor, + norm: cute.Tensor, + copy: cute.TiledCopy, + ): + tid = cute.arch.thread_idx()[0] + bid = cute.arch.block_idx()[0] + row, part = bid // self.tiles, bid % self.tiles + x, y, c, _ = _lane_inputs( + d, t, idx, (1, self.width), row, part, copy.get_slice(tid), self.N + ) + smem = cutlass.utils.SmemAllocator() + buf, _ = self._allocate_reduction_buffer_and_mbar(smem, copy.layout_tv_tiled) + add, minimum = cute.ReductionOp.ADD, cute.ReductionOp.MIN + q = cute.math.exp(x - norm[row, 0], fastmath=True) + p = cute.math.exp(y - norm[row, 1], fastmath=True) + take = q <= p + ov, mass = row_reduce( + (cute.where(take, q, p), cute.where(take, q, 0.0)), + add, + 128, + (buf[None, None, 0], buf[None, None, 1]), + ) + i, j = row_reduce( + ( + cute.where(x == norm[row, 2], c.to(Float32), Float32(self.N)), + cute.where(y == norm[row, 3], c.to(Float32), Float32(self.N)), + ), + minimum, + 128, + (buf[None, None, 2], buf[None, None, 3]), + init_val=Float32(self.N), + ) + if tid == 0: + parts[row, 0, part] = ov + parts[row, 1, part] = mass + parts[row, 2, part] = i + parts[row, 3, part] = j + + @cute.kernel + def merge( + self, + d: cute.Tensor, + parts: cute.Tensor, + norm: cute.Tensor, + stats: cute.Tensor, + labels: cute.Tensor, + ce: cute.Tensor, + tv: cute.Tensor, + correct: cute.Tensor, + ): + row = cute.arch.block_idx()[0] + tid = cute.arch.thread_idx()[0] + smem = cutlass.utils.SmemAllocator() + layout = cute.make_layout((1, (4, 1), 4)) + buf = smem.allocate_tensor(Float32, layout) + a, b, c, e = Float32(0), Float32(0), Float32(0), Float32(0) + c, e = Float32(self.N), Float32(self.N) + if tid < self.tiles: + a, b, c, e = ( + parts[row, 0, tid], + parts[row, 1, tid], + parts[row, 2, tid], + parts[row, 3, tid], + ) + ov, mass = row_reduce( + (a, b), cute.ReductionOp.ADD, 128, (buf[None, None, 0], buf[None, None, 1]) + ) + i, j = row_reduce( + (c, e), + cute.ReductionOp.MIN, + 128, + (buf[None, None, 2], buf[None, None, 3]), + init_val=Float32(self.N), + ) + if tid == 0: + label = min(Int32(j), self.N - 1) + labels[row] = Int64(label) + correct[row] = i == j + ce[row] = norm[row, 0] - Float32(d[row, label]) + tv[row] = 1.0 - ov + stats[row, 0] = norm[row, 0] + stats[row, 1] = norm[row, 1] + stats[row, 2] = mass + + +@lru_cache(maxsize=32) +def _overlap_compile(vocab, indexed): + rows = cute.sym_int() + x = make_fake_tensor(BFloat16, (rows, vocab), math.gcd(vocab, 8)) + y = make_fake_tensor(BFloat16, (cute.sym_int(), vocab), math.gcd(vocab, 8)) if indexed else x + ids = make_fake_tensor(Int64, (rows,)) + p = make_fake_tensor(Float32, (rows, 4, (vocab + _TILE - 1) // _TILE)) + n = make_fake_tensor(Float32, (rows, 4)) + f = make_fake_tensor(Float32, (rows,)) + s = make_fake_tensor(Float32, (rows, 3)) + correct = make_fake_tensor(cutlass.Boolean, (rows,)) + return [ + cute.compile( + _Overlap(vocab, merge), + x, + y, + ids if indexed else None, + p, + n, + s, + ids, + f, + f, + correct, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + for merge in (False, True) + ] + + +class _PinnedNorm(ReductionBase): + """Partition the upstream logical CTA across a thread-block cluster. Each logical + lane keeps the same FP32 summation order, including the online tail tile.""" + + def __init__(self, vocab): + super().__init__(BFloat16, vocab, stage=4) + block = min(131072, 1 << (vocab - 1).bit_length()) + warps = 32 if block >= 32768 else 16 if block >= 8192 else 8 if block >= 2048 else 4 + self.cluster_n = max(1, warps // 4) + self.block = block + self.vec = min(math.gcd(8, vocab), max(1, block // (32 * warps))) + self.passes = max(1, block // (128 * self.cluster_n * self.vec)) + self.groups = (vocab + block - 1) // block + + def _threads_per_row(self): + return 128 + + @cute.jit + def __call__( + self, d: cute.Tensor, t: cute.Tensor, idx, norm: cute.Tensor, stream: cuda.CUstream + ): + cp = copy_utils.tiled_copy_2d(BFloat16, 128, 128, self.vec) + cluster = (1, self.cluster_n, 1) if const_expr(self.cluster_n > 1) else None + self.kernel(d, None, norm, cp, 0).launch( + grid=(d.shape[0], self.cluster_n, 1), block=(128, 1, 1), cluster=cluster, stream=stream + ) + self.kernel(t, idx, norm, cp, 1).launch( + grid=(d.shape[0], self.cluster_n, 1), block=(128, 1, 1), cluster=cluster, stream=stream + ) + + @cute.kernel + def kernel( + self, d: cute.Tensor, idx, norm: cute.Tensor, cp: cute.TiledCopy, side: cutlass.Constexpr + ): + tid = cute.arch.thread_idx()[0] + row, part, _ = cute.arch.block_idx() + smem = cutlass.utils.SmemAllocator() + buf, mb = self._allocate_reduction_buffer_and_mbar(smem, cp.layout_tv_tiled) + self._initialize_cluster(tid, mb, 4) + yrow = row + if const_expr(idx is not None): + yrow = Int32(idx[row]) + thr = cp.get_slice(tid) + gmax = -Float32.inf + denom = Float32(0) + for group in cutlass.range_constexpr(self.groups): + tile = (1, 128 * self.vec) + # Omit padded tail columns to preserve the reference FP32 summation order. + passes = min( + self.passes, + (self.N - group * self.block + 128 * self.cluster_n * self.vec - 1) + // (128 * self.cluster_n * self.vec), + ) + cache = cute.make_rmem_tensor((self.vec, passes), Float32) + for k in cutlass.range_constexpr(passes): + shard = (group * self.passes + k) * self.cluster_n + part + dx = thr.partition_S(cute.local_tile(d, tile, (yrow, shard))) + ix = thr.partition_S( + cute.local_tile(cute.make_identity_tensor(d.shape), tile, (row, shard)) + ) + pred = copy_utils.predicate_k(ix, limit=self.N) + reg = cute.make_rmem_tensor_like(dx) + reg.fill(-BFloat16.inf) + copy_utils.copy(dx, reg, pred=pred) + for j in cutlass.range_constexpr(self.vec): + cache[j, k] = Float32(reg[j]) + m = row_reduce( + cache.load(), + cute.ReductionOp.MAX, + 128, + buf[None, None, 2 * group], + mb + 2 * group if const_expr(self.cluster_n > 1) else None, + init_val=-Float32.inf, + hook_fn=cute.arch.cluster_wait + if const_expr(self.cluster_n > 1 and group == 0) + else None, + ) + newmax = cute.arch.fmax(gmax, m) + ex = cute.make_rmem_tensor_like(cache) + ex.store(cute.math.exp(cache.load() - newmax, fastmath=True)) + total = Float32(0) + for k in cutlass.range_constexpr(passes): + for j in cutlass.range_constexpr(self.vec): + total = total + ex[j, k] + z = row_reduce( + total, + cute.ReductionOp.ADD, + 128, + buf[None, None, 2 * group + 1], + mb + 2 * group + 1 if const_expr(self.cluster_n > 1) else None, + ) + if const_expr(group == 0): + denom = z + else: + denom = cute.math.fma(denom, cute.math.exp(gmax - newmax, fastmath=True), z) + gmax = newmax + if (tid == 0) & (part == 0): + norm[row, side] = gmax + cute.math.log(denom) + norm[row, side + 2] = gmax + + +@lru_cache(maxsize=32) +def _norm_compile(vocab, indexed): + rows = cute.sym_int() + x = make_fake_tensor(BFloat16, (rows, vocab), math.gcd(vocab, 8)) + y = make_fake_tensor(BFloat16, (cute.sym_int(), vocab), math.gcd(vocab, 8)) if indexed else x + ids = make_fake_tensor(Int64, (rows,)) if indexed else None + n = make_fake_tensor(Float32, (rows, 4)) + return cute.compile( + _PinnedNorm(vocab), + x, + y, + ids, + n, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + + +@lru_cache(maxsize=32) +def _grad_compile(vocab, indexed): + rows = cute.sym_int() + x = make_fake_tensor(BFloat16, (rows, vocab), math.gcd(vocab, 8)) + y = make_fake_tensor(BFloat16, (cute.sym_int(), vocab), math.gcd(vocab, 8)) if indexed else x + ids = make_fake_tensor(Int64, (rows,)) + f = make_fake_tensor(Float32, (rows,)) + stats = make_fake_tensor(Float32, (rows, 3)) + return cute.compile( + _Grad(vocab), + x, + y, + ids if indexed else None, + stats, + ids, + f, + f, + x, + cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True), + options="--enable-tvm-ffi", + ) + + +class _TokenLoss(torch.autograd.Function): + @staticmethod + def forward(ctx, logits, teacher, indices): + vocab = logits.shape[-1] + x = logits.view(-1, vocab) + y = teacher.view(-1, vocab) + rows = x.shape[0] + ce = x.new_empty(rows, dtype=torch.float32) + tv = torch.empty_like(ce) + ids = x.new_empty(rows, dtype=torch.int64) + stats = x.new_empty((rows, 3), dtype=torch.float32) + correct = x.new_empty(rows, dtype=torch.bool) + partial = stats.new_empty((rows, 4, (vocab + _TILE - 1) // _TILE)) + norm = stats.new_empty((rows, 4)) + _norm_compile(vocab, indices is not None)(x, y, indices, norm) + for kernel in _overlap_compile(vocab, indices is not None): + kernel(x, y, indices, partial, norm, stats, ids, ce, tv, correct) + ctx.save_for_backward(logits, teacher, indices, stats, ids) + ctx.set_materialize_grads(False) + shape = logits.shape[:-1] + labels = ids.view(shape) + correct = correct.view(shape) + ctx.mark_non_differentiable(labels, correct) + return ce.view(shape), tv.view(shape), labels, correct + + @staticmethod + def backward(ctx, grad_ce, grad_tv, _labels, _correct): + logits, teacher, indices, stats, ids = ctx.saved_tensors + x = logits.view(-1, logits.shape[-1]) + dx = torch.empty_like(x) + + def upstream(g): + return stats.new_zeros(x.shape[0]) if g is None else g.reshape(-1).contiguous() + + _grad_compile(x.shape[-1], indices is not None)( + x, + teacher.view(-1, x.shape[-1]), + indices, + stats, + ids, + upstream(grad_ce), + upstream(grad_tv), + dx, + ) + return dx.view_as(logits), None, None + + +def speculators_cutedsl_loss(logits, teacher, indices=None): + return _TokenLoss.apply(logits, teacher, indices) + + +@gemm_epilogue() +def _markov_residual(acc, c): + # Match BF16 linear -> BF16 add. A standard FP32 GEMM residual epilogue + # drops the intermediate rounding and can change the TV subgradient. + if const_expr(isinstance(acc, F2)): + rounded = F2(Float32(BFloat16(acc.lo)), Float32(BFloat16(acc.hi))) + else: + rounded = Float32(BFloat16(acc)) + return {"D": rounded + c} + + +class _MarkovProjection(torch.autograd.Function): + @staticmethod + def forward(ctx, base, embedding, weight): + a = embedding.reshape(-1, embedding.shape[-1]) + out = _markov_residual(a, weight.T, base.view(-1, base.shape[-1]), tuned=False)["D"] + ctx.save_for_backward(a, weight) + ctx.embedding_shape = embedding.shape + return out.view_as(base) + + @staticmethod + def backward(ctx, grad): + a, weight = ctx.saved_tensors + g = grad.reshape(-1, grad.shape[-1]) + return ( + grad, + (g @ weight).view(ctx.embedding_shape) if ctx.needs_input_grad[1] else None, + g.T @ a if ctx.needs_input_grad[2] else None, + ) + + +markov_residual_projection = _MarkovProjection.apply diff --git a/aspec/models/dflash.py b/aspec/models/dflash.py index 7c4bf17..4a7f2e0 100644 --- a/aspec/models/dflash.py +++ b/aspec/models/dflash.py @@ -468,6 +468,8 @@ def _block_mask_options_for_layer(self, layer_id: int) -> dict: if config is not None and getattr(config, "layer_types", None): if config.layer_types[layer_id] == "sliding_attention": options["context_window"] = config.sliding_window + if not getattr(config, "sliding_window_non_causal", True): + options["is_causal"] = True return options def _compute_logits( diff --git a/aspec/models/dspark.py b/aspec/models/dspark.py index c96c166..91d0a8b 100644 --- a/aspec/models/dspark.py +++ b/aspec/models/dspark.py @@ -18,34 +18,37 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -"""DSpark training model: DFlash training wrapper + Markov / L1 / confidence losses. +"""DSpark training with selectable DeepSpec and Red Hat objectives. -Reuses :class:`DFlashModel`'s anchor sampling, block-causal FlexAttention mask, -and MASK-token noise construction verbatim, then layers on the DSpark training -objective: +The shared DFlash backbone supplies anchor sampling, block attention and MASK-token +inputs. A Markov head biases Draft logits using the observed previous token, and a +confidence head learns the detached Draft/Target probability overlap. - - Markov-biased draft logits (teacher-forced previous token). - - Cross-entropy against the ground-truth next tokens (hard labels). - - L1 distribution distillation: ``|softmax(draft) - softmax(target)|`` where the - target distribution is the frozen LM head applied to the *target's* final - hidden state at the aligned position (requires ``last_hidden_states``). - - Confidence head BCE against the empirical per-token accept rate. +DeepSpec uses next-token CE and full L1 distillation with contiguous supervision. +The optional Red Hat recipe uses teacher-argmax CE, total variation and independent +slot supervision with per-row normalization. Both return additive loss terms so the +trainer can normalize across ranks and gradient-accumulation steps. -Combined: ``ce_alpha*ce + l1_alpha*l1 + confidence_alpha*confidence``. - -Loss formulation adapted from DeepSeek's DeepSpec -(deepspec/modeling/dspark/loss.py, MIT). Additive loss terms are returned to -the trainer so normalization covers every rank and gradient-accumulation step. +The default loss is adapted from DeepSeek's DeepSpec (deepspec/modeling/dspark/loss.py, +MIT). The Red Hat objective reimplements vllm-project/speculators b9c51bc DSpark +metrics, fused CE/TV and row reductions within ASpec's chunked heads. """ from typing import List, Optional, Tuple import torch +import torch.distributed as dist import torch.nn.functional as F +from aspec.config.draft_algorithm import DSparkLossRecipe from aspec.models.dflash import DFlashModel from aspec.models.ops.acceptance import acceptance_components, acceptance_statistics -from aspec.models.ops.loss import compiled_l1_from_logits, use_sm90_tiled_l1 +from aspec.models.ops.loss import ( + compiled_l1_from_logits, + speculators_token_losses, + use_cutedsl_speculators_loss, + use_sm90_tiled_l1, +) from aspec.models.ops.output_blocks import ( empty_prediction_terms, output_block_counts, @@ -69,7 +72,7 @@ def _cached_projection_chunks(counts, row_budget, block_size, *, copies, limit=N class DSparkModel(DFlashModel): - """DSpark training wrapper (DFlash backbone + Markov/L1/confidence heads).""" + """DFlash backbone with Markov logits and selectable DSpark distillation losses.""" prediction_start = 0 @@ -89,6 +92,7 @@ def __init__( anchor_sampling_scope: str = "row", l1_target_sampling: tuple[float, float, int] = (1.0, 1.0, -1), l1_draft_sampling: tuple[float, float, int] = (1.0, 1.0, -1), + loss_recipe: str = "deepspec", ): # Reuse DFlash anchor/mask/noise machinery. The "decay" objective drives # the per-within-block position weighting shared by all DSpark terms. @@ -111,6 +115,9 @@ def __init__( self.confidence_head_alpha = float(confidence_head_alpha) self.l1_sampling = (l1_target_sampling, l1_draft_sampling) self.l1_sampling_enabled = self.l1_sampling != ((1.0, 1.0, -1), (1.0, 1.0, -1)) + self.loss_recipe = DSparkLossRecipe(loss_recipe) + if self.loss_recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1 and self.l1_sampling_enabled: + raise ValueError("speculators_redhat_v1 requires unfiltered distributions at temperature one") self.l1_loss_backend = None self.l1_kernel_invocations = {} self._use_l1_kernel_optimizations = True @@ -157,8 +164,10 @@ def uses_target_distribution(self) -> bool: return False # The confidence head regresses onto an accept rate derived from the # verifier distribution, so the L1 weight alone does not cover it. - return super().uses_target_distribution or ( - self.draft_model.confidence_head is not None and self.confidence_head_alpha > 0 + return ( + self.loss_recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1 + or super().uses_target_distribution + or (self.draft_model.confidence_head is not None and self.confidence_head_alpha > 0) ) def _decay_weights(self, device: torch.device) -> torch.Tensor: @@ -172,6 +181,15 @@ def _decay_weights(self, device: torch.device) -> torch.Tensor: return torch.exp(-(k - self.prediction_start).clamp_min(0).float() / self.loss_decay_gamma) return torch.ones_like(k, dtype=torch.float32) + def _speculators_row_weights(self, mask, decay_weights): + """Normalize within a logical row before pooling rows across DP/microsteps.""" + valid = mask.sum((1, 2), keepdim=True) + if self.draft_context_parallel_size > 1: + dist.all_reduce(valid, group=self.direct_cp_group) + # CP shards contribute disjoint numerators for the same logical rows. + row_count = mask.new_tensor(mask.shape[0] / self.draft_context_parallel_size) + return decay_weights / (valid + 1e-5), decay_weights / (valid + 1e-8), row_count + def forward( self, input_ids: torch.Tensor, @@ -256,14 +274,19 @@ def forward( torch.zeros_like(safe_label_indices), ) - # eval mask = contiguous supervised prefix per block (DeepSpec - # build_eval_mask): block kept, label in-bounds, target token supervised, - # then cumprod so a gap truncates the rest of the block. + warmup_kind = self._indexer_warmup_kind() + speculators = self.loss_recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1 and warmup_kind is None + teacher_indices = (safe_label_indices - 1).clamp_min(0) + mask_indices = teacher_indices if speculators else safe_label_indices target_loss_mask = torch.gather( - loss_mask.unsqueeze(1).expand(-1, n_blocks, -1), 2, safe_label_indices + loss_mask.unsqueeze(1).expand(-1, n_blocks, -1), 2, mask_indices ) eval_bool = block_keep_mask.unsqueeze(-1) & valid_label_mask & (target_loss_mask > 0.5) - eval_bool = eval_bool.to(torch.int32).cumprod(dim=-1).bool() + if speculators and attention_mask is not None: + # Padded BSHD rows keep the same label domain as an unpadded row. + eval_bool &= attention_mask.gather(1, safe_label_indices.flatten(1)).view_as(eval_bool).bool() + if not speculators: + eval_bool = eval_bool.to(torch.int32).cumprod(dim=-1).bool() eval_mask = eval_bool.float() # [B, nb, bs] eval_mask = eval_mask * ( torch.arange(self.block_size, device=device) >= self.prediction_start @@ -272,7 +295,6 @@ def forward( decay_weight_mask = eval_mask * self._decay_weights(device) local_den = decay_weight_mask.sum() - warmup_kind = self._indexer_warmup_kind() if warmup_kind is not None: if warmup_kind == "csa": numerator, loss_components = self._csa_training_loss( @@ -308,6 +330,12 @@ def forward( (numerator, local_den.detach()), ) + confidence_weights = None + if speculators: + decay_weight_mask, confidence_weights, local_den = self._speculators_row_weights( + eval_mask, decay_weight_mask + ) + hidden_4d = draft_hidden.view(bsz, n_blocks, self.block_size, -1) target_ids = torch.gather( input_ids.unsqueeze(1).expand(-1, n_blocks, -1), 2, safe_label_indices @@ -318,12 +346,12 @@ def forward( if self.uses_target_distribution: if last_hidden_states is None: raise ValueError( - "DSpark L1/confidence losses require target last_hidden_states; set " + "DSpark teacher-dependent losses require target last_hidden_states; set " "inference.store_last_hidden_states=true in the run config." ) # target distribution for the token at label_indices = target LM head # applied to the target hidden one position earlier (anchor+j). - tgt_idx = (safe_label_indices - 1).clamp(min=0) # [B, nb, bs] + tgt_idx = teacher_indices # [B, nb, bs] aligned_hidden = self._align_target_hidden_states( last_hidden_states, tgt_idx, @@ -470,14 +498,25 @@ def forward( # Keep only small FP32 L1 statistics across checkpoint replay; full # vocabulary logits still follow the projection budget above. loss_cache = {} if checkpoint_heads and self._reuse_l1_statistics else None - values = (*values, cached_teacher, cached_draft, cached_positions, loss_cache) + conf_weights = confidence_weights[:, start:stop] if confidence_weights is not None else None + if conf_weights is not None and selection is not None: + conf_weights = select_blocks(conf_weights, selection) + values = (*values, cached_teacher, cached_draft, cached_positions, loss_cache, conf_weights) if checkpoint_heads: terms = checkpoint(self._prediction_terms, *values, use_reentrant=False, preserve_rng_state=False) + elif (speculators and n_blocks <= chunk_size and bsz == 1 and cached_teacher is None + and self._reuse_head_projections and self._reuse_context_teacher_projection + and self.draft_context_parallel_size == 1 and not lm_head_weight.requires_grad + and not last_hidden_states.requires_grad and last_hidden_states.shape[1] == seq_len): + positions = tgt_idx[:, start:stop] + if selection is not None: + positions = select_blocks(positions, selection) + terms = self._prediction_terms(*values, teacher_context=(last_hidden_states, positions)) else: terms = self._prediction_terms(*values) head_terms.append(terms) - ce_num, l1_num, conf_num, ce_position_sum, correct_position_sum, acceptance_totals = ( + ce_num, distance_num, conf_num, ce_position_sum, correct_position_sum, acceptance_totals = ( sum(parts[1:], parts[0]) for parts in zip(*head_terms, strict=True) ) @@ -485,7 +524,7 @@ def forward( # before compensating for FSDP's mean gradient reduction. loss_numerator = ( self.ce_loss_alpha * ce_num - + self.l1_loss_alpha * l1_num + + self.l1_loss_alpha * distance_num + self.confidence_head_alpha * conf_num ) moe_numerator, moe_components = self._moe_training_loss( @@ -513,7 +552,7 @@ def forward( component_denominator = local_den.detach() loss_components = { "ce_loss": (ce_num.detach(), component_denominator), - "l1_loss": (l1_num.detach(), component_denominator), + self.loss_recipe.distribution_metric: (distance_num.detach(), component_denominator), "confidence_loss": (conf_num.detach(), component_denominator), **moe_components, **csa_components, @@ -561,26 +600,60 @@ def _project_draft(self, hidden, head_weight): def _prediction_terms(self, hidden, target_ids, prev_ids, head_weight, aligned_target_hidden, eval_mask, weights, cached_target_logits=None, - cached_draft_logits=None, cached_target_positions=None, loss_cache=None): + cached_draft_logits=None, cached_target_positions=None, loss_cache=None, + confidence_weights=None, *, teacher_context=None): """Additive DSpark head terms for a bounded block slice.""" # Explicit token-major GEMM: a block slice can be non-contiguous even # though the complete hidden stream is contiguous. Do not broadcast a # vocabulary projection over many tiny [block_size, hidden] matrices. logits = (cached_draft_logits if cached_draft_logits is not None else self._project_draft(hidden, head_weight)) - if self.draft_model.markov_head is not None: - logits = self.draft_model.markov_head.apply_block_logits(logits, token_ids=prev_ids) vocab = logits.shape[-1] - ce = F.cross_entropy(logits.reshape(-1, vocab), target_ids.reshape(-1), - reduction="none").view_as(weights) - ce_num = (ce * weights).sum() - l1_num = logits.new_zeros((), dtype=torch.float32) + speculators = self.loss_recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1 + cute_loss = (speculators and self._use_l1_kernel_optimizations + and use_cutedsl_speculators_loss(logits, cached_target_logits, + indexed=cached_target_positions is not None)) + markov = self.draft_model.markov_head + prev_emb = None + fused_correct = None + if markov is not None: + if (cute_loss and logits.numel() // vocab >= 256 and vocab % 8 == 0 + and markov.markov_rank % 8 == 0 and markov.markov_w2.weight.is_contiguous() + and markov.markov_w2.weight.dtype == logits.dtype): + from aspec.kernels.cutedsl.speculators_loss import markov_residual_projection + + prev_emb = markov.get_prev_embeddings(prev_ids) + logits = markov_residual_projection(logits, prev_emb, markov.markov_w2.weight) + calls = self.l1_kernel_invocations + calls["cutedsl_markov_projection"] = calls.get("cutedsl_markov_projection", 0) + 1 + else: + logits = markov.apply_block_logits(logits, token_ids=prev_ids) + if not speculators: + ce = F.cross_entropy(logits.reshape(-1, vocab), target_ids.reshape(-1), + reduction="none").view_as(weights) + distance_num = logits.new_zeros((), dtype=torch.float32) accept_rate = None if aligned_target_hidden is not None: + # A long prompt can contain a short supervised region. Many anchors + # then share teacher positions even when projecting the full context + # would cost more than projecting all query rows. + # Small teacher projections do not amortize sorting and its host + # synchronization on SM100. Keep its regular CuTeDSL head path. + if (cute_loss and teacher_context is not None + and torch.cuda.get_device_capability(logits.device) == (9, 0)): + context, positions = teacher_context + unique, inverse = torch.unique(positions.reshape(-1), return_inverse=True) + # Small reductions cost more in the indexed path on Hopper. + if inverse.numel() - unique.numel() >= 64: + cached_target_logits = F.linear(context[0].index_select(0, unique), head_weight)[None] + cached_target_positions = inverse.view_as(positions) + self.loss_head_cached_context_tokens_observed = unique.numel() + self.loss_head_cached_teacher_blocks_observed = hidden.shape[1] indexed_teacher = ( - not self.l1_sampling_enabled - and self._use_l1_kernel_optimizations and cached_target_positions is not None - and use_sm90_tiled_l1(logits, cached_target_logits, True) + cached_target_positions is not None and (cute_loss or ( + not speculators and not self.l1_sampling_enabled and self._use_l1_kernel_optimizations + and use_sm90_tiled_l1(logits, cached_target_logits, True) + )) ) target_indices = None if indexed_teacher: @@ -599,36 +672,54 @@ def _prediction_terms(self, hidden, target_ids, prev_ids, head_weight, target_logits = cached_target_logits if cached_target_logits is not None else F.linear( aligned_target_hidden.reshape(-1, aligned_target_hidden.shape[-1]), head_weight ).view_as(logits) - if not self.l1_sampling_enabled and ( + if cute_loss and not target_logits.requires_grad and target_logits.dtype == logits.dtype: + from aspec.kernels.cutedsl.speculators_loss import speculators_cutedsl_loss + + ce, distance, target_ids, fused_correct = speculators_cutedsl_loss( + logits, target_logits, target_indices) + self.l1_loss_backend = "cutedsl_ce_tv" + kernel = ("indexed_" if indexed_teacher else "") + self.l1_loss_backend + calls = self.l1_kernel_invocations + calls[kernel] = calls.get(kernel, 0) + 1 + elif speculators: + ce, distance, target_ids = speculators_token_losses(logits, target_logits) + self.l1_loss_backend = "torch_compiled_tv" + elif not self.l1_sampling_enabled and ( indexed_teacher or use_sm90_tiled_l1(logits, target_logits) ): from aspec.models.ops.tiled_l1 import tiled_l1_from_logits - l1 = tiled_l1_from_logits(logits, target_logits, target_indices=target_indices, - _replay_cache=loss_cache) + distance = tiled_l1_from_logits(logits, target_logits, target_indices=target_indices, + _replay_cache=loss_cache) self.l1_loss_backend = "tiled_fp32" kernel = ("indexed_" if indexed_teacher else "") + self.l1_loss_backend self.l1_kernel_invocations[kernel] = self.l1_kernel_invocations.get(kernel, 0) + 1 else: - l1 = (compiled_l1_from_logits(logits, target_logits, *self.l1_sampling) - if self.l1_sampling_enabled else compiled_l1_from_logits(logits, target_logits)) + distance = (compiled_l1_from_logits(logits, target_logits, *self.l1_sampling) + if self.l1_sampling_enabled else compiled_l1_from_logits(logits, target_logits)) self.l1_loss_backend = "torch_compiled" if self.l1_loss_alpha > 0: - l1_num = (l1 * weights).sum() - accept_rate = (1.0 - 0.5 * l1).clamp(0.0, 1.0) + distance_num = (distance * weights).sum() + accept_rate = 1.0 - distance if speculators else (1.0 - 0.5 * distance).clamp(0.0, 1.0) + ce_num = (ce * weights).sum() conf_num = logits.new_zeros((), dtype=torch.float32) if self.draft_model.confidence_head is not None and self.confidence_head_alpha > 0: if self.draft_model.confidence_head_with_markov: - prev_emb = self.draft_model.markov_head.get_prev_embeddings(prev_ids).to(hidden.dtype) + # Keep the two embedding backward reductions independent. Sharing + # this lookup changes BF16 scatter/add rounding for repeated IDs. + prev_emb = markov.get_prev_embeddings(prev_ids).to(hidden.dtype) conf_features = torch.cat([hidden, prev_emb], dim=-1) else: conf_features = hidden - prediction = self.draft_model.confidence_head(conf_features).float() + prediction = self.draft_model.confidence_head(conf_features) + confidence_target = accept_rate.detach() + if speculators: + confidence_target = confidence_target.to(prediction.dtype) conf_num = (F.binary_cross_entropy_with_logits( - prediction, accept_rate.detach(), reduction="none" - ) * weights).sum() + prediction.float(), confidence_target.float(), reduction="none" + ) * (weights if confidence_weights is None else confidence_weights)).sum() with torch.no_grad(): - correct = (logits.argmax(-1) == target_ids) & (eval_mask > 0.5) + correct = ((logits.argmax(-1) == target_ids) if fused_correct is None else fused_correct) & (eval_mask > 0.5) position_loss = (ce * eval_mask).sum((0, 1)) position_correct = correct.float().sum((0, 1)) acceptance_totals = ( @@ -639,4 +730,4 @@ def _prediction_terms(self, hidden, target_ids, prev_ids, head_weight, (2, self.block_size + 1), dtype=torch.float32, ) ) - return ce_num, l1_num, conf_num, position_loss, position_correct, acceptance_totals + return ce_num, distance_num, conf_num, position_loss, position_correct, acceptance_totals diff --git a/aspec/models/ops/loss.py b/aspec/models/ops/loss.py index 64fe4e8..ad2d4be 100644 --- a/aspec/models/ops/loss.py +++ b/aspec/models/ops/loss.py @@ -18,6 +18,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from functools import lru_cache +from importlib.util import find_spec from typing import Optional import torch @@ -46,6 +48,31 @@ def use_sm90_tiled_l1(logits: torch.Tensor, target: torch.Tensor, indexed: bool ) +@lru_cache(maxsize=1) +def _have_cutedsl_loss(): + return all(find_spec(pkg) is not None for pkg in ("quack", "cutlass")) + + +def use_cutedsl_speculators_loss(logits, teacher=None, indexed=False): + """At most two reference normalization blocks and one CTA of overlap partials.""" + return ( + not torch.compiler.is_compiling() and logits.is_cuda + and logits.ndim > 0 and logits.numel() > 0 and logits.dtype == torch.bfloat16 + and logits.is_contiguous() and logits.data_ptr() % 16 == 0 + and 1 <= logits.shape[-1] <= 196608 + and (teacher is None or ( + teacher.device == logits.device and teacher.dtype == logits.dtype + and teacher.is_contiguous() and teacher.data_ptr() % 16 == 0 + and not teacher.requires_grad + and (teacher.shape == logits.shape if not indexed else + teacher.ndim == 3 and teacher.shape[0] == logits.shape[0] + and teacher.shape[-1] == logits.shape[-1]) + )) + and torch.cuda.get_device_capability(logits.device) in {(9, 0), (10, 0)} + and _have_cutedsl_loss() + ) + + def _forward_kl_from_logits(logits: torch.Tensor, target_p: torch.Tensor) -> torch.Tensor: logits_f32 = logits.float() return torch.logsumexp(logits_f32, dim=-1) - (target_p * logits_f32).sum(-1) @@ -91,6 +118,19 @@ def compiled_l1_from_logits(logits: torch.Tensor, target_logits: torch.Tensor, return (draft_probs - target_probs).abs().sum(dim=-1) +@torch.compile(dynamic=None) +def speculators_token_losses(logits: torch.Tensor, teacher_logits: torch.Tensor): + """Reimplement Speculators b9c51bc fused CE/TV with FP32 reductions.""" + labels = teacher_logits.argmax(-1) + draft = logits.float() + ce = draft.logsumexp(-1) - draft.gather(-1, labels.unsqueeze(-1)).squeeze(-1) + q = draft.softmax(-1) + p = teacher_logits.detach().float().softmax(-1) + # The published fused TV backward assigns equality to the Draft branch. + overlap = torch.where(q <= p, q, p).sum(-1) + return ce, 1.0 - overlap, labels + + def _project( hidden: torch.Tensor, lm_head_weight: torch.Tensor, diff --git a/aspec/models/target/target_utils.py b/aspec/models/target/target_utils.py index f0a2cfb..77c2b9a 100644 --- a/aspec/models/target/target_utils.py +++ b/aspec/models/target/target_utils.py @@ -29,7 +29,8 @@ import torch.nn as nn from huggingface_hub import snapshot_download from safetensors import safe_open -from transformers import AutoConfig + +from aspec.config.utils import load_target_config logger = logging.getLogger(__name__) @@ -85,7 +86,7 @@ def from_pretrained( dtype: torch.dtype = torch.bfloat16, trust_remote_code: bool = False, ) -> "TargetLMHead": - config = AutoConfig.from_pretrained( + config = load_target_config( model_path, cache_dir=cache_dir, trust_remote_code=trust_remote_code ) instance = cls(config, trust_remote_code=trust_remote_code) @@ -324,7 +325,7 @@ def load_synced_target_lm_head( trust_remote_code=trust_remote_code, ) else: - config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code) + config = load_target_config(model_path, trust_remote_code=trust_remote_code) head = TargetLMHead(config, trust_remote_code=trust_remote_code) if load_norm: head._init_norm_structure() diff --git a/aspec/planning.py b/aspec/planning.py index 4b0d3d8..0db4b58 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -34,7 +34,7 @@ from omegaconf import DictConfig, ListConfig, OmegaConf -from aspec.config.draft_algorithm import DRAFT_ALGORITHMS +from aspec.config.draft_algorithm import DRAFT_ALGORITHMS, DSparkLossRecipe from aspec.config.draft_attention import ( draft_thd_contract, resolve_draft_attention, @@ -444,6 +444,12 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]: except ValueError as exc: raise PlanCompileError(str(exc)) from exc backend = _select(config, "training.attention_backend") + try: + recipe = DSparkLossRecipe(_select(config, "training.dspark_loss_recipe", "deepspec")) + except ValueError as exc: + raise PlanCompileError(f"training.dspark_loss_recipe must be one of {list(map(str, DSparkLossRecipe))}") from exc + if recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1 and family != "dspark": + raise PlanCompileError("training.dspark_loss_recipe=speculators_redhat_v1 requires a DSpark Draft") l1_sampling_enabled = False for side in ("target", "draft"): prefix = f"training.dspark_l1_{side}_" @@ -460,6 +466,8 @@ def _load_draft_descriptor(config: DictConfig) -> dict[str, Any]: raise PlanCompileError(f"{prefix}top_k must be -1 or a positive integer") l1_sampling_enabled |= (temperature, top_p, top_k) != (1.0, 1.0, -1) if l1_sampling_enabled: + if recipe == DSparkLossRecipe.SPECULATORS_REDHAT_V1: + raise PlanCompileError("speculators_redhat_v1 requires unfiltered distributions at temperature one") indexer_warmup = any((attention.get(kind) or {}).get("training_stage") == "indexer_warmup" for kind in ("csa", "qsa", "dsa")) if family != "dspark" or _select(config, "training.dspark_l1_loss_alpha") <= 0 or indexer_warmup: @@ -785,6 +793,12 @@ def _resolve_target_family(target_path: str) -> str: local_config = _read_local_target_config(target_path) markers: list[str] = [target_path] if local_config is not None: + if ( + local_config.get("model_type") == "glm5_next" + and local_config.get("architectures") == ["Glm5NextForConditionalGeneration"] + and (local_config.get("text_config") or {}).get("num_hidden_layers") == 45 + ): + return "glm5_3_flash" markers.append(str(local_config.get("model_type", ""))) architectures = local_config.get("architectures", []) if isinstance(architectures, list): @@ -795,6 +809,8 @@ def _resolve_target_family(target_path: str) -> str: and "BailingMoeV3ForCausalLM" in architectures ): return "ling3" + elif target_path.lower().rstrip("/") in {"zai-org/glm-5.3-flash", "zhipuai/glm-5.3-flash"}: + return "glm5_3_flash" elif target_path.rstrip("/").split("/")[-1].lower() in { "ling-3.0-tiny", "ling-3.0-flash", "ling-3.1-flash", }: @@ -802,7 +818,7 @@ def _resolve_target_family(target_path: str) -> str: if any("qwen3" in marker.lower().replace("_", "") for marker in markers): return "qwen3" raise PlanCompileError( - "The supported target scope is Qwen3 and Ling-3.x, but the target model family " + "The supported target scope includes Qwen3, Ling-3.x and GLM-5.3-Flash, but the target model family " f"could not be established from {target_path!r}" ) @@ -1452,6 +1468,7 @@ def _workload( elif family == "dspark": objective = { "family": family, + "loss_recipe": _select(config, "training.dspark_loss_recipe", "deepspec"), "block_size": _select(config, "training.dflash_block_size"), "num_anchors": _select(config, "training.dspark_num_anchors"), "num_target_layers": _select(config, "training.dspark_num_target_layers"), @@ -2018,6 +2035,9 @@ def compile_execution_plan( source["resolved_config_sha256"] = _resolved_config_sha256(config) draft = _load_draft_descriptor(config) target_family = _resolve_target_family(target_path) + if target_family == "glm5_3_flash" and _select(config, "inference.target_prefill_memory_optimized", False): + raise PlanCompileError("GLM-5.3-Flash uses buffered native DFlash feature capture") + runtime = _runtime_values(config, draft, source_context_parallel_size=source_context_parallel_size) try: context_compiler = compile_offline_context_contract if offline else compile_sglang_context_contract diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index 3738fab..6fcd17f 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -87,6 +87,7 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: "ce_loss_alpha", "l1_loss_alpha") if prefix == "dspark": topology["training_objective"]["lm_head_precision"] = getattr(args, "dspark_lm_head_precision", "bf16") + topology["training_objective"]["loss_recipe"] = getattr(args, "dspark_loss_recipe", "deepspec") fields += ("confidence_head_alpha",) for side in ("target", "draft"): for name, default in (("temperature", 1.0), ("top_p", 1.0), ("top_k", -1)): @@ -163,6 +164,8 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec saved_objective = (saved_topology or {}).get("training_objective") or {} if saved_objective.get("lm_head_precision", "bf16") != getattr(args, "dspark_lm_head_precision", "bf16"): raise RuntimeError("Exact resume cannot change dspark_lm_head_precision; use weights-only initialization") + if saved_objective.get("loss_recipe", "deepspec") != current_topology["training_objective"]["loss_recipe"]: + raise RuntimeError("Exact resume cannot change dspark_loss_recipe; use weights-only initialization") for side in ("target", "draft"): for name, default in (("temperature", 1.0), ("top_p", 1.0), ("top_k", -1)): key = f"l1_{side}_{name}" @@ -825,8 +828,44 @@ def resolve_initial_draft_checkpoint(path: str) -> Path: return checkpoint_path +def _bind_speculators_dspark(directory, config, tensors, target_bindings): + """Validate the published Draft's layout and replace its frozen head with the Target binding.""" + config_path = directory / "config.json" + if not config_path.is_file(): + return + published = json.loads(config_path.read_text()) + if published.get("speculators_model_type") != "dspark": + return + + # Speculators captures before a layer; ASpec numbers completed layer outputs. + policies = { + "aux_hidden_state_layer_ids": [layer + 1 for layer in config.target_layer_ids], + "mask_token_id": config.mask_token_id, + "block_size": config.block_size, + "sample_from_anchor": True, + "sliding_window_non_causal": getattr(config, "sliding_window_non_causal", True), + } + if any(published.get(k) != expected for k, expected in policies.items()): + raise ValueError("Speculators DSpark capture, mask, block or attention policy mismatch") + geometry = published["transformer_layer_config"] + fields = ("sliding_window", "layer_types", "rms_norm_eps", "num_attention_heads", + "num_key_value_heads", "head_dim") + if (any(geometry.get(field) != getattr(config, field) for field in fields) + or geometry.get("rope_parameters", {}).get("rope_theta") != config.rope_theta): + raise ValueError("Speculators DSpark layer geometry differs from the configured Draft") + + from aspec.models.draft.base import load_tensor_from_pretrained + + model_path, embedding_key, lm_head_key = target_bindings + for name, key in (("embed_tokens.weight", embedding_key), ("lm_head.weight", lm_head_key)): + if name not in tensors or not torch.equal(tensors[name], load_tensor_from_pretrained(model_path, key)): + raise ValueError(f"Published DSpark {name} differs from the frozen Target binding") + del tensors["lm_head.weight"] + + def load_initial_draft_weights( - draft_model: torch.nn.Module, path: str, *, allow_target_embedding: bool = False + draft_model: torch.nn.Module, path: str, *, allow_target_embedding: bool = False, + target_bindings: tuple[str, str, str] | None = None, ) -> Path: """Load a published draft checkpoint into a freshly constructed draft model. @@ -842,6 +881,8 @@ def load_initial_draft_weights( manifest_path = checkpoint_path.parent / "export_manifest.json" native = manifest_path.is_file() and json.loads(manifest_path.read_text()).get("storage_format") == "aspec_native" conditioning = getattr(draft_model.config, "mtp_conditioning", None) + if target_bindings and not native: + _bind_speculators_dspark(checkpoint_path.parent, draft_model.config, tensors, target_bindings) transition = False if conditioning: saved_config_path = checkpoint_path.parent / "config.json" diff --git a/aspec/training/dflash_trainer.py b/aspec/training/dflash_trainer.py index fbd5a28..1c3571d 100644 --- a/aspec/training/dflash_trainer.py +++ b/aspec/training/dflash_trainer.py @@ -150,8 +150,14 @@ def init_model( draft_model.initialize_mtp_conditioning(target_model_path) initial_path = getattr(self.args, "initial_draft_model_path", None) if initial_path: + target_bindings = ( + target_model_path, + getattr(self.args, "embedding_key", "model.embed_tokens.weight"), + getattr(self.args, "lm_head_key", "lm_head.weight"), + ) checkpoint.load_initial_draft_weights( - draft_model, initial_path, allow_target_embedding=True + draft_model, initial_path, allow_target_embedding=True, + target_bindings=target_bindings if config.model_type == "qwen3_dspark" else None, ) draft_model.load_embedding( target_model_path, @@ -205,7 +211,7 @@ def init_model( self._anchor_slot_offset = self.dflash.prediction_start if hasattr(self, "confidence_head_alpha"): self._extra_loss_component_keys = [ - "ce_loss", "l1_loss", "confidence_loss", + "ce_loss", self.dflash.loss_recipe.distribution_metric, "confidence_loss", *(f"suffix_accept_rate@{index}" for index in range( self.block_size - self.dflash.prediction_start)), "suffix_tau_probabilistic", @@ -366,6 +372,8 @@ def get_feature_contract_status(self) -> dict: } ) status["execution_metadata"] = execution + if hasattr(self.dflash, "loss_recipe"): + execution["dspark_loss_recipe"] = [self.dflash.loss_recipe.value] status["draft_attention"] = {**observed, "forward_observed": True} if self._indexer_warmup_contract() is not None: main_frozen = not any( diff --git a/aspec/training/dspark_trainer.py b/aspec/training/dspark_trainer.py index f53586e..42d6055 100644 --- a/aspec/training/dspark_trainer.py +++ b/aspec/training/dspark_trainer.py @@ -27,6 +27,7 @@ from argparse import Namespace +from aspec.config.draft_algorithm import DSparkLossRecipe from aspec.models.draft.dspark import ( DSparkConfig, DSparkDraftModel, @@ -58,7 +59,8 @@ def __init__(self, args: Namespace): # Reuse the shared component reducer: all acceptance counts join the # existing batched collective and host transfer for train and eval. self._extra_loss_component_keys = [ - *self._extra_loss_component_keys, + "ce_loss", DSparkLossRecipe(getattr(args, "dspark_loss_recipe", "deepspec")).distribution_metric, + "confidence_loss", *(f"accept_rate@{index}" for index in range(self.block_size)), "tau_probabilistic", ] @@ -90,6 +92,7 @@ def _build_training_wrapper(self, draft_model): return DSparkModel( draft_model=draft_model, block_size=self.block_size, + loss_recipe=getattr(self.args, "dspark_loss_recipe", "deepspec"), num_anchors=self.num_anchors, loss_decay_gamma=self.loss_decay_gamma, ce_loss_alpha=self.ce_loss_alpha, diff --git a/configs/recipes/glm5_3_flash_dspark.yaml b/configs/recipes/glm5_3_flash_dspark.yaml new file mode 100644 index 0000000..a392b8f --- /dev/null +++ b/configs/recipes/glm5_3_flash_dspark.yaml @@ -0,0 +1,103 @@ +# GLM-5.3-Flash FP8 Target, TP4, text-only DSpark training. +model: + target_model_path: ${oc.env:ASPEC_TARGET_MODEL,zai-org/GLM-5.3-Flash} + native_context_length: 1048576 + trust_remote_code: false + draft_model_config: aspec/config/dspark_glm5_3_flash_draft_config.json + embedding_key: model.language_model.embed_tokens.weight + lm_head_key: lm_head.weight + norm_key: model.language_model.norm.weight + +dataset: + eval_interval: 100 + eval_micro_batch_size: 1 + train_data_path: ${oc.env:ASPEC_TRAIN_DATA} + eval_data_path: ${oc.env:ASPEC_EVAL_DATA} + renderer: glm5_3_flash + prompt_key: conversations + min_loss_tokens: 32 + shuffle_dataset: true + +training: + attention_backend: flex_attention + draft_context_parallel_size: 1 + micro_batch_size: 1 + draft_accumulation_steps: 16 + learning_rate: 6e-4 + min_lr: 6e-5 + weight_decay: 0.0 + max_concurrent_batches: 1 + streaming_dispatch_with_training: true + pipeline_dispatch_with_training: true + max_grad_norm: 1.0 + max_seq_length: 8192 + num_epochs: 3 + seed: 42 + training_num_gpus_per_node: 1 + training_num_nodes: 1 + placement_strategy: inference_first + fsdp_strategy: REPLICATE + fsdp_reduce_dtype: bfloat16 + prefetch_depth: 1 + save_interval: 100 + save_per_epoch: false + save_final_checkpoint: true + max_checkpoints: 2 + warmup_ratio: 0.04 + dflash_block_size: 8 + dflash_packing: false + dspark_loss_recipe: deepspec + dspark_num_anchors: 512 + dspark_num_target_layers: 6 + dspark_loss_decay_gamma: 4.0 + dspark_ce_loss_alpha: 0.1 + dspark_l1_loss_alpha: 0.9 + dspark_confidence_head_alpha: 1.0 + +inference: + inference_engine_type: sgl + store_last_hidden_states: true + target_prefill_memory_optimized: false + inference_num_gpus: 4 + inference_num_gpus_per_engine: 4 + inference_num_gpus_per_node: 5 + inference_fetch_batch: 8 + inference_buffer_threshold: 2 + max_sample_pool_size: 8 + inference_batch_size: 2 + sglang: + # Cold sharded checkpoint loading can exceed the generic five-minute budget. + init_timeout: 1200 + tp_size: 4 + extra_args: + watchdog_timeout: 1800 + dsa_prefill_backend: tilelang + dsa_decode_backend: tilelang + kv_cache_dtype: bf16 + moe_runner_backend: deep_gemm + mem_fraction_static: 0.9 + +mooncake: + data_plane: transfer_engine + protocol: nvlink_intra + require_nvlink: true + enable_gpu_direct: true + allow_host_fallback: false + enable_hard_pin: true + max_seq_len: 8192 + hidden_dim: 4096 + global_segment_size: 16GB + local_buffer_size: 4GB + host_buffer_size: 1GB + gpu_ring_slots: 8 + source_gpu_ring_slots: 8 + destination_gpu_ring_slots: 1 + kv_lease_ttl_s: 600 + get_retry_max_wait_seconds: 600 + +logging: + report_to: none +debug: + enable_perf_metrics: true +output_dir: outputs/glm5-3-flash-dspark +cache_dir: cache/glm5-3-flash-dspark diff --git a/docs/framework/REFERENCES.md b/docs/framework/REFERENCES.md index fa68237..cb6d3ce 100644 --- a/docs/framework/REFERENCES.md +++ b/docs/framework/REFERENCES.md @@ -25,6 +25,9 @@ the retained source identities; applicable license and notice texts are distribu | [Speculators](https://github.com/vllm-project/speculators) `3702320434bbdaa853f0846a66e6b6b8f1d97aec` | Adapted P-EAGLE model/loss and Qwen3 first-layer fusion; deterministic dense layout and ASpec input/observation boundary | | [Domino](https://github.com/jianuo-huang/Domino) `44e1e6912ba9d301899739a54bfda2ce8826980a` and [arXiv:2605.29707](https://arxiv.org/abs/2605.29707) | Public method and label-layout reference for the GRU correction head; independently implemented on the retained DFlash block backbone | | [SpecForge Domino](https://github.com/sgl-project/SpecForge/pull/571) merge `7de39e32fc845caf59bca9841595145d24e1796f` | Adapted Domino objective, predecessor alignment and label-shift handling (MIT); ASpec supplies configuration, fail-closed scope and DFlash integration | +| [Speculators DSpark](https://github.com/vllm-project/speculators/tree/b9c51bcfc134923eae2b6a6f8246ae3500d5c42f/src/speculators/models/dspark) | Reference for the optional Red Hat loss, row reduction, confidence target and causal sliding-block attention | +| [QuACK](https://github.com/Dao-AILab/quack) 0.6.5 ([Apache-2.0 license](../../licenses/QuACK-LICENSE)) | Hopper/Blackwell CE/TV adapts the softmax tile layout and uses the cluster reduction and GEMM epilogue APIs; ASpec implements the upstream loss and BF16 rounding. | +| [Red Hat GLM-5.3-Flash DSpark](https://huggingface.co/RedHatAI/GLM-5.3-Flash-speculator.dspark/tree/738c283a7e860d7dafce5785161466bc556e5cae) | Five-layer Draft geometry, capture layers and frozen Target bindings for published-weight initialization | | [Ling-3.0-flash](https://huggingface.co/inclusionAI/Ling-3.0-flash) `fcc60a52b34c69c6271d0ac8a8cb0815020b6a43` | Adapted MoE routing and independently retained reference class bodies; Apache-2.0 source headers preserved | | [EAGLE](https://github.com/SafeAILab/EAGLE), [GPT-NeoX](https://github.com/EleutherAI/gpt-neox) and Transformers through TorchSpec | Model/data helpers in `aspec/models/draft/base.py`, shared checkpoint reads in `aspec/models/checkpoint.py`, and `aspec/data/preprocessing.py`; original source headers retain the transitive attribution | diff --git a/docs/support.md b/docs/support.md index 49f4391..02e1e98 100644 --- a/docs/support.md +++ b/docs/support.md @@ -10,6 +10,7 @@ and tests do not imply CPU training support. | Path | Implemented behavior | Boundary | | --- | --- | --- | | DSpark / DFlash | Training, exact recovery, evaluation and export; Dense and selected MoE/attention structures | Objectives, feature bindings and assets remain algorithm-specific | +| GLM-5.3-Flash / DSpark | Official FP8 TP4 Target, five-layer BF16 Draft, DeepSpec and Red Hat loss recipes, checkpoint recovery and native export | Text-only, CP1, one Draft GPU. [Training workflow and scope](user-guide/recipes.md#glm-53-flash-dspark) | | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | | Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; `lambda_base` curriculum, export and consumer acceptance pending; see [scope](user-guide/domino.md) | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index b05d98c..c0e5d21 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -20,7 +20,9 @@ RAY_ADDRESS=local aspec --config configs/recipes/qwen3_8b_eagle3.yaml \ model.target_model_path=/path/to/Qwen3-8B ``` -DSpark retains CE/L1/Markov/confidence semantics; DFlash does not use those auxiliary targets. +DSpark defaults to DeepSpec CE/L1/confidence training with a Markov head. Set +`training.dspark_loss_recipe=speculators_redhat_v1` for teacher-argmax CE and TV. +See [DSpark loss recipes](training.md#dspark-loss-recipes) for the masks and reductions. EAGLE3 uses recurrent TTT and its selected distillation objective. Its pruned vocabulary preserves the checkpoint's `d2t/t2d` mapping across recovery/export. Teacher argmax outside the selected vocabulary is filtered; omitted probability mass is not reassigned to the reduced head. @@ -55,6 +57,9 @@ aspec --config configs/recipes/qwen3_8b_eagle3.yaml --dry-run \ TTT shifts, teacher halos and supervision remain document-local. Unsupported factorized/ambiguous heads fail export rather than producing an unfaithful serving asset. +For text-only DSpark training against the official GLM-5.3-Flash FP8 Target, see the +[GLM training workflow](recipes.md#glm-53-flash-dspark). + ## P-EAGLE core prerequisite The [P-EAGLE model/loss API](peagle.md) provides a dense unsampled Qwen3 parallel diff --git a/docs/user-guide/recipes.md b/docs/user-guide/recipes.md index 68cb315..19bb32a 100644 --- a/docs/user-guide/recipes.md +++ b/docs/user-guide/recipes.md @@ -24,6 +24,7 @@ connections on your hardware before training. | `ling-tiny` | Ling-3.0-tiny / Dense DSpark | 1 node, 2 GPUs: 1T + 1D; 4K | Train a small Ling Draft | | `ling-flash` | Ling-3.0-flash / Dense DSpark | 1 node, 3 GPUs: 2T (TP2) + 1D; 4K | Train against a larger Ling Target | | `ling-flash31` | Ling-3.1-flash / MLA + LingMoE DSpark | 1 node, 8 GPUs: 4T (TP4/EP4) + 4D; 4K | Train a structured Draft against the FP8 Ling Target | +| `glm5-flash` | GLM-5.3-Flash FP8 / DSpark | 1 node, 4T (TP4) + 1D, 8K | Train a text Draft from scratch or continue from Red Hat weights. [GLM training guide](#glm-53-flash-dspark) | | `shared-target` | Qwen3-8B / two independent DSpark Drafts | 1 node, 8 GPUs: 4T + 2D × 2; 4K | Compare independent Drafts sharing Target features | | `qwen27b-long` | Qwen3.8-27B / DSpark | 1 node, 8 GPUs: 3 × TP2 Target + 2D (CP2); 256K-class | Distribute long-context training with CP | | `multinode` | Qwen3-8B / DSpark | 2 nodes × 8 GPUs; each 4T + 4D; 4K | Keep features node-local and synchronize gradients across nodes | @@ -130,6 +131,62 @@ Set the same train/eval environment variables, then run this YAML directly with See [decode reproducibility](../support.md#decode-reproducibility) before interpreting acceptance or inference timing. +## GLM-5.3-Flash DSpark + +The recipe uses the official FP8 Target on four GPUs and trains a BF16 Draft on a fifth GPU. +Use the checkpoint's chat template with text-only `system`, `user` and `assistant` messages +in disjoint conversation JSONL files. Assistant `reasoning_content` is supported. +[Official Target](https://modelscope.cn/models/ZhipuAI/GLM-5.3-Flash) · [Draft configuration](../../aspec/config/dspark_glm5_3_flash_draft_config.json) + +Activate the patched Target environment and start the service. DeepGEMM compiles kernels for +the shapes used by the workload instead of precompiling its full shape range: + +```bash +SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 CUDA_VISIBLE_DEVICES=0,1,2,3 aspec target +``` + +In the Draft environment, set the inputs and choose one initialization. The first command trains +with the default DeepSpec loss. The second loads the published BF16 Draft, verifies its frozen +embedding and output head against the Target, and starts a new optimizer with Red Hat's loss. + +```bash +export ASPEC_TARGET_MODEL=/models/GLM-5.3-Flash +export ASPEC_TRAIN_DATA=/data/glm-train.jsonl +export ASPEC_EVAL_DATA=/data/glm-eval.jsonl +export CUDA_VISIBLE_DEVICES=4 + +aspec recipes train glm5-flash --output outputs/glm-scratch +aspec recipes train glm5-flash --output outputs/glm-warm \ + --set model.initial_draft_model_path=/models/GLM-5.3-Flash-speculator.dspark \ + --set training.dspark_loss_recipe=speculators_redhat_v1 \ + --set training.learning_rate=1e-6 --set training.min_lr=1e-7 +``` + +The published Draft directory must contain its original `config.json` and `model.safetensors`. +Its six pre-layer captures (20, 28, 32, 36, 40, 44) correspond to ASpec's zero-based layer +outputs (19, 27, 31, 35, 39, 43). The five Draft layers use a 2,048-token anchor-relative +context window and causal attention within each eight-slot block. +[Red Hat checkpoint and configuration](https://huggingface.co/RedHatAI/GLM-5.3-Flash-speculator.dspark/tree/738c283a7e860d7dafce5785161466bc556e5cae) · [Loss definitions](training.md#dspark-loss-recipes) + +To resume an interrupted ASpec run, use the same loss recipe, dataset and LR horizon. Pass +`--set training.load_path=/absolute/path/to/previous-run/checkpoints` with a fresh output path. +The checkpoint restores the optimizer, scheduler and committed work. To export one completed +iteration, run the following from the Draft environment: + +```bash +aspec export --format aspec --input-dir /path/to/checkpoints/iter_0000005 \ + --output-dir /path/to/glm-draft-asset --target-model-path "$ASPEC_TARGET_MODEL" \ + --embedding-key model.language_model.embed_tokens.weight +python -c 'import sys; from aspec.draft_artifact import load_native_draft; load_native_draft(sys.argv[1])' /path/to/glm-draft-asset +``` + +The maintained GLM path uses text-only 8K inputs, CP1 and one Draft GPU. Warm-start +initialization loads model weights and starts a new optimizer; it does not restore the published +Draft's original optimizer state. Native exports preserve the Draft tensors and frozen Target +bindings. Full-corpus convergence, reproduction of the published Draft's quality, integrated +speculative-serving throughput, multimodal inputs and training the Target's native MTP layer +remain outside the validated scope. + ## Additional structures SWA/CSA/QSA/DSA require their documented structure and stage contracts. MLA materializes K/V. diff --git a/docs/user-guide/training.md b/docs/user-guide/training.md index 66eacbf..6305d4f 100644 --- a/docs/user-guide/training.md +++ b/docs/user-guide/training.md @@ -180,9 +180,42 @@ Run `aspec --config offline.yaml --dry-run`, then `aspec --config offline.yaml`. Dataset metadata binds the Target configuration, tokenizer and weights by SHA256, plus the feature layers, dtype, layout and normalization convention. Replay validates sample checksums and tensor shapes. Set `training.max_seq_length` to cover all recorded sequences; changing preprocessing requires regenerating the dataset. The dataset identity participates in exact-resume validation. -## DSpark L1 sampling distributions - -DSpark L1 supports independent Target and Draft sampling parameters. Both sides default to `temperature=1.0`, `top_p=1.0` and `top_k=-1` (disabled). Configure either side or both: +## DSpark loss recipes + +Set `training.dspark_loss_recipe` in the run YAML or pass it as a CLI override. +Omitting it selects `deepspec`. + +| Recipe | CE label at anchor `a`, slot `k` | Distribution term | Supervision and reduction | +| --- | --- | --- | --- | +| `deepspec` | Data token at `a+k+1` | Full L1 | Use the contiguous supervised prefix starting at `a+1`. Divide the decayed sum by total valid decay weight across the optimizer window. | +| `speculators_redhat_v1` | Teacher argmax at `a+k` | Total variation (TV) | Read the mask at each `a+k` independently. Divide each row's decayed sum by its valid slot count, then average the rows. An empty row contributes zero. | + +Both use the observed token at `a+k` for the Markov head. The confidence head learns +the detached probability overlap. The existing `dspark_l1_loss_alpha` weights the +selected distribution term. Training and evaluation report it as `l1_loss` or `tv_loss`. +The default coefficients are 0.1 for CE, 0.9 for the distribution term and 1 for confidence. +In a packed batch the Red Hat reduction counts each physical packed row once. + +The Red Hat objective reimplements the GLM-5.3-Flash draft's pinned Speculators loss +on ASpec's sampled anchors, with document-local supervision. Anchor selection, Draft +architecture, Target feature capture and optimizer are configured separately. +Exact resume requires the saved recipe. Checkpoints predating this field use `deepspec`. +Use weights-only initialization to change recipes. [Upstream loss and reductions](https://github.com/vllm-project/speculators/tree/b9c51bcfc134923eae2b6a6f8246ae3500d5c42f/src/speculators/losses) + +On Hopper (SM90) and Blackwell (SM100), the Draft training environment runs contiguous BF16 Red Hat logits through +CuTeDSL. The normalizer follows the pinned fused reference's FP32 reduction order: rounding can +otherwise change which branch receives the TV gradient at equal probabilities. CE, TV and greedy +agreement share a tiled vocabulary scan. A cached teacher can be read by position without expanding +its vocabulary tensor. Hopper also projects only unique teacher positions in sparsely supervised +head chunks; Blackwell skips this extra deduplication to avoid the sorting and synchronization cost. +On eligible head chunks the Markov GEMM epilogue rounds its projection to BF16 before adding +the base logits. +Other devices, unsupported layouts or missing CuTeDSL/QuACK +use the portable PyTorch loss. This does not change the DeepSpec recipe or the Target precision. + +### DSpark L1 sampling distributions + +The DeepSpec recipe supports independent Target and Draft sampling parameters. Both sides default to `temperature=1.0`, `top_p=1.0` and `top_k=-1` (disabled). ```yaml training: diff --git a/licenses/QuACK-LICENSE b/licenses/QuACK-LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/licenses/QuACK-LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tests/cpu/test_dspark_loss_recipes.py b/tests/cpu/test_dspark_loss_recipes.py new file mode 100644 index 0000000..48c11df --- /dev/null +++ b/tests/cpu/test_dspark_loss_recipes.py @@ -0,0 +1,188 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""DSpark loss contracts for the Red Hat Speculators b9c51bc training objective.""" + +import copy +from pathlib import Path + +import pytest +import torch +import torch.nn.functional as F + +from aspec.config import load_config +from aspec.config.train_config import config_to_flat_args +from aspec.models.dspark import DSparkModel +from aspec.models.ops.loss import speculators_token_losses +from aspec.planning import PlanCompileError, compile_execution_plan +from aspec.training.checkpoint import _logical_training_topology, _validate_exact_resume_topology +from aspec.training.dspark_trainer import DSparkTrainer +from tests.test_dspark_loss_alignment import TinyDraft, objective_fixture + +RECIPE = "speculators_redhat_v1" +CONFIG = Path(__file__).resolve().parents[2] / "configs/examples/sglang_qwen3_8b_dspark.yaml" + + +def reference_row(draft, batch, row, anchors, keep): + """Independent dense reference, one upstream-style forward per packed row.""" + positions = anchors[:, None] + torch.arange(4) + prev = draft.markov_head.markov_w1(batch["input_ids"][row, positions]) + hidden = draft.hidden.view(-1, 4, 8) + head = batch["lm_head_weight"] + logits = F.linear(hidden, head) + draft.markov_head.markov_w2(prev) + teacher = F.linear(batch["last_hidden_states"][row, positions], head) + mask = batch["loss_mask"][row, positions] * keep[:, None] + decay = torch.exp(-torch.arange(4) / 4) + labels = teacher.argmax(-1) + ce = F.cross_entropy(logits.flatten(0, 1), labels.flatten(), reduction="none").view_as(mask) + overlap = torch.minimum(logits.softmax(-1), teacher.softmax(-1)).sum(-1) + confidence = draft.confidence_head(torch.cat([hidden, prev], dim=-1)) + bce = F.binary_cross_entropy_with_logits(confidence, overlap.detach(), reduction="none") + return { + key: (value * mask * decay).sum() / (mask.sum() + eps) + for key, value, eps in ( + ("ce_loss", ce, 1e-5), + ("tv_loss", 1 - overlap, 1e-5), + ("confidence_loss", bce, 1e-8), + ) + }, mask + + +@pytest.mark.parametrize("empty_second_row", [False, True]) +def test_speculators_rows_components_and_gradients(monkeypatch, empty_second_row): + torch.manual_seed(91) + draft = TinyDraft() + reference = copy.deepcopy(draft) + # Two blocks per head chunk exercises compaction and checkpoint replay. + model = DSparkModel(draft, block_size=4, num_anchors=2, loss_recipe=RECIPE) + batch, anchors, keep = objective_fixture() + anchors = anchors.masked_fill(~keep, 0).repeat(2, 1) + keep = keep.repeat(2, 1) + keep[1, 1:] = False + if empty_second_row: + keep[1].zero_() + for key in ("input_ids", "loss_mask", "ctx_doc_ids", "last_hidden_states", "base_position_ids"): + batch[key] = torch.cat([batch[key], batch[key].clone()]) + batch["loss_mask"][1, 4] = 0 + monkeypatch.setattr( + model, + "_draft_backbone", + lambda *a, **kw: ( + draft.hidden.expand(2, -1, -1), + anchors, + keep, + anchors.shape[1], + ), + ) + # Run the tensor operations eagerly on the CPU CI lane. + monkeypatch.setattr( + "aspec.models.dspark.speculators_token_losses", speculators_token_losses.__wrapped__ + ) + actual = model(**batch) + rows = [reference_row(reference, batch, i, anchors[i], keep[i]) for i in range(2)] + expected_terms = {key: (rows[0][0][key] + rows[1][0][key]) / 2 for key in rows[0][0]} + expected = ( + 0.1 * expected_terms["ce_loss"] + + 0.9 * expected_terms["tv_loss"] + + expected_terms["confidence_loss"] + ) + torch.testing.assert_close(actual[0], expected) + assert actual[6][1].item() == 2 + assert "l1_loss" not in actual[5] + for key, value in expected_terms.items(): + numerator, denominator = actual[5][key] + torch.testing.assert_close(numerator / denominator, value) + torch.testing.assert_close(actual[4], sum(mask.sum(0) for _, mask in rows)) + assert rows[0][1][1].tolist() == [1, 1, 0, 1] # a later slot survives a supervision gap + actual[0].backward() + expected.backward() + for param, ref in zip(draft.parameters(), reference.parameters(), strict=True): + torch.testing.assert_close(param.grad, ref.grad, atol=1e-6, rtol=1e-5) + + +def test_speculators_fused_tv_subgradient_at_partial_tie(): + # A permutation keeps the softmax normalizer equal, with one tied probability. + draft = torch.tensor([[0.0, 1.0, 2.0]], requires_grad=True) + teacher = torch.tensor([[1.0, 0.0, 2.0]], requires_grad=True) + ce, tv, labels = speculators_token_losses.__wrapped__(draft, teacher) + q, p = draft.detach().softmax(-1), teacher.detach().softmax(-1) + indicator = (q <= p).float() + tv.sum().backward() + torch.testing.assert_close(draft.grad, -q * (indicator - (q * indicator).sum(-1, keepdim=True))) + assert teacher.grad is None + torch.testing.assert_close(ce, F.cross_entropy(draft, labels, reduction="none")) + + +def test_cp_shards_contribute_one_row_across_accumulation_steps(monkeypatch): + model = DSparkModel(TinyDraft(), block_size=4, loss_recipe=RECIPE) + model.draft_context_parallel_size = 2 + model.direct_cp_group = object() + coefficient = torch.tensor(1.0, requires_grad=True) + numerator = denominator = 0 + expected_rows = [] + for valid in ([1, 0, 0, 0, 1, 1, 1, 1], [0, 0, 0, 0, 1, 0, 0, 0]): + mask = torch.tensor(valid).float().view(1, 2, 4) + decay = torch.exp(-torch.arange(4) / 4) + values = torch.arange(1.0, 9.0).view_as(mask) + expected_rows.append((values * mask * decay).sum() / (mask.sum() + 1e-5)) + for rank in range(2): + local = mask[:, rank : rank + 1] + + def all_reduce(count, group): + assert group is model.direct_cp_group + count.add_(mask[:, 1 - rank].sum()) + + monkeypatch.setattr("aspec.models.dspark.dist.all_reduce", all_reduce) + weights, _, rows = model._speculators_row_weights(local, local * decay) + numerator += (coefficient * values[:, rank : rank + 1] * weights).sum() + denominator += rows + assert denominator == 2 + (numerator / denominator).backward() + torch.testing.assert_close(coefficient.grad, torch.stack(expected_rows).mean()) + + +def test_ce_only_speculators_requires_teacher(): + model = DSparkModel(TinyDraft(), l1_loss_alpha=0, confidence_head_alpha=0, loss_recipe=RECIPE) + assert model.uses_target_hidden_states + + +def test_recipe_plan_wrapper_and_exact_resume(monkeypatch): + overrides = ["training.attention_backend=flex_attention"] + base = load_config(str(CONFIG), cli_args=overrides) + config = load_config( + str(CONFIG), cli_args=[*overrides, f"training.dspark_loss_recipe={RECIPE}"] + ) + default = compile_execution_plan(base) + plan = compile_execution_plan(config) + assert default.workload.objective["loss_recipe"] == "deepspec" + assert plan.workload.objective["loss_recipe"] == RECIPE + assert plan.plan_id != default.plan_id + args = config_to_flat_args(config) + args.draft_algorithm, args.world_size = "dspark", 1 + monkeypatch.setattr( + "aspec.training.trainer.Trainer.__init__", + lambda trainer, args: setattr(trainer, "args", args), + ) + trainer = DSparkTrainer(args) + wrapper = trainer._build_training_wrapper(TinyDraft()) + assert wrapper.loss_recipe == RECIPE + assert "tv_loss" in trainer._extra_loss_component_keys + metadata = {"world_size": 1, "training_topology": _logical_training_topology(args, 1)} + _validate_exact_resume_topology(args, metadata) + legacy = copy.deepcopy(metadata) + del legacy["training_topology"]["training_objective"]["loss_recipe"] + with pytest.raises(RuntimeError, match="dspark_loss_recipe"): + _validate_exact_resume_topology(args, legacy) + args.dspark_loss_recipe = "deepspec" + _validate_exact_resume_topology(args, legacy) + with pytest.raises(RuntimeError, match="dspark_loss_recipe"): + _validate_exact_resume_topology(args, metadata) + + +@pytest.mark.parametrize( + "override", ["training.dspark_loss_recipe=typo", "training.dspark_l1_target_top_k=8"] +) +def test_invalid_recipe_config(override): + config = load_config(str(CONFIG), cli_args=[f"training.dspark_loss_recipe={RECIPE}", override]) + with pytest.raises(PlanCompileError, match="dspark_loss_recipe|unfiltered"): + compile_execution_plan(config) diff --git a/tests/cpu/test_glm5_flash_draft.py b/tests/cpu/test_glm5_flash_draft.py new file mode 100644 index 0000000..6606959 --- /dev/null +++ b/tests/cpu/test_glm5_flash_draft.py @@ -0,0 +1,125 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +import json +from pathlib import Path + +import pytest +import torch +from torch import nn + +from aspec.config.utils import load_target_config +from aspec.data.renderers.glm5 import Glm5FlashRenderer +from aspec.models.dflash import _create_dflash_mask_mod +from aspec.models.draft.dspark import DSparkConfig +from aspec.models.dspark import DSparkModel +from aspec.planning import _resolve_target_family +from aspec.training.checkpoint import load_initial_draft_weights + +ROOT = Path(__file__).resolve().parents[2] +CONFIG = ROOT / "aspec/config/dspark_glm5_3_flash_draft_config.json" + + +def test_glm_config_and_capture_coordinates(tmp_path): + raw = { + "model_type": "glm5_next", + "architectures": ["Glm5NextForConditionalGeneration"], + "text_config": { + "model_type": "glm5_next_text", + "num_hidden_layers": 45, + "hidden_size": 4096, + "vocab_size": 154880, + }, + } + (tmp_path / "config.json").write_text(json.dumps(raw)) + assert _resolve_target_family(str(tmp_path)) == "glm5_3_flash" + outer = load_target_config(str(tmp_path)) + assert getattr(outer, "text_config", outer).vocab_size == 154880 + config = DSparkConfig(**json.loads(CONFIG.read_text())) + assert [k + 1 for k in config.target_layer_ids] == [20, 28, 32, 36, 40, 44] + draft = nn.Module() + draft.config = config + wrapper = DSparkModel(draft, block_size=8) + options = wrapper._block_mask_options_for_layer(0) + assert options == {"context_window": 2048, "is_causal": True} + mask = _create_dflash_mask_mod( + torch.tensor([[2050]]), torch.tensor([[True]]), 2100, 8, **options + ) + ids = torch.tensor([1, 2, 2049, 2050, 2100, 2102, 2103]) + assert mask(0, 0, torch.tensor(2), ids).tolist() == [ + False, + True, + True, + False, + True, + True, + False, + ] + + +class Tokenizer: + roles = {"<|system|>": 7, "<|user|>": 8, "<|assistant|>": 9} + + def encode(self, text, **kwargs): + return [self.roles[text]] + + def convert_ids_to_tokens(self, token): + return next(k for k, value in self.roles.items() if value == token) + + def apply_chat_template(self, messages, **kwargs): + assert kwargs == dict(tokenize=True, add_generation_prompt=False, return_dict=False) + return [7, 1, 8, 2, 9, 3, 4, 8, 5, 9, 6] + + +def test_native_role_masks(): + renderer = Glm5FlashRenderer(Tokenizer()) + messages = [{"role": "user", "content": "prompt"}] + _, mask = renderer.render(messages, max_seq_length=20) + assert mask == [0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1] + _, mask = renderer.render(messages, max_seq_length=7, last_turn_only=True) + assert mask == [0] * 7 + with pytest.raises(ValueError, match="reserved role"): + renderer.render([{"role": "user", "content": "<|assistant|>"}], max_seq_length=20) + + +def test_speculators_published_head_binding(tmp_path): + from safetensors.torch import save_file + + config = DSparkConfig(**json.loads(CONFIG.read_text())) + draft = nn.Module() + draft.config = config + draft.embed_tokens = nn.Embedding(3, 2) + published = { + "speculators_model_type": "dspark", + "aux_hidden_state_layer_ids": [20, 28, 32, 36, 40, 44], + "mask_token_id": 154856, + "block_size": 8, + "sample_from_anchor": True, + "sliding_window_non_causal": False, + "transformer_layer_config": { + "sliding_window": 2048, + "layer_types": ["sliding_attention"] * 5, + "rope_parameters": {"rope_theta": 10000}, + "rms_norm_eps": 1e-5, + "num_attention_heads": 64, + "num_key_value_heads": 64, + "head_dim": 64, + }, + } + (tmp_path / "config.json").write_text(json.dumps(published)) + target = tmp_path / "target" + target.mkdir() + embedding = torch.randn(3, 2) + head = torch.randn(3, 2) + weights = {"embed_tokens.weight": embedding, "lm_head.weight": head} + save_file(weights, tmp_path / "model.safetensors") + save_file(weights, target / "model.safetensors") + bindings = (str(target), "embed_tokens.weight", "lm_head.weight") + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + torch.testing.assert_close(draft.embed_tokens.weight, embedding) + save_file({**weights, "lm_head.weight": -head}, target / "model.safetensors") + with pytest.raises(ValueError, match="lm_head.weight differs"): + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + published["aux_hidden_state_layer_ids"][-1] -= 1 + (tmp_path / "config.json").write_text(json.dumps(published)) + with pytest.raises(ValueError, match="capture"): + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) diff --git a/tests/cpu/test_release_recipes.py b/tests/cpu/test_release_recipes.py index 59674f6..efc032f 100644 --- a/tests/cpu/test_release_recipes.py +++ b/tests/cpu/test_release_recipes.py @@ -30,6 +30,8 @@ def test_catalog_plans(name, monkeypatch): "ASPEC_TARGET_MODEL", "Qwen/Qwen3.8-27B" if name == "qwen27b-long" else recipes.catalog()[name]["model"], ) + if name == "glm5-flash": + monkeypatch.setenv("ASPEC_TARGET_MODEL", "zai-org/GLM-5.3-Flash") if name.startswith("ling"): monkeypatch.setenv( "ASPEC_TARGET_MODEL", "inclusionAI/" + recipes.catalog()[name]["model"] @@ -83,7 +85,7 @@ def test_qwen8b_recipes_plan_without_target_override(name, monkeypatch): def test_catalog_exactly_covers_formal_files(): entries = recipes.catalog() - assert len(entries) == 13 + assert len(entries) == 14 declared = {p for entry in entries.values() for p in entry["configs"]} assert declared == { str(p.relative_to(recipes.ROOT)) for p in (recipes.ROOT / "configs/recipes").glob("*.yaml") diff --git a/tests/cpu/test_target_lm_head.py b/tests/cpu/test_target_lm_head.py index 0a11b2f..b838567 100644 --- a/tests/cpu/test_target_lm_head.py +++ b/tests/cpu/test_target_lm_head.py @@ -190,8 +190,8 @@ def _patch_dist(monkeypatch, rank: int, peer_failed: bool = False) -> _Recording def _patch_follower_config(monkeypatch) -> None: monkeypatch.setattr( target_utils, - "AutoConfig", - SimpleNamespace(from_pretrained=lambda *args, **kwargs: _multimodal_config()), + "load_target_config", + lambda *args, **kwargs: _multimodal_config(), ) diff --git a/tests/test_dspark_speculators_cutedsl.py b/tests/test_dspark_speculators_cutedsl.py new file mode 100644 index 0000000..232a786 --- /dev/null +++ b/tests/test_dspark_speculators_cutedsl.py @@ -0,0 +1,218 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Red Hat CE/TV and the Markov epilogue, including BF16 rounding.""" + +import os + +import pytest +import torch + +pytestmark = pytest.mark.skipif( + os.getenv("ASPEC_RUN_CUTEDSL_INTEGRATION") != "1" + or not torch.cuda.is_available() + or torch.cuda.get_device_capability() not in {(9, 0), (10, 0)}, + reason="requires a Hopper/Blackwell CuTeDSL integration environment", +) + + +@pytest.mark.parametrize("vocab", [257, 154880]) +@pytest.mark.parametrize("indexed", [False, True]) +def test_speculators_cute_components_and_tied_gradient(vocab, indexed): + from aspec.kernels.cutedsl.speculators_loss import speculators_cutedsl_loss + + gen = torch.Generator(device="cuda").manual_seed(6) + rows = 32 + x = torch.randn(1, rows, vocab, generator=gen, device="cuda", dtype=torch.bfloat16) + # The identical row and a permutation exercise the <= branch independently + # of random probabilities. Duplicate maxima straddle vocabulary tiles. + x[0, :, [2, vocab - 1]] = 4 + t = x.clone() + t[:, 1:, :2] = x[:, 1:, [1, 0]] + idx = None + if indexed: + idx = torch.arange(rows, device=x.device).roll(1) + idx[-1] = idx[0] + x = x.index_select(1, idx) + teacher = t if idx is None else t.index_select(1, idx) + x.requires_grad_() + ce, tv, label, correct = speculators_cutedsl_loss(x, t, idx) + q = x.detach().double().softmax(-1) + p = teacher.double().softmax(-1) + w = torch.linspace(-0.5, 0.5, rows, device=x.device).view(1, -1) + w[:, 5] = 0 + target = teacher.argmax(-1) + torch.testing.assert_close(label, target) + torch.testing.assert_close(correct, target == x.argmax(-1)) + ref_ce = x.double().logsumexp(-1) - x.double().gather(-1, target[..., None]).squeeze(-1) + torch.testing.assert_close(ce.double(), ref_ce, rtol=0, atol=2e-6) + torch.testing.assert_close(tv.double(), 1 - torch.minimum(q, p).sum(-1), rtol=0, atol=2e-6) + # The swap preserves the partition sum; compare logits for exact mathematical ties. + indicator = (x.detach() <= teacher).double() + mass = (q * indicator).sum(-1, keepdim=True) + gt = (w[..., None] * q * (mass - indicator)).bfloat16() + gc = q.clone() + gc.scatter_add_(-1, target[..., None], -torch.ones_like(target[..., None], dtype=q.dtype)) + gc = (gc * (-w[..., None])).bfloat16() + for loss, coeff, grad in ((ce, -w, gc), (tv, w, gt)): + (g,) = torch.autograd.grad(loss, x, coeff, retain_graph=True) + torch.testing.assert_close(g, grad, rtol=0.009, atol=2e-7) + (g,) = torch.autograd.grad((ce, tv), x, (-w, w)) + torch.testing.assert_close(g, gc + gt, rtol=0.012, atol=2e-7) + + +def test_markov_epilogue_rounds_before_add_and_both_inputs_receive_gradient(): + from aspec.kernels.cutedsl.speculators_loss import markov_residual_projection + + gen = torch.Generator(device="cuda").manual_seed(42) + + def sample(shape): + return torch.randn( + shape, generator=gen, dtype=torch.bfloat16, device="cuda" + ).requires_grad_() + + base, emb, weight = sample((1, 32, 8, 4096)), sample((1, 32, 8, 256)), sample((4096, 256)) + ref = base + torch.nn.functional.linear(emb, weight) + out = markov_residual_projection(base, emb, weight) + torch.testing.assert_close(out, ref, rtol=0, atol=0) + g = torch.randn(ref.shape, generator=gen, dtype=ref.dtype, device=ref.device) + xs = (base, emb, weight) + for actual, expected in zip( + torch.autograd.grad(out, xs, g), torch.autograd.grad(ref, xs, g), strict=True + ): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("context_length", [256, 262144]) +def test_markov_confidence_preserves_separate_embedding_gradient_reductions(context_length): + import copy + + from aspec.kernels.cutedsl.speculators_loss import speculators_cutedsl_loss + from aspec.models.draft.dspark import AcceptRatePredictor, VanillaMarkov + from aspec.models.dspark import DSparkModel + + torch.manual_seed(113) + draft = torch.nn.Module() + draft.markov_head = VanillaMarkov(vocab_size=1024, markov_rank=256) + draft.confidence_head = AcceptRatePredictor(32 + 256) + draft.confidence_head_with_markov = True + draft.cuda().bfloat16() + reference = copy.deepcopy(draft) + hidden = torch.randn(1, 32, 8, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True) + expected_hidden = hidden.detach().clone().requires_grad_() + teacher = torch.randn_like(hidden) + head = torch.randn(1024, 32, device="cuda", dtype=torch.bfloat16) + previous = torch.randint(4, (1, 32, 8), device="cuda") + weights = torch.rand(1, 32, 8, device="cuda") / 256 + context = teacher.new_zeros(1, context_length, 32) + context[:, :256] = teacher.view(1, 256, 32) + model = DSparkModel(draft, loss_recipe="speculators_redhat_v1") + actual = model._prediction_terms( + hidden, + previous, + previous, + head, + teacher, + torch.ones_like(weights), + weights, + teacher_context=(context, torch.arange(256, device="cuda").view(1, 32, 8)), + ) + (0.1 * actual[0] + 0.9 * actual[1] + actual[2]).backward() + + logits = torch.nn.functional.linear(expected_hidden, head) + logits = reference.markov_head.apply_block_logits(logits, token_ids=previous) + ce, tv, _, _ = speculators_cutedsl_loss(logits, torch.nn.functional.linear(teacher, head)) + embedding = reference.markov_head.get_prev_embeddings(previous) + prediction = reference.confidence_head(torch.cat([expected_hidden, embedding], dim=-1)) + target = (1 - tv).detach().to(prediction.dtype) + bce = torch.nn.functional.binary_cross_entropy_with_logits( + prediction.float(), target.float(), reduction="none" + ) + expected = tuple((value * weights).sum() for value in (ce, tv, bce)) + (0.1 * expected[0] + 0.9 * expected[1] + expected[2]).backward() + assert model.l1_kernel_invocations["cutedsl_markov_projection"] == 1 + for value, wanted in zip(actual[:3], expected, strict=True): + torch.testing.assert_close(value, wanted, rtol=0, atol=0) + torch.testing.assert_close(hidden.grad, expected_hidden.grad, rtol=0, atol=0) + for value, wanted in zip(draft.parameters(), reference.parameters(), strict=True): + torch.testing.assert_close(value.grad, wanted.grad, rtol=0, atol=0) + + +def test_pinned_speculators_fused_gpu_oracle(): + """Optional release gate: supply the unmodified b9c51bc src/speculators/losses/fused.py.""" + import hashlib + import importlib.util + import sys + from pathlib import Path + + from aspec.kernels.cutedsl.speculators_loss import speculators_cutedsl_loss + + reference = os.environ.get("ASPEC_SPECULATORS_FUSED_REFERENCE") + if not reference: + pytest.skip("ASPEC_SPECULATORS_FUSED_REFERENCE not supplied") + path = Path(reference).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == ( + "0465967844a5850facdb3f5c05c22a3f94569a180121c51e867939fa878b6a77" + ) + spec = importlib.util.spec_from_file_location("_aspec_pinned_speculators_fused", path) + lib = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = lib + spec.loader.exec_module(lib) + for vocab in (257, 154880): + gen = torch.Generator(device="cuda").manual_seed(79) + d = torch.randn(1, 32, vocab, generator=gen, device="cuda", dtype=torch.bfloat16) + t = d.roll(vocab // 2 + 1, dims=-1) + d.requires_grad_() + ce, tv, label, correct = speculators_cutedsl_loss(d, t) + torch.testing.assert_close(label, t.argmax(-1)) + torch.testing.assert_close(correct, d.argmax(-1) == label) + ref_ce, ref_tv = lib.fused_ce_loss(d, t), lib.fused_tv_loss(d, t) + torch.testing.assert_close(ce, ref_ce, rtol=0, atol=1e-6) + torch.testing.assert_close(tv, ref_tv, rtol=0, atol=3e-7) + # Compare TV alone: CE at the teacher label can drown out the wrong subgradient. + (actual,) = torch.autograd.grad(tv.sum(), d) + (expected,) = torch.autograd.grad(ref_tv.sum(), d) + torch.testing.assert_close(actual, expected, rtol=0.008, atol=1e-9) + + +@pytest.mark.parametrize("stride, last_offset, unique_rows", [(1, 0, 25), (8, -1, 0), (8, 0, 0)]) +def test_teacher_projection_policy_preserves_values_and_gradients(stride, last_offset, unique_rows): + from aspec.models.dspark import DSparkModel + + gen = torch.Generator(device="cuda").manual_seed(402) + teacher = torch.randn(1, 256, 32, generator=gen, device="cuda", dtype=torch.bfloat16) + head = torch.randn(1024, 32, generator=gen, device="cuda", dtype=torch.bfloat16) + ids = torch.randint(1024, (1, 256), generator=gen, device="cuda") + active = 4 + stride * torch.arange(18, device="cuda") + active[-1] += last_offset + anchors = torch.cat([active[:7], active.new_zeros(1), active[7:]])[None] + keep = anchors != 0 + positions = anchors[:, :, None] + torch.arange(8, device="cuda") + # Equal draft and teacher distributions exercise the TV subgradient at a tie. + hidden = teacher[0, positions].reshape(1, -1, 32).requires_grad_() + draft = torch.nn.Module() + draft.markov_head = draft.confidence_head = None + model = DSparkModel( + draft, + block_size=8, + num_anchors=19, + confidence_head_alpha=0, + loss_recipe="speculators_redhat_v1", + ) + model._draft_backbone = lambda *args, **kwargs: (hidden, anchors, keep, 19) + + def evaluate(): + outputs = model(ids, [], torch.ones_like(ids), head, last_hidden_states=teacher) + (grad,) = torch.autograd.grad(outputs[0], hidden) + return (*outputs[:5], grad) + + model._reuse_head_projections = False + reference = evaluate() + model._reuse_head_projections = True + actual = evaluate() + if torch.cuda.get_device_capability() == (10, 0): + unique_rows = 0 + assert model.loss_head_cached_context_tokens_observed == unique_rows + assert model.loss_head_cached_teacher_blocks_observed == (18 if unique_rows else 0) + assert model.l1_kernel_invocations.get("indexed_cutedsl_ce_tv", 0) == int(unique_rows > 0) + for observed, expected in zip(actual, reference, strict=True): + torch.testing.assert_close(observed, expected, rtol=0, atol=0) diff --git a/tools/recipe_catalog.json b/tools/recipe_catalog.json index 6c0873f..e496572 100644 --- a/tools/recipe_catalog.json +++ b/tools/recipe_catalog.json @@ -146,5 +146,16 @@ "target_gpus": 4, "draft_gpus": 4, "native_only_export": true + }, + "glm5-flash": { + "configs": [ + "configs/recipes/glm5_3_flash_dspark.yaml" + ], + "model": "GLM-5.3-Flash", + "nodes": 1, + "gpus": 5, + "target_gpus": 4, + "draft_gpus": 1, + "native_only_export": true } } From aa02b0e4432a1bda9f348d6bdcc1f96cd6a85dd6 Mon Sep 17 00:00:00 2001 From: zhouaihui Date: Tue, 17 Nov 2026 15:09:13 +0800 Subject: [PATCH 2/7] perf(glm): default draft training to FA4 attention Use FA4 in the GLM Flash 8K recipe while retaining explicit FlexAttention selection and independent loss recipes. Document backend consistency for reproduction and resume. Cover MHA64 sliding-window attention, causal Draft slots, sparse tile boundaries, inactive blocks and partial tails against FP32 SDPA outputs and gradients. --- configs/recipes/glm5_3_flash_dspark.yaml | 2 +- docs/user-guide/recipes.md | 6 ++- tests/test_direct_cp_gpu.py | 66 +++++++++++++++++++++++- 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/configs/recipes/glm5_3_flash_dspark.yaml b/configs/recipes/glm5_3_flash_dspark.yaml index a392b8f..b90d090 100644 --- a/configs/recipes/glm5_3_flash_dspark.yaml +++ b/configs/recipes/glm5_3_flash_dspark.yaml @@ -19,7 +19,7 @@ dataset: shuffle_dataset: true training: - attention_backend: flex_attention + attention_backend: fa4 draft_context_parallel_size: 1 micro_batch_size: 1 draft_accumulation_steps: 16 diff --git a/docs/user-guide/recipes.md b/docs/user-guide/recipes.md index 19bb32a..81c792f 100644 --- a/docs/user-guide/recipes.md +++ b/docs/user-guide/recipes.md @@ -133,7 +133,11 @@ or inference timing. ## GLM-5.3-Flash DSpark -The recipe uses the official FP8 Target on four GPUs and trains a BF16 Draft on a fifth GPU. +The recipe uses the official FP8 Target on four GPUs and trains a BF16 Draft with FA4 attention +on a fifth GPU. Set `training.attention_backend=flex_attention` to use FlexAttention instead; +this choice is independent of the DSpark loss recipe. +Keep the attention backend fixed when reproducing or resuming a run: changing it can alter +BF16 rounding and the training trajectory. Use the checkpoint's chat template with text-only `system`, `user` and `assistant` messages in disjoint conversation JSONL files. Assistant `reasoning_content` is supported. [Official Target](https://modelscope.cn/models/ZhipuAI/GLM-5.3-Flash) · [Draft configuration](../../aspec/config/dspark_glm5_3_flash_draft_config.json) diff --git a/tests/test_direct_cp_gpu.py b/tests/test_direct_cp_gpu.py index 967c507..0bbcf97 100644 --- a/tests/test_direct_cp_gpu.py +++ b/tests/test_direct_cp_gpu.py @@ -14,7 +14,7 @@ # ruff: noqa: I001 -"""Opt-in two-GPU correctness check for Direct CP collectives and FA4.""" +"""Opt-in CUDA correctness checks for Direct CP attention and collectives.""" import os @@ -31,6 +31,70 @@ from aspec.models.ops.direct_cp import DirectCPMetadata, direct_cp_attention + +@pytest.mark.parametrize("context_length", [2103, 8192]) +def test_glm_swa_fa4_matches_dense_mask_forward_and_backward(context_length): + """GLM's MHA64 window and causal Draft slots survive sparse FA4 execution.""" + if not torch.cuda.is_available(): + pytest.skip("CUDA is unavailable") + from torch.nn.attention import SDPBackend, sdpa_kernel + + from aspec.models.dflash import _create_dflash_mask_mod + + torch.manual_seed(73) + anchors = torch.cat(( + torch.arange(2040, 2072, device="cuda"), + torch.arange(context_length - 100, context_length - 68, device="cuda"), + torch.tensor([0, 1, 2047, 2048, 2050, context_length - 3, + context_length - 8, 100], device="cuda"), + ))[None] + keep = torch.ones_like(anchors, dtype=torch.bool) + keep[0, -1] = False + block_size = 8 + queries = anchors.numel() * block_size + shapes = [(1, 64, queries, 64), (1, 64, context_length, 64), + (1, 64, context_length, 64), (1, 64, queries, 64), (1, 64, queries, 64)] + actual_inputs = [torch.randn(*shape, device="cuda", dtype=torch.bfloat16, + requires_grad=True) for shape in shapes] + reference_inputs = [value.detach().float().requires_grad_() for value in actual_inputs] + metadata = DirectCPMetadata( + anchor_positions=anchors, block_keep_mask=keep, + context_position_ids=torch.arange(context_length, device="cuda")[None], + block_size=block_size, cp_rank=0, cp_size=1, group=None, + sequence_length=context_length, backend="fa4", context_window=2048, + draft_is_causal=True, + ) + actual, _ = direct_cp_attention(*actual_inputs, metadata) + mask_mod = _create_dflash_mask_mod(anchors, keep, context_length, block_size, + context_window=2048, is_causal=True) + dense_mask = mask_mod( + 0, 0, torch.arange(queries, device="cuda")[:, None], + torch.arange(context_length + queries, device="cuda")[None, :], + ) + token_keep = (keep[:, :, None] + & (anchors[:, :, None] + torch.arange(block_size, device="cuda") + < context_length)).flatten() + key_keep = torch.cat((torch.ones(context_length, device="cuda", dtype=torch.bool), + token_keep)) + dense_mask = (dense_mask & token_keep[:, None] & key_keep[None, :])[None, None] + query, context_key, context_value, draft_key, draft_value = reference_inputs + with sdpa_kernel(SDPBackend.MATH): + reference = torch.nn.functional.scaled_dot_product_attention( + query, torch.cat((context_key, draft_key), dim=2), + torch.cat((context_value, draft_value), dim=2), attn_mask=dense_mask, + ) + upstream = torch.randn_like(actual) + (actual.float() * upstream.float()).sum().backward() + (reference * upstream.float()).sum().backward() + pairs = [(actual, reference), *((value.grad, expected.grad) for value, expected + in zip(actual_inputs, reference_inputs, strict=True))] + for value, expected in pairs: + assert torch.isfinite(value).all() + relative = (value.float() - expected).norm() / expected.norm().clamp_min(1e-12) + assert relative < 0.01, float(relative) + assert torch.count_nonzero(actual[:, :, -block_size:]) == 0 + + def test_cp2_fa4_matches_unsharded_math_forward_and_backward(): if int(os.getenv("WORLD_SIZE", "1")) != 2: pytest.skip("launch with torchrun --standalone --nproc-per-node=2") From bf7ff11e548034ec0609ae4732629387399695a6 Mon Sep 17 00:00:00 2001 From: zhouaihui Date: Tue, 17 Nov 2026 16:15:05 +0800 Subject: [PATCH 3/7] fix(glm): validate published RoPE and reasoning fields Reject unsupported native reasoning aliases and invalidate renderer caches. Compare effective published RoPE policies, including scaling and YaRN defaults, before initializing a Draft. Cover independent teacher normalization with separate CE/TV forward and gradient references for indexed and expanded inputs. --- aspec/data/renderers/glm5.py | 7 +- aspec/training/checkpoint.py | 33 ++++++++- docs/user-guide/recipes.md | 3 +- tests/cpu/test_glm5_flash_draft.py | 87 +++++++++++++++++++++++- tests/test_dspark_speculators_cutedsl.py | 22 ++++-- 5 files changed, 141 insertions(+), 11 deletions(-) diff --git a/aspec/data/renderers/glm5.py b/aspec/data/renderers/glm5.py index b6718c1..27fe708 100644 --- a/aspec/data/renderers/glm5.py +++ b/aspec/data/renderers/glm5.py @@ -5,7 +5,7 @@ class Glm5FlashRenderer: - CACHE_VERSION = "v1" + CACHE_VERSION = "v2" roles = ("system", "user", "assistant") def __init__(self, tokenizer): @@ -34,6 +34,11 @@ def render( or msg.get("tool_calls") ): raise ValueError("Expected GLM system/user/assistant text messages") + for alias in ("thinking", "thinking_content", "reasoning"): + if alias in msg: + raise ValueError( + f"GLM messages must store reasoning in reasoning_content, not {alias!r}" + ) for field in ("content", "reasoning_content"): value = msg.get(field) if value is None: diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index 6fcd17f..4a43201 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -25,6 +25,7 @@ import os import time from pathlib import Path +from types import SimpleNamespace from typing import Any import torch @@ -828,6 +829,28 @@ def resolve_initial_draft_checkpoint(path: str) -> Path: return checkpoint_path +def _speculators_dspark_rope_policy(config): + from aspec.config.utils import resolve_dflash_rope_parameters + + params = resolve_dflash_rope_parameters(config) + kind = params["rope_type"] + policy = (kind, params["rope_theta"], 1.0 if kind == "default" else params["factor"]) + if kind != "yarn": + return policy + # Use the runtime's YaRN implementation to normalize optional defaults + # and compare every parameter that affects frequencies or attention scale. + from transformers import PretrainedConfig + from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS + + dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + resolved = PretrainedConfig( + rope_parameters=params, head_dim=dim, hidden_size=dim, num_attention_heads=1, + max_position_embeddings=config.max_position_embeddings, + ) + frequencies, attention_scale = ROPE_INIT_FUNCTIONS[kind](resolved, device=torch.device("cpu")) + return (*policy, tuple(frequencies.tolist()), float(attention_scale)) + + def _bind_speculators_dspark(directory, config, tensors, target_bindings): """Validate the published Draft's layout and replace its frozen head with the Target binding.""" config_path = directory / "config.json" @@ -850,9 +873,15 @@ def _bind_speculators_dspark(directory, config, tensors, target_bindings): geometry = published["transformer_layer_config"] fields = ("sliding_window", "layer_types", "rms_norm_eps", "num_attention_heads", "num_key_value_heads", "head_dim") - if (any(geometry.get(field) != getattr(config, field) for field in fields) - or geometry.get("rope_parameters", {}).get("rope_theta") != config.rope_theta): + if any(geometry.get(field) != getattr(config, field) for field in fields): raise ValueError("Speculators DSpark layer geometry differs from the configured Draft") + published_config = SimpleNamespace(**{ + "hidden_size": config.hidden_size, + "max_position_embeddings": config.max_position_embeddings, + **geometry, + }) + if _speculators_dspark_rope_policy(published_config) != _speculators_dspark_rope_policy(config): + raise ValueError("Speculators DSpark RoPE policy differs from the configured Draft") from aspec.models.draft.base import load_tensor_from_pretrained diff --git a/docs/user-guide/recipes.md b/docs/user-guide/recipes.md index 81c792f..2cf261a 100644 --- a/docs/user-guide/recipes.md +++ b/docs/user-guide/recipes.md @@ -139,7 +139,8 @@ this choice is independent of the DSpark loss recipe. Keep the attention backend fixed when reproducing or resuming a run: changing it can alter BF16 rounding and the training trajectory. Use the checkpoint's chat template with text-only `system`, `user` and `assistant` messages -in disjoint conversation JSONL files. Assistant `reasoning_content` is supported. +in disjoint conversation JSONL files. Native assistant messages must use `reasoning_content` +for reasoning text; reasoning alias fields are rejected. [Official Target](https://modelscope.cn/models/ZhipuAI/GLM-5.3-Flash) · [Draft configuration](../../aspec/config/dspark_glm5_3_flash_draft_config.json) Activate the patched Target environment and start the service. DeepGEMM compiles kernels for diff --git a/tests/cpu/test_glm5_flash_draft.py b/tests/cpu/test_glm5_flash_draft.py index 6606959..7f80fd7 100644 --- a/tests/cpu/test_glm5_flash_draft.py +++ b/tests/cpu/test_glm5_flash_draft.py @@ -1,6 +1,7 @@ # Copyright 2026 Ant Group Co., Ltd. # SPDX-License-Identifier: Apache-2.0 import json +from copy import deepcopy from pathlib import Path import pytest @@ -81,7 +82,37 @@ def test_native_role_masks(): renderer.render([{"role": "user", "content": "<|assistant|>"}], max_seq_length=20) -def test_speculators_published_head_binding(tmp_path): +@pytest.mark.parametrize("field", ["thinking", "thinking_content", "reasoning"]) +@pytest.mark.parametrize("with_canonical", [False, True]) +def test_native_reasoning_aliases_are_rejected(field, with_canonical): + renderer = Glm5FlashRenderer(Tokenizer()) + message = {"role": "assistant", "content": "answer", field: "reasoning"} + if with_canonical: + message["reasoning_content"] = "canonical reasoning" + with pytest.raises(ValueError, match="reasoning_content"): + renderer.render([message], max_seq_length=20) + + +@pytest.mark.parametrize("field", ["thinking", "thinking_content", "reasoning"]) +def test_sharegpt_reasoning_remains_canonical_before_rendering(field): + from aspec.data.preprocessing import _normalize_conversation + + class ReasoningTokenizer(Tokenizer): + def apply_chat_template(self, messages, **kwargs): + assert messages == [ + {"role": "assistant", "content": "answer", "reasoning_content": "reasoning"} + ] + return [9, 10, 11] + + messages = _normalize_conversation([{"from": "gpt", "value": "answer", field: "reasoning"}]) + renderer = Glm5FlashRenderer(ReasoningTokenizer()) + ids, mask = renderer.render(messages, max_seq_length=20) + assert ids == [9, 10, 11] + assert mask == [0, 1, 1] + + +@pytest.fixture +def published_dspark(tmp_path): from safetensors.torch import save_file config = DSparkConfig(**json.loads(CONFIG.read_text())) @@ -114,12 +145,64 @@ def test_speculators_published_head_binding(tmp_path): save_file(weights, tmp_path / "model.safetensors") save_file(weights, target / "model.safetensors") bindings = (str(target), "embed_tokens.weight", "lm_head.weight") + return draft, published, embedding, head, target, bindings + + +def test_speculators_published_head_binding(tmp_path, published_dspark): + from safetensors.torch import save_file + + draft, published, embedding, head, target, bindings = published_dspark load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) torch.testing.assert_close(draft.embed_tokens.weight, embedding) - save_file({**weights, "lm_head.weight": -head}, target / "model.safetensors") + save_file({"embed_tokens.weight": embedding, "lm_head.weight": -head}, + target / "model.safetensors") with pytest.raises(ValueError, match="lm_head.weight differs"): load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) published["aux_hidden_state_layer_ids"][-1] -= 1 (tmp_path / "config.json").write_text(json.dumps(published)) with pytest.raises(ValueError, match="capture"): load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + + +@pytest.mark.parametrize("published_rope,configured_rope", [ + ({}, {"rope_type": "linear", "factor": 2.0}), + ({"rope_type": "linear", "factor": 2.0}, {}), + ({"rope_type": "linear", "factor": 2.0}, {"rope_type": "linear", "factor": 4.0}), + ({"rope_theta": 20000}, {}), + ({"rope_type": "yarn", "factor": 2.0, "beta_fast": 64}, + {"rope_type": "yarn", "factor": 2.0, "beta_fast": 32}), +]) +def test_published_dspark_rejects_different_rope(tmp_path, published_dspark, + published_rope, configured_rope): + draft, published, _, _, _, bindings = published_dspark + geometry = published["transformer_layer_config"] + geometry["rope_parameters"] = {"rope_theta": 10000, **published_rope} + geometry["max_position_embeddings"] = draft.config.max_position_embeddings + draft.config.rope_parameters = {"rope_theta": 10000, **configured_rope} + (tmp_path / "config.json").write_text(json.dumps(published)) + with pytest.raises(ValueError, match="RoPE|geometry"): + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + + +@pytest.mark.parametrize("published_updates,configured_updates", [ + ({"rope_parameters": None}, {}), + ({"rope_theta": 10000, "rope_parameters": None}, {}), + ({"rope_parameters": {"rope_type": "default", "factor": 1.0, "rope_theta": 10000, + "partial_rotary_factor": 1.0}}, {}), + ({"rope_theta": 10000, "rope_parameters": None, + "rope_scaling": {"type": "linear", "factor": 2.0}}, + {"rope_parameters": {"rope_type": "linear", "factor": 2.0, "rope_theta": 10000}}), + ({"rope_parameters": {"rope_type": "yarn", "factor": 2.0, "rope_theta": 10000, + "beta_fast": 32, "beta_slow": 1}}, + {"rope_parameters": {"type": "yarn", "factor": 2.0, "rope_theta": 10000}}), +]) +def test_published_dspark_accepts_equivalent_rope(tmp_path, published_dspark, + published_updates, configured_updates): + draft, published, embedding, _, _, bindings = published_dspark + published["transformer_layer_config"].update(deepcopy(published_updates)) + published["transformer_layer_config"]["max_position_embeddings"] = draft.config.max_position_embeddings + for key, value in configured_updates.items(): + setattr(draft.config, key, deepcopy(value)) + (tmp_path / "config.json").write_text(json.dumps(published)) + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + torch.testing.assert_close(draft.embed_tokens.weight, embedding) diff --git a/tests/test_dspark_speculators_cutedsl.py b/tests/test_dspark_speculators_cutedsl.py index 232a786..8cd9e66 100644 --- a/tests/test_dspark_speculators_cutedsl.py +++ b/tests/test_dspark_speculators_cutedsl.py @@ -17,7 +17,8 @@ @pytest.mark.parametrize("vocab", [257, 154880]) @pytest.mark.parametrize("indexed", [False, True]) -def test_speculators_cute_components_and_tied_gradient(vocab, indexed): +@pytest.mark.parametrize("teacher_kind", ["tie", "independent"]) +def test_speculators_cute_components_and_gradient(vocab, indexed, teacher_kind): from aspec.kernels.cutedsl.speculators_loss import speculators_cutedsl_loss gen = torch.Generator(device="cuda").manual_seed(6) @@ -26,8 +27,11 @@ def test_speculators_cute_components_and_tied_gradient(vocab, indexed): # The identical row and a permutation exercise the <= branch independently # of random probabilities. Duplicate maxima straddle vocabulary tiles. x[0, :, [2, vocab - 1]] = 4 - t = x.clone() - t[:, 1:, :2] = x[:, 1:, [1, 0]] + if teacher_kind == "tie": + t = x.clone() + t[:, 1:, :2] = x[:, 1:, [1, 0]] + else: + t = torch.randn(x.shape, generator=gen, device=x.device, dtype=x.dtype) * 1.5 + 4 idx = None if indexed: idx = torch.arange(rows, device=x.device).roll(1) @@ -38,6 +42,14 @@ def test_speculators_cute_components_and_tied_gradient(vocab, indexed): ce, tv, label, correct = speculators_cutedsl_loss(x, t, idx) q = x.detach().double().softmax(-1) p = teacher.double().softmax(-1) + if teacher_kind == "independent": + student_norm = x.detach().double().logsumexp(-1) + teacher_norm = teacher.double().logsumexp(-1) + assert (teacher_norm - student_norm).abs().min() > 0.5 + wrong_teacher = (teacher.double() - student_norm[..., None]).exp() + expected_tv = 1 - torch.minimum(q, p).sum(-1) + wrong_tv = 1 - torch.minimum(q, wrong_teacher).sum(-1) + assert (expected_tv - wrong_tv).abs().min() > 1e-3 w = torch.linspace(-0.5, 0.5, rows, device=x.device).view(1, -1) w[:, 5] = 0 target = teacher.argmax(-1) @@ -46,8 +58,8 @@ def test_speculators_cute_components_and_tied_gradient(vocab, indexed): ref_ce = x.double().logsumexp(-1) - x.double().gather(-1, target[..., None]).squeeze(-1) torch.testing.assert_close(ce.double(), ref_ce, rtol=0, atol=2e-6) torch.testing.assert_close(tv.double(), 1 - torch.minimum(q, p).sum(-1), rtol=0, atol=2e-6) - # The swap preserves the partition sum; compare logits for exact mathematical ties. - indicator = (x.detach() <= teacher).double() + # Raw logits resolve exact ties only when the partition sums are identical. + indicator = ((x.detach() <= teacher) if teacher_kind == "tie" else (q <= p)).double() mass = (q * indicator).sum(-1, keepdim=True) gt = (w[..., None] * q * (mass - indicator)).bfloat16() gc = q.clone() From cb89f3e2d6b7d13dbc7660855f91475466294de1 Mon Sep 17 00:00:00 2001 From: Longxmas <1185267696@qq.com> Date: Thu, 19 Nov 2026 16:42:44 +0800 Subject: [PATCH 4/7] fix(dspark): validate exported Draft block attention --- aspec/commands/convert_to_hf.py | 6 ++++ aspec/draft_artifact.py | 3 ++ tests/cpu/test_dspark_loss_recipes.py | 7 +++-- tests/cpu/test_glm5_flash_draft.py | 43 +++++++++++++++++++++++++++ tests/test_fp8_lm_head.py | 5 ++-- 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/aspec/commands/convert_to_hf.py b/aspec/commands/convert_to_hf.py index d71b58f..3c9f4a7 100644 --- a/aspec/commands/convert_to_hf.py +++ b/aspec/commands/convert_to_hf.py @@ -261,6 +261,12 @@ def _checkpoint_training_contract(model_dir: Path, raw_config: dict) -> dict: effective = AutoDraftModelConfig.from_dict(raw_config) if draft_model_semantics(effective) != draft_model_semantics(saved_semantics): raise ValueError("Export model semantics conflict with the checkpoint; use its saved draft_config.json") + saved_block_attention = (topology.get("draft_attention_policy") or {}).get("draft_block_attention") + if algorithm in {"dspark", "dflash"} and saved_block_attention is not None: + from aspec.config.draft_attention import resolve_draft_attention + + if resolve_draft_attention(raw_config)["draft_block_attention"] != saved_block_attention: + raise ValueError("Export Draft block attention conflicts with the checkpoint; use its saved draft_config.json") from aspec.config.draft_structure import checkpoint_ffn_policy, resolve_draft_structure saved_ffn = checkpoint_ffn_policy(model_dir.parent, topology if metadata_path.is_file() else {}) diff --git a/aspec/draft_artifact.py b/aspec/draft_artifact.py index 10e5f17..30e2dd3 100644 --- a/aspec/draft_artifact.py +++ b/aspec/draft_artifact.py @@ -84,6 +84,9 @@ def validate_native_asset(path, *, expected_config=None, weights_only=False): expected_attention = resolve_draft_attention( expected_config if isinstance(expected_config, dict) else expected_config.to_dict() ) + if (manifest["algorithm"] in {"dspark", "dflash"} + and attention["draft_block_attention"] != expected_attention["draft_block_attention"]): + raise ValueError("Native artifact Draft block attention disagrees with the requested Draft") if attention.get("dsa") is not None or expected_attention.get("dsa") is not None: def dsa_semantics(policy, *, phase_transition): dsa = policy.get("dsa") diff --git a/tests/cpu/test_dspark_loss_recipes.py b/tests/cpu/test_dspark_loss_recipes.py index 48c11df..c7c0ec3 100644 --- a/tests/cpu/test_dspark_loss_recipes.py +++ b/tests/cpu/test_dspark_loss_recipes.py @@ -146,8 +146,10 @@ def test_ce_only_speculators_requires_teacher(): assert model.uses_target_hidden_states -def test_recipe_plan_wrapper_and_exact_resume(monkeypatch): - overrides = ["training.attention_backend=flex_attention"] +@pytest.mark.parametrize("precision", ["bf16", "fp8"]) +def test_recipe_plan_wrapper_and_exact_resume(monkeypatch, precision): + overrides = ["training.attention_backend=flex_attention", + f"training.dspark_lm_head_precision={precision}"] base = load_config(str(CONFIG), cli_args=overrides) config = load_config( str(CONFIG), cli_args=[*overrides, f"training.dspark_loss_recipe={RECIPE}"] @@ -156,6 +158,7 @@ def test_recipe_plan_wrapper_and_exact_resume(monkeypatch): plan = compile_execution_plan(config) assert default.workload.objective["loss_recipe"] == "deepspec" assert plan.workload.objective["loss_recipe"] == RECIPE + assert plan.workload.objective["lm_head_precision"] == precision assert plan.plan_id != default.plan_id args = config_to_flat_args(config) args.draft_algorithm, args.world_size = "dspark", 1 diff --git a/tests/cpu/test_glm5_flash_draft.py b/tests/cpu/test_glm5_flash_draft.py index 7f80fd7..441a0f3 100644 --- a/tests/cpu/test_glm5_flash_draft.py +++ b/tests/cpu/test_glm5_flash_draft.py @@ -57,6 +57,49 @@ def test_glm_config_and_capture_coordinates(tmp_path): ] +def test_native_asset_and_checkpoint_export_bind_sliding_block_causality(tmp_path): + from safetensors.torch import save_file + + from aspec.commands.convert_to_hf import _checkpoint_training_contract + from aspec.config.draft_algorithm import draft_model_semantics + from aspec.config.draft_attention import resolve_draft_attention + from aspec.draft_artifact import _sha256, validate_native_asset + + causal = json.loads(CONFIG.read_text()) + default = dict(causal) + del default["sliding_window_non_causal"] + assert resolve_draft_attention(default) == resolve_draft_attention({ + **default, "sliding_window_non_causal": True, + }) + saved_attention = resolve_draft_attention(causal) + + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(causal)) + weights_path = tmp_path / "model.safetensors" + save_file({"embed_tokens.weight": torch.zeros(1, 1)}, weights_path) + (tmp_path / "export_manifest.json").write_text(json.dumps({ + "storage_format": "aspec_native", + "algorithm": "dspark", + "config_sha256": _sha256(config_path), + "model_sha256": _sha256(weights_path), + "draft_structure": {"attention": saved_attention}, + })) + validate_native_asset(tmp_path, expected_config=causal, weights_only=True) + with pytest.raises(ValueError, match="Native artifact Draft block attention"): + validate_native_asset(tmp_path, expected_config=default, weights_only=True) + + model_dir = tmp_path / "model" + model_dir.mkdir() + (tmp_path / "meta.json").write_text(json.dumps({"training_topology": { + "draft_algorithm": "dspark", + "draft_model_semantics": draft_model_semantics(causal), + "draft_attention_policy": saved_attention, + }})) + _checkpoint_training_contract(model_dir, causal) + with pytest.raises(ValueError, match="Export Draft block attention"): + _checkpoint_training_contract(model_dir, default) + + class Tokenizer: roles = {"<|system|>": 7, "<|user|>": 8, "<|assistant|>": 9} diff --git a/tests/test_fp8_lm_head.py b/tests/test_fp8_lm_head.py index ba4151a..731d4fb 100644 --- a/tests/test_fp8_lm_head.py +++ b/tests/test_fp8_lm_head.py @@ -171,7 +171,8 @@ def test_nan_propagates_through_forward_and_input_gradient(): assert hidden.grad[:2].isfinite().all() -def test_native_dspark_losses_keep_teacher_and_validation_bf16(monkeypatch): +@pytest.mark.parametrize("loss_recipe", ["deepspec", "speculators_redhat_v1"]) +def test_native_dspark_losses_keep_teacher_and_validation_bf16(monkeypatch, loss_recipe): from types import SimpleNamespace from aspec.models.draft.dspark import AcceptRatePredictor, VanillaMarkov @@ -184,7 +185,7 @@ def test_native_dspark_losses_keep_teacher_and_validation_bf16(monkeypatch): draft.markov_head = VanillaMarkov(vocab_size=256, markov_rank=16) draft.confidence_head = AcceptRatePredictor(144) draft.confidence_head_with_markov = True - model = DSparkModel(draft).cuda().to(torch.bfloat16) + model = DSparkModel(draft, loss_recipe=loss_recipe).cuda().to(torch.bfloat16) hidden = torch.randn(1, 3, 7, 128, device="cuda", dtype=torch.bfloat16, requires_grad=True) teacher = torch.randn_like(hidden, requires_grad=False) weight = torch.randn(256, 128, device="cuda", dtype=torch.bfloat16) * 0.02 From f6a1e8b771355a92fac0b0524ebed799d9225df3 Mon Sep 17 00:00:00 2001 From: Longxmas <1185267696@qq.com> Date: Thu, 19 Nov 2026 17:14:28 +0800 Subject: [PATCH 5/7] fix(dspark): validate GLM warm-start metadata and normalize block attention --- aspec/commands/convert_to_hf.py | 7 ++- aspec/config/draft_attention.py | 9 +++ aspec/draft_artifact.py | 4 +- aspec/planning.py | 1 + aspec/training/checkpoint.py | 42 +++++++++----- aspec/training/dflash_trainer.py | 6 +- tests/cpu/test_glm5_flash_draft.py | 90 ++++++++++++++++++++++++------ tests/cpu/test_release_recipes.py | 4 +- 8 files changed, 127 insertions(+), 36 deletions(-) diff --git a/aspec/commands/convert_to_hf.py b/aspec/commands/convert_to_hf.py index 3c9f4a7..22f0011 100644 --- a/aspec/commands/convert_to_hf.py +++ b/aspec/commands/convert_to_hf.py @@ -263,9 +263,12 @@ def _checkpoint_training_contract(model_dir: Path, raw_config: dict) -> dict: raise ValueError("Export model semantics conflict with the checkpoint; use its saved draft_config.json") saved_block_attention = (topology.get("draft_attention_policy") or {}).get("draft_block_attention") if algorithm in {"dspark", "dflash"} and saved_block_attention is not None: - from aspec.config.draft_attention import resolve_draft_attention + from aspec.config.draft_attention import ( + draft_block_attention_matches, + resolve_draft_attention, + ) - if resolve_draft_attention(raw_config)["draft_block_attention"] != saved_block_attention: + if not draft_block_attention_matches(saved_block_attention, resolve_draft_attention(raw_config)): raise ValueError("Export Draft block attention conflicts with the checkpoint; use its saved draft_config.json") from aspec.config.draft_structure import checkpoint_ffn_policy, resolve_draft_structure diff --git a/aspec/config/draft_attention.py b/aspec/config/draft_attention.py index ac82ffe..21064f1 100644 --- a/aspec/config/draft_attention.py +++ b/aspec/config/draft_attention.py @@ -430,6 +430,15 @@ def dsa_positive_integer(name: str, default: int) -> int: } +def draft_block_attention_matches(saved: str | list[str], current: Mapping) -> bool: + """Compare scalar and per-layer serializations over the same layer schedule.""" + policy = current["draft_block_attention"] + layers = len(current["layer_types"]) + return ([saved] * layers if isinstance(saved, str) else saved) == ( + [policy] * layers if isinstance(policy, str) else policy + ) + + def draft_context_shard_alignment(attention: Mapping) -> int: """Derive the minimum context-shard alignment from active sparse structures.""" diff --git a/aspec/draft_artifact.py b/aspec/draft_artifact.py index 30e2dd3..88958b5 100644 --- a/aspec/draft_artifact.py +++ b/aspec/draft_artifact.py @@ -74,7 +74,7 @@ def validate_native_asset(path, *, expected_config=None, weights_only=False): raise ValueError("Native artifact MTP conditioning semantics mismatch") if expected_config is not None and draft_model_semantics(config) != draft_model_semantics(expected_config): raise ValueError("Native artifact semantics disagree with the requested Draft") - from aspec.config.draft_attention import resolve_draft_attention + from aspec.config.draft_attention import draft_block_attention_matches, resolve_draft_attention attention = resolve_draft_attention(config.to_dict()) recorded_dsa = ((manifest.get("draft_structure") or {}).get("attention") or {}).get("dsa") @@ -85,7 +85,7 @@ def validate_native_asset(path, *, expected_config=None, weights_only=False): expected_config if isinstance(expected_config, dict) else expected_config.to_dict() ) if (manifest["algorithm"] in {"dspark", "dflash"} - and attention["draft_block_attention"] != expected_attention["draft_block_attention"]): + and not draft_block_attention_matches(attention["draft_block_attention"], expected_attention)): raise ValueError("Native artifact Draft block attention disagrees with the requested Draft") if attention.get("dsa") is not None or expected_attention.get("dsa") is not None: def dsa_semantics(policy, *, phase_transition): diff --git a/aspec/planning.py b/aspec/planning.py index 0db4b58..c71b51f 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -181,6 +181,7 @@ def to_json(self) -> str: def to_runtime_args(self) -> argparse.Namespace: values = copy.deepcopy(dict(self._runtime_values)) values["aspec_plan_id"] = self.plan_id + values["aspec_target_family"] = self.workload.models.get("target", {}).get("family") values["aspec_feature_contract"] = copy.deepcopy(dict(self.workload.features)) values["aspec_locality_domains"] = copy.deepcopy( list(self.policy.resources.get("locality_domains", ())) diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index 4a43201..671952f 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -269,11 +269,19 @@ def matches_saved(value, current): saved_topology = dict(saved_topology) saved_topology["draft_model_semantics"] = draft_model_semantics(saved_topology["draft_model_semantics"]) + saved_policy = saved_topology.get("draft_attention_policy") or {} + if (current_topology.get("draft_algorithm") in {"dspark", "dflash"} + and "draft_block_attention" in saved_policy and "draft_block_attention" in current_policy): + from aspec.config.draft_attention import draft_block_attention_matches + + if draft_block_attention_matches(saved_policy["draft_block_attention"], current_policy): + saved_topology = {**saved_topology, "draft_attention_policy": { + **saved_policy, "draft_block_attention": current_policy["draft_block_attention"], + }} # Early native MTP checkpoints used the generic Draft policy label even # though their only decoder path was official causal, teacher-forced MTP. # Normalize that known serialization error at this boundary; never accept # a genuinely different attention policy or rewrite the stored checkpoint. - saved_policy = saved_topology.get("draft_attention_policy") or {} saved_semantics = saved_topology.get("draft_model_semantics") or {} objective = saved_topology.get("training_objective") or {} if (saved_topology.get("draft_algorithm") == current_topology.get("draft_algorithm") == "mtp" @@ -851,15 +859,8 @@ def _speculators_dspark_rope_policy(config): return (*policy, tuple(frequencies.tolist()), float(attention_scale)) -def _bind_speculators_dspark(directory, config, tensors, target_bindings): +def _bind_speculators_dspark(published, config, tensors, target_bindings): """Validate the published Draft's layout and replace its frozen head with the Target binding.""" - config_path = directory / "config.json" - if not config_path.is_file(): - return - published = json.loads(config_path.read_text()) - if published.get("speculators_model_type") != "dspark": - return - # Speculators captures before a layer; ASpec numbers completed layer outputs. policies = { "aux_hidden_state_layer_ids": [layer + 1 for layer in config.target_layer_ids], @@ -868,6 +869,8 @@ def _bind_speculators_dspark(directory, config, tensors, target_bindings): "sample_from_anchor": True, "sliding_window_non_causal": getattr(config, "sliding_window_non_causal", True), } + # An omitted flag means causal sliding blocks in the Speculators schema. + published = {"sliding_window_non_causal": False, **published} if any(published.get(k) != expected for k, expected in policies.items()): raise ValueError("Speculators DSpark capture, mask, block or attention policy mismatch") geometry = published["transformer_layer_config"] @@ -895,6 +898,7 @@ def _bind_speculators_dspark(directory, config, tensors, target_bindings): def load_initial_draft_weights( draft_model: torch.nn.Module, path: str, *, allow_target_embedding: bool = False, target_bindings: tuple[str, str, str] | None = None, + require_speculators_config: bool = False, ) -> Path: """Load a published draft checkpoint into a freshly constructed draft model. @@ -904,14 +908,26 @@ def load_initial_draft_weights( would otherwise leave part of the model randomly initialised for a whole run. """ checkpoint_path = resolve_initial_draft_checkpoint(path) + manifest_path = checkpoint_path.parent / "export_manifest.json" + native = manifest_path.is_file() and json.loads(manifest_path.read_text()).get("storage_format") == "aspec_native" + published = None + if target_bindings and not native: + config_path = checkpoint_path.parent / "config.json" + if config_path.is_file(): + candidate = json.loads(config_path.read_text()) + if candidate.get("speculators_model_type") == "dspark": + published = candidate + if require_speculators_config and published is None: + raise ValueError( + "GLM DSpark warm-start requires the original Speculators DSpark config.json " + "for capture policy and frozen Target binding validation" + ) with safe_open(checkpoint_path, framework="pt", device="cpu") as f: tensors = {key: f.get_tensor(key) for key in f.keys()} - manifest_path = checkpoint_path.parent / "export_manifest.json" - native = manifest_path.is_file() and json.loads(manifest_path.read_text()).get("storage_format") == "aspec_native" conditioning = getattr(draft_model.config, "mtp_conditioning", None) - if target_bindings and not native: - _bind_speculators_dspark(checkpoint_path.parent, draft_model.config, tensors, target_bindings) + if published is not None: + _bind_speculators_dspark(published, draft_model.config, tensors, target_bindings) transition = False if conditioning: saved_config_path = checkpoint_path.parent / "config.json" diff --git a/aspec/training/dflash_trainer.py b/aspec/training/dflash_trainer.py index 1c3571d..6289d5a 100644 --- a/aspec/training/dflash_trainer.py +++ b/aspec/training/dflash_trainer.py @@ -155,9 +155,13 @@ def init_model( getattr(self.args, "embedding_key", "model.embed_tokens.weight"), getattr(self.args, "lm_head_key", "lm_head.weight"), ) + is_dspark = config.model_type == "qwen3_dspark" checkpoint.load_initial_draft_weights( draft_model, initial_path, allow_target_embedding=True, - target_bindings=target_bindings if config.model_type == "qwen3_dspark" else None, + target_bindings=target_bindings if is_dspark else None, + require_speculators_config=( + is_dspark and getattr(self.args, "aspec_target_family", None) == "glm5_3_flash" + ), ) draft_model.load_embedding( target_model_path, diff --git a/tests/cpu/test_glm5_flash_draft.py b/tests/cpu/test_glm5_flash_draft.py index 441a0f3..80566ee 100644 --- a/tests/cpu/test_glm5_flash_draft.py +++ b/tests/cpu/test_glm5_flash_draft.py @@ -57,24 +57,31 @@ def test_glm_config_and_capture_coordinates(tmp_path): ] -def test_native_asset_and_checkpoint_export_bind_sliding_block_causality(tmp_path): +@pytest.mark.parametrize("layer_type", ["sliding_attention", "full_attention"]) +def test_draft_assets_and_exact_resume_compare_effective_block_attention(tmp_path, layer_type): + from types import SimpleNamespace + from safetensors.torch import save_file from aspec.commands.convert_to_hf import _checkpoint_training_contract - from aspec.config.draft_algorithm import draft_model_semantics from aspec.config.draft_attention import resolve_draft_attention from aspec.draft_artifact import _sha256, validate_native_asset + from aspec.training.checkpoint import ( + _logical_training_topology, + _validate_exact_resume_topology, + ) - causal = json.loads(CONFIG.read_text()) - default = dict(causal) + configured = json.loads(CONFIG.read_text()) + configured["layer_types"] = [layer_type] * configured["num_hidden_layers"] + default = dict(configured) del default["sliding_window_non_causal"] assert resolve_draft_attention(default) == resolve_draft_attention({ **default, "sliding_window_non_causal": True, }) - saved_attention = resolve_draft_attention(causal) + saved_attention = resolve_draft_attention(configured) config_path = tmp_path / "config.json" - config_path.write_text(json.dumps(causal)) + config_path.write_text(json.dumps(configured)) weights_path = tmp_path / "model.safetensors" save_file({"embed_tokens.weight": torch.zeros(1, 1)}, weights_path) (tmp_path / "export_manifest.json").write_text(json.dumps({ @@ -84,20 +91,30 @@ def test_native_asset_and_checkpoint_export_bind_sliding_block_causality(tmp_pat "model_sha256": _sha256(weights_path), "draft_structure": {"attention": saved_attention}, })) - validate_native_asset(tmp_path, expected_config=causal, weights_only=True) - with pytest.raises(ValueError, match="Native artifact Draft block attention"): - validate_native_asset(tmp_path, expected_config=default, weights_only=True) - model_dir = tmp_path / "model" model_dir.mkdir() - (tmp_path / "meta.json").write_text(json.dumps({"training_topology": { - "draft_algorithm": "dspark", - "draft_model_semantics": draft_model_semantics(causal), - "draft_attention_policy": saved_attention, - }})) - _checkpoint_training_contract(model_dir, causal) - with pytest.raises(ValueError, match="Export Draft block attention"): + args = SimpleNamespace( + draft_algorithm="dspark", draft_model_config_obj=configured, + dflash_block_size=configured["block_size"], world_size=1, + ) + metadata = {"world_size": 1, "training_topology": _logical_training_topology(args, 1)} + (tmp_path / "meta.json").write_text(json.dumps(metadata)) + validate_native_asset(tmp_path, expected_config=configured, weights_only=True) + _checkpoint_training_contract(model_dir, configured) + _validate_exact_resume_topology(args, metadata) + + args.draft_model_config_obj = default + if layer_type == "full_attention": + validate_native_asset(tmp_path, expected_config=default, weights_only=True) _checkpoint_training_contract(model_dir, default) + _validate_exact_resume_topology(args, metadata) + else: + with pytest.raises(ValueError, match="Native artifact Draft block attention"): + validate_native_asset(tmp_path, expected_config=default, weights_only=True) + with pytest.raises(ValueError, match="Export Draft block attention"): + _checkpoint_training_contract(model_dir, default) + with pytest.raises(RuntimeError, match="draft_attention_policy"): + _validate_exact_resume_topology(args, metadata) class Tokenizer: @@ -207,6 +224,45 @@ def test_speculators_published_head_binding(tmp_path, published_dspark): load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) +def test_published_sliding_policy_uses_speculators_defaults(tmp_path, published_dspark): + draft, published, embedding, _, _, bindings = published_dspark + config_path = tmp_path / "config.json" + del published["sliding_window_non_causal"] + config_path.write_text(json.dumps(published)) + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + torch.testing.assert_close(draft.embed_tokens.weight, embedding) + + config_path.write_text(json.dumps({**published, "sliding_window_non_causal": True})) + with pytest.raises(ValueError, match="attention policy"): + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + del published["sample_from_anchor"] + config_path.write_text(json.dumps(published)) + with pytest.raises(ValueError, match="capture"): + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + + +@pytest.mark.parametrize("config", [None, {"speculators_model_type": "dflash"}]) +def test_bound_glm_warmstart_requires_speculators_config(tmp_path, published_dspark, config): + from safetensors.torch import save_file + + draft, _, embedding, _, _, bindings = published_dspark + config_path = tmp_path / "config.json" + if config is None: + config_path.unlink() + else: + config_path.write_text(json.dumps(config)) + + weights_path = tmp_path / "model.safetensors" + weights_path.write_bytes(b"unreadable checkpoint") + with pytest.raises(ValueError, match=r"requires the original Speculators DSpark config.json"): + load_initial_draft_weights(draft, str(weights_path), target_bindings=bindings, + require_speculators_config=True) + + save_file({"embed_tokens.weight": embedding}, weights_path) + load_initial_draft_weights(draft, str(weights_path), target_bindings=bindings) + torch.testing.assert_close(draft.embed_tokens.weight, embedding) + + @pytest.mark.parametrize("published_rope,configured_rope", [ ({}, {"rope_type": "linear", "factor": 2.0}), ({"rope_type": "linear", "factor": 2.0}, {}), diff --git a/tests/cpu/test_release_recipes.py b/tests/cpu/test_release_recipes.py index efc032f..1bd7abb 100644 --- a/tests/cpu/test_release_recipes.py +++ b/tests/cpu/test_release_recipes.py @@ -51,7 +51,9 @@ def test_catalog_plans(name, monkeypatch): compile_multi_draft_plan(config) else: - compile_execution_plan(config) + plan = compile_execution_plan(config) + if name == "glm5-flash": + assert plan.to_runtime_args().aspec_target_family == "glm5_3_flash" if name != "quickstart": assert config.training.num_train_steps is None assert config.training.lr_total_steps is None From 24159b503b9b15c5e485568bc6b6a1a7373a4cda Mon Sep 17 00:00:00 2001 From: zhouaihui Date: Thu, 19 Nov 2026 19:54:01 +0800 Subject: [PATCH 6/7] fix(dspark): honor published anchor sampling defaults Resolve omitted sample_from_anchor using the pinned DSpark schema while rejecting explicit conflicts. Cover effective block visibility across DSpark and DFlash assets, exports and exact resume, and validate published metadata before tensor loading. --- aspec/training/checkpoint.py | 4 +-- tests/cpu/test_glm5_flash_draft.py | 49 +++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index 671952f..17cbab8 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -869,8 +869,8 @@ def _bind_speculators_dspark(published, config, tensors, target_bindings): "sample_from_anchor": True, "sliding_window_non_causal": getattr(config, "sliding_window_non_causal", True), } - # An omitted flag means causal sliding blocks in the Speculators schema. - published = {"sliding_window_non_causal": False, **published} + # DSpark overrides DFlash's anchor-sampling default in the pinned schema. + published = {"sliding_window_non_causal": False, "sample_from_anchor": True, **published} if any(published.get(k) != expected for k, expected in policies.items()): raise ValueError("Speculators DSpark capture, mask, block or attention policy mismatch") geometry = published["transformer_layer_config"] diff --git a/tests/cpu/test_glm5_flash_draft.py b/tests/cpu/test_glm5_flash_draft.py index 80566ee..cd12c5e 100644 --- a/tests/cpu/test_glm5_flash_draft.py +++ b/tests/cpu/test_glm5_flash_draft.py @@ -57,8 +57,18 @@ def test_glm_config_and_capture_coordinates(tmp_path): ] -@pytest.mark.parametrize("layer_type", ["sliding_attention", "full_attention"]) -def test_draft_assets_and_exact_resume_compare_effective_block_attention(tmp_path, layer_type): +@pytest.mark.parametrize("architecture,algorithm", [ + ("DSparkDraftModel", "dspark"), ("DFlashDraftModel", "dflash"), +]) +@pytest.mark.parametrize("layer_types", [ + ["full_attention"] * 5, + ["sliding_attention"] * 5, + ["sliding_attention", "full_attention", "sliding_attention", "full_attention", "full_attention"], +]) +@pytest.mark.parametrize("non_causal", [False, True]) +def test_draft_assets_and_exact_resume_compare_effective_block_attention( + tmp_path, architecture, algorithm, layer_types, non_causal, +): from types import SimpleNamespace from safetensors.torch import save_file @@ -72,12 +82,15 @@ def test_draft_assets_and_exact_resume_compare_effective_block_attention(tmp_pat ) configured = json.loads(CONFIG.read_text()) - configured["layer_types"] = [layer_type] * configured["num_hidden_layers"] + configured.update(architectures=[architecture], layer_types=layer_types, + sliding_window_non_causal=non_causal) + if algorithm == "dflash": + configured["model_type"] = "dflash" default = dict(configured) - del default["sliding_window_non_causal"] - assert resolve_draft_attention(default) == resolve_draft_attention({ - **default, "sliding_window_non_causal": True, - }) + if non_causal: + default["sliding_window_non_causal"] = False + else: + del default["sliding_window_non_causal"] saved_attention = resolve_draft_attention(configured) config_path = tmp_path / "config.json" @@ -86,7 +99,7 @@ def test_draft_assets_and_exact_resume_compare_effective_block_attention(tmp_pat save_file({"embed_tokens.weight": torch.zeros(1, 1)}, weights_path) (tmp_path / "export_manifest.json").write_text(json.dumps({ "storage_format": "aspec_native", - "algorithm": "dspark", + "algorithm": algorithm, "config_sha256": _sha256(config_path), "model_sha256": _sha256(weights_path), "draft_structure": {"attention": saved_attention}, @@ -94,7 +107,7 @@ def test_draft_assets_and_exact_resume_compare_effective_block_attention(tmp_pat model_dir = tmp_path / "model" model_dir.mkdir() args = SimpleNamespace( - draft_algorithm="dspark", draft_model_config_obj=configured, + draft_algorithm=algorithm, draft_model_config_obj=configured, dflash_block_size=configured["block_size"], world_size=1, ) metadata = {"world_size": 1, "training_topology": _logical_training_topology(args, 1)} @@ -104,7 +117,7 @@ def test_draft_assets_and_exact_resume_compare_effective_block_attention(tmp_pat _validate_exact_resume_topology(args, metadata) args.draft_model_config_obj = default - if layer_type == "full_attention": + if all(kind == "full_attention" for kind in layer_types): validate_native_asset(tmp_path, expected_config=default, weights_only=True) _checkpoint_training_contract(model_dir, default) _validate_exact_resume_topology(args, metadata) @@ -237,12 +250,17 @@ def test_published_sliding_policy_uses_speculators_defaults(tmp_path, published_ load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) del published["sample_from_anchor"] config_path.write_text(json.dumps(published)) + load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) + config_path.write_text(json.dumps({**published, "sample_from_anchor": False})) with pytest.raises(ValueError, match="capture"): load_initial_draft_weights(draft, str(tmp_path), target_bindings=bindings) @pytest.mark.parametrize("config", [None, {"speculators_model_type": "dflash"}]) -def test_bound_glm_warmstart_requires_speculators_config(tmp_path, published_dspark, config): +@pytest.mark.parametrize("bare_file", [False, True]) +def test_bound_glm_warmstart_requires_speculators_config( + tmp_path, published_dspark, config, bare_file, monkeypatch, +): from safetensors.torch import save_file draft, _, embedding, _, _, bindings = published_dspark @@ -254,9 +272,12 @@ def test_bound_glm_warmstart_requires_speculators_config(tmp_path, published_dsp weights_path = tmp_path / "model.safetensors" weights_path.write_bytes(b"unreadable checkpoint") - with pytest.raises(ValueError, match=r"requires the original Speculators DSpark config.json"): - load_initial_draft_weights(draft, str(weights_path), target_bindings=bindings, - require_speculators_config=True) + with monkeypatch.context() as context: + context.setattr("aspec.training.checkpoint.safe_open", + lambda *a, **kw: pytest.fail("Tensors opened before published config validation")) + with pytest.raises(ValueError, match=r"requires the original Speculators DSpark config.json"): + load_initial_draft_weights(draft, str(weights_path if bare_file else tmp_path), + target_bindings=bindings, require_speculators_config=True) save_file({"embed_tokens.weight": embedding}, weights_path) load_initial_draft_weights(draft, str(weights_path), target_bindings=bindings) From 07b5b1754647aa90238878ac0b337756abbfebdf Mon Sep 17 00:00:00 2001 From: zhouaihui Date: Thu, 19 Nov 2026 19:54:11 +0800 Subject: [PATCH 7/7] fix(glm): default training to native cuDNN attention Select native cuDNN MHA64 in the GLM recipe and forward the PyTorch deterministic-algorithm setting to cuDNN Frontend. Reject unsupported internal GQA determinism and document the opt-in 8K and 256K repeatability checks. --- aspec/models/ops/cudnn_flex_attention.py | 4 ++ configs/recipes/glm5_3_flash_dspark.yaml | 2 +- docs/contributing/validation.md | 16 ++++++++ docs/user-guide/recipes.md | 6 +-- tests/cpu/test_cudnn_gqa_mode.py | 40 +++++++++++++++++++ tests/cpu/test_release_recipes.py | 1 + tests/test_cudnn_deterministic_gpu.py | 49 ++++++++++++++++++++++++ 7 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 tests/test_cudnn_deterministic_gpu.py diff --git a/aspec/models/ops/cudnn_flex_attention.py b/aspec/models/ops/cudnn_flex_attention.py index 968bac8..5eb7c12 100644 --- a/aspec/models/ops/cudnn_flex_attention.py +++ b/aspec/models/ops/cudnn_flex_attention.py @@ -382,7 +382,10 @@ def cudnn_flex_attention( raise ValueError("materialized MLA currently requires Hq=Hkv") effective_mode = resolve_cudnn_gqa_mode(gqa_mode, physical_gqa=physical_gqa) + deterministic = torch.are_deterministic_algorithms_enabled() if effective_mode == "internal": + if deterministic: + raise NotImplementedError("Deterministic backward is unavailable for internal cuDNN GQA") from aspec.models.ops.cudnn_gqa_backward import internal_gqa_attention return internal_gqa_attention( @@ -423,6 +426,7 @@ def cudnn_flex_attention( value, mask_plan=plan, softmax_scale=scale, + deterministic=deterministic, return_lse=return_lse, ) if return_lse: diff --git a/configs/recipes/glm5_3_flash_dspark.yaml b/configs/recipes/glm5_3_flash_dspark.yaml index b90d090..5f403f4 100644 --- a/configs/recipes/glm5_3_flash_dspark.yaml +++ b/configs/recipes/glm5_3_flash_dspark.yaml @@ -19,7 +19,7 @@ dataset: shuffle_dataset: true training: - attention_backend: fa4 + attention_backend: cudnn_flex_attention draft_context_parallel_size: 1 micro_batch_size: 1 draft_accumulation_steps: 16 diff --git a/docs/contributing/validation.md b/docs/contributing/validation.md index 72dbc60..86fe834 100644 --- a/docs/contributing/validation.md +++ b/docs/contributing/validation.md @@ -31,6 +31,22 @@ CUDA_VISIBLE_DEVICES='' ASPEC_RUN_RAY_INTEGRATION=1 \ This verifies driver retention until reader-process fencing, including Ray's automatic exit cleanup. It does not certify unreturning CUDA/RDMA operations, forced driver loss or host partitions. +## Native cuDNN repeatability + +The native cuDNN adapter forwards `torch.use_deterministic_algorithms(True)` to the Frontend's +deterministic backward selector. A fixed RNG seed alone does not select that path. The internal +joint-GQA path rejects deterministic mode rather than silently running a nondeterministic kernel. + +On an allocated GPU in the pinned Draft environment, check repeated MHA64 outputs and Q/K/V +gradients at 8K and 256K with: + +```bash +CUBLAS_WORKSPACE_CONFIG=:4096:8 ASPEC_RUN_CUDNN_FLEX_INTEGRATION=1 \ + bash tools/run_in_env.sh python -m pytest -q tests/test_cudnn_deterministic_gpu.py +``` + +These checks establish repeatability of the native operator's outputs and gradients. + ## Bounded GPU training, recovery and native assets Activate the installed venv, use the documented environment wrapper, and select only GPUs allocated diff --git a/docs/user-guide/recipes.md b/docs/user-guide/recipes.md index 2cf261a..0f02371 100644 --- a/docs/user-guide/recipes.md +++ b/docs/user-guide/recipes.md @@ -133,9 +133,9 @@ or inference timing. ## GLM-5.3-Flash DSpark -The recipe uses the official FP8 Target on four GPUs and trains a BF16 Draft with FA4 attention -on a fifth GPU. Set `training.attention_backend=flex_attention` to use FlexAttention instead; -this choice is independent of the DSpark loss recipe. +The recipe uses the official FP8 Target on four GPUs and trains a BF16 Draft with native cuDNN +MHA64 attention (`training.attention_backend=cudnn_flex_attention`) on a fifth GPU. +Attention selection is independent of the DSpark loss recipe. Keep the attention backend fixed when reproducing or resuming a run: changing it can alter BF16 rounding and the training trajectory. Use the checkpoint's chat template with text-only `system`, `user` and `assistant` messages diff --git a/tests/cpu/test_cudnn_gqa_mode.py b/tests/cpu/test_cudnn_gqa_mode.py index 9d51afc..6686651 100644 --- a/tests/cpu/test_cudnn_gqa_mode.py +++ b/tests/cpu/test_cudnn_gqa_mode.py @@ -65,6 +65,46 @@ def test_pinned_frontend_fails_closed(monkeypatch): joint._validate_version.cache_clear() +@pytest.mark.parametrize("enabled", [False, True]) +def test_native_cudnn_propagates_deterministic_algorithms(monkeypatch, enabled): + import sys + from types import SimpleNamespace + + import torch + + from aspec.models.ops.cudnn_flex_attention import CudnnFlexMask, cudnn_flex_attention + + query = SimpleNamespace(is_cuda=True, dtype=torch.bfloat16, ndim=4, shape=(1, 8, 64, 64)) + mask = CudnnFlexMask(torch.zeros(1, 3, 8, dtype=torch.int32)) + plan, seen = object(), [] + monkeypatch.setattr(CudnnFlexMask, "plan_for", lambda *args, **kwargs: plan) + monkeypatch.setattr(torch, "are_deterministic_algorithms_enabled", lambda: enabled) + + def execute(q, k, v, **kwargs): + seen.append(kwargs) + return q + + monkeypatch.setitem(sys.modules, "cudnn.flex_attention", SimpleNamespace(flex_attn_func=execute)) + assert cudnn_flex_attention(query, query, query, mask=mask, scale=0.125) is query + assert seen == [{"mask_plan": plan, "softmax_scale": 0.125, + "deterministic": enabled, "return_lse": False}] + + +def test_internal_cudnn_determinism_fails_closed(monkeypatch): + from types import SimpleNamespace + + import torch + + from aspec.models.ops.cudnn_flex_attention import CudnnFlexMask, cudnn_flex_attention + + query = SimpleNamespace(is_cuda=True, dtype=torch.bfloat16, ndim=4, shape=(1, 8, 4, 128)) + key = SimpleNamespace(is_cuda=True, dtype=torch.bfloat16, ndim=4, shape=(1, 8, 1, 128)) + mask = CudnnFlexMask(torch.zeros(1, 3, 8, dtype=torch.int32)) + monkeypatch.setattr(torch, "are_deterministic_algorithms_enabled", lambda: True) + with pytest.raises(NotImplementedError, match="Deterministic backward"): + cudnn_flex_attention(query, key, key, mask=mask, scale=0.125, gqa_mode="internal") + + @pytest.mark.parametrize("mode", ["native", "external", "internal"]) @pytest.mark.parametrize("backend", ["cudnn_flex_attention", "math"]) def test_direct_cp_propagates_explicit_route_and_observation(monkeypatch, mode, backend): diff --git a/tests/cpu/test_release_recipes.py b/tests/cpu/test_release_recipes.py index 1bd7abb..0d415f3 100644 --- a/tests/cpu/test_release_recipes.py +++ b/tests/cpu/test_release_recipes.py @@ -54,6 +54,7 @@ def test_catalog_plans(name, monkeypatch): plan = compile_execution_plan(config) if name == "glm5-flash": assert plan.to_runtime_args().aspec_target_family == "glm5_3_flash" + assert plan.to_runtime_args().attention_backend == "cudnn_flex_attention" if name != "quickstart": assert config.training.num_train_steps is None assert config.training.lr_total_steps is None diff --git a/tests/test_cudnn_deterministic_gpu.py b/tests/test_cudnn_deterministic_gpu.py new file mode 100644 index 0000000..b0e9850 --- /dev/null +++ b/tests/test_cudnn_deterministic_gpu.py @@ -0,0 +1,49 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Native cuDNN MHA64 forward and backward repeatability.""" + +import os + +import pytest +import torch + +pytestmark = pytest.mark.skipif( + os.getenv("ASPEC_RUN_CUDNN_FLEX_INTEGRATION") != "1" or not torch.cuda.is_available(), + reason="requires the native CUDA cuDNN integration environment", +) + + +@pytest.mark.parametrize("context_length", [8192, 262144]) +def test_native_mha64_deterministic_forward_and_backward(context_length): + from aspec.models.ops.cudnn_flex_attention import build_dspark_mla_mask, cudnn_flex_attention + + generator = torch.Generator(device="cuda").manual_seed(93) + queries, heads, dim = 16, 4, 64 + anchors = torch.tensor([[7, context_length - 9]], device="cuda") + mask = build_dspark_mla_mask( + anchor_positions=anchors, block_keep_mask=torch.ones_like(anchors, dtype=torch.bool), + context_length=context_length, block_size=8, is_causal=True, context_window=2048, + ) + inputs = [torch.randn(1, length, heads, dim, device="cuda", dtype=torch.bfloat16, + generator=generator) for length in (queries, context_length + queries, + context_length + queries)] + gradient = torch.randn(inputs[0].shape, device="cuda", dtype=torch.bfloat16, + generator=generator) + enabled = torch.are_deterministic_algorithms_enabled() + warn_only = torch.is_deterministic_algorithms_warn_only_enabled() + try: + torch.use_deterministic_algorithms(True) + with torch.backends.cudnn.flags(deterministic=True, benchmark=False): + repeats = [] + for _ in range(2): + q, k, v = [value.detach().clone().requires_grad_() for value in inputs] + output = cudnn_flex_attention(q, k, v, mask=mask, scale=dim ** -0.5, + gqa_mode="native") + grads = torch.autograd.grad(output, (q, k, v), gradient) + assert torch.isfinite(output).all() + assert all(torch.isfinite(value).all() for value in grads) + repeats.append((output.detach(), *grads)) + for first, second in zip(*repeats, strict=True): + assert torch.equal(first, second) + finally: + torch.use_deterministic_algorithms(enabled, warn_only=warn_only)