From 66b957e2fc2808db686a37def6aafdfa329877c0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 5 Sep 2026 19:53:36 -0700 Subject: [PATCH] convert: support compressed-tensors mixed-precision NVFP4 checkpoints Every NVFP4 checkpoint published by Unsloth (unsloth/Qwen3.6-35B-A3B-NVFP4, unsloth/Qwen3.6-35B-A3B-NVFP4-Fast, unsloth/Qwen3.6-27B-NVFP4) uses the compressed-tensors "mixed-precision" format with two config groups: one float-quantized FP8 group covering the attention projections and lm_head, and one nvfp4-pack-quantized group covering the MoE experts. The converter rejected all of them with NotImplementedError: Can't handle multiple config groups for compressed-tensors yet because the nvfp4_compressed_tensors gate required every group to be nvfp4-pack-quantized. Relax both copies of that gate to accept a checkpoint in which any group is NVFP4, and handle the rest of the checkpoint: - _generate_nvfp4_tensors now identifies NVFP4 tensors by dtype and block geometry rather than by scale rank alone. The FP8 group also carries a 2D weight_scale of shape [out, 1], so the existing "scale.ndim < 2" test let FP8 tensors fall into the NVFP4 repacking path. - The nvfp4 branch of dequant_model now dequantizes the leftover FP8 weights the same way the float-quantized branch does, and drops the unused input_scale, k_scale and v_scale sidecars. Previously it did nothing, so those tensors reached the writer still quantized. With this, converting unsloth/Qwen3.6-35B-A3B-NVFP4-Fast with --fp8-as-q8 produces a MOSTLY_NVFP4 GGUF whose 240 expert tensors are GGML_TYPE_NVFP4 and whose attention and lm_head tensors are Q8_0. --- conversion/base.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/conversion/base.py b/conversion/base.py index 56547ace009..81b03ecfc28 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -489,7 +489,7 @@ def dequant_packed(w: Tensor, scale: Tensor, shape_tensor: Tensor, zero_point: T quant_format == "nvfp4-pack-quantized" or quant_format == "mixed-precision" and bool(groups) - and all(g.get("format") == "nvfp4-pack-quantized" for g in groups.values() if isinstance(g, dict)) + and any(g.get("format") == "nvfp4-pack-quantized" for g in groups.values() if isinstance(g, dict)) ) if len(groups) > 1 and not nvfp4_compressed_tensors: @@ -538,8 +538,27 @@ def dequant_packed(w: Tensor, scale: Tensor, shape_tensor: Tensor, zero_point: T if (base_name + "_zero_point") in self.model_tensors: tensors_to_remove.append(base_name + "_zero_point") elif nvfp4_compressed_tensors: - # Don't error from compressed-tensors, we'll handle them in _generate_nvfp4_tensors - pass + # NVFP4 tensors were already repacked by _generate_nvfp4_tensors and removed + # from model_tensors. For a "mixed-precision" checkpoint whatever weight_scale + # entries are left belong to the non-NVFP4 config group (FP8 per-channel); + # dequantize them exactly like the float-quantized branch above. + for name in self.model_tensors.keys(): + if name.endswith(".weight_scale"): + weight_name = name.removesuffix("_scale") + if weight_name not in self.model_tensors: + tensors_to_remove.append(name) + continue + w = self.model_tensors[weight_name] + s = self.model_tensors[name] + is_fp8 = False + if self._fp8_as_q8: + is_fp8 = w().dtype in (torch.float8_e4m3fn, torch.float8_e5m2) + self.model_tensors[weight_name] = lambda w=w, s=s: dequant_simple(w(), s(), None) + tensors_to_remove.append(name) + if is_fp8: + self._fp8_dequantized.add(weight_name) + elif name.endswith((".input_scale", ".k_scale", ".v_scale", ".weight_scale_2")): + tensors_to_remove.append(name) else: raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported") elif quant_method == "modelopt": @@ -751,9 +770,16 @@ def _generate_nvfp4_tensors(self): weight = LazyTorchTensor.to_eager(self.model_tensors[name]()) scale = LazyTorchTensor.to_eager(self.model_tensors[scale_name]()) - # Skip non-NVFP4 tensors (e.g. FP8 with per-channel 1D scales) + # Skip non-NVFP4 tensors (e.g. FP8 with per-channel 1D scales). + # In a compressed-tensors "mixed-precision" checkpoint the FP8 group also has a + # 2D weight_scale of shape [out, 1], so shape alone is not enough: an NVFP4 + # tensor is nibble-packed uint8 with an E4M3 scale, one per 16 values. if scale.ndim < 2: continue + if weight.dtype != torch.uint8 or scale.dtype != torch.float8_e4m3fn: + continue + if scale.shape[-1] * 16 != weight.shape[-1] * 2: + continue scale2 = LazyTorchTensor.to_eager(self.model_tensors.get(scale2_name, lambda: torch.tensor(1.0))()) input_scale = LazyTorchTensor.to_eager(self.model_tensors.get(input_scale_name, lambda: torch.tensor(1.0))()) @@ -858,7 +884,7 @@ def prepare_tensors(self): quant_format == "nvfp4-pack-quantized" or quant_format == "mixed-precision" and bool(quant_groups) - and all(g.get("format") == "nvfp4-pack-quantized" for g in quant_groups.values() if isinstance(g, dict)) + and any(g.get("format") == "nvfp4-pack-quantized" for g in quant_groups.values() if isinstance(g, dict)) ) if quant_algo != "NVFP4": if nvfp4_compressed_tensors: