feat(maple): BF16 KV tier, q4k head, head overlays, and six measured default fixes

crates/hipfire-arch-gemma4/map.md

@@ -31,7 +31,7 @@ _Generated by `scripts/check-crate-maps.py` from the tree — do not edit inside
 | [`src/gemma4.rs`](src/gemma4.rs) | 1,088 | 13 | 0 |
 | [`src/gemma4_vision.rs`](src/gemma4_vision.rs) | 16 | 3 | 0 |
 | [`src/lib.rs`](src/lib.rs) | 48 | 8 | 0 |
-| [`src/lowered.rs`](src/lowered.rs) | 5,876 | 35 | 0 |
+| [`src/lowered.rs`](src/lowered.rs) | 5,883 | 35 | 0 |
 | [`src/speculative.rs`](src/speculative.rs) | 252 | 6 | 0 |
 
 ### Public API surface
@@ -60,6 +60,6 @@ _Generated by `scripts/check-crate-maps.py` from the tree — do not edit inside
 
 ### Totals
 
-- 10 modules · 12,179 lines · 111 public items · 18 tests · 8 examples
+- 10 modules · 12,186 lines · 111 public items · 18 tests · 8 examples
 
 <!-- crate-map:generated:end -->

crates/hipfire-arch-gemma4/src/lowered.rs

@@ -3038,6 +3038,7 @@ fn sliding_layer_decode_impl(
             quant_q4: false,
             quant_int8: false,
             quant_hfq8: false,
+            quant_bf16: false,
             f32_policy: hipfire_dispatch::families::kv_tier::F32AttnPolicy::Simple,
             v_mode_bits: kv_cache.v_mode_bits(),
             pos,
@@ -3449,6 +3450,7 @@ fn full_layer_decode_impl(
             quant_q4: false,
             quant_int8: false,
             quant_hfq8: false,
+            quant_bf16: false,
             f32_policy: hipfire_dispatch::families::kv_tier::F32AttnPolicy::Simple,
             v_mode_bits: kv_cache.v_mode_bits(),
             pos,
@@ -4244,6 +4246,7 @@ fn forward_prefill_batch_v2(
                         quant_q4: false,
                         quant_int8: false,
                         quant_hfq8: false,
+                        quant_bf16: false,
                         f32_policy: hipfire_dispatch::families::kv_tier::F32AttnPolicy::Simple,
                         v_mode_bits: kv_sliding.v_mode_bits(),
                         pos,
@@ -4576,6 +4579,7 @@ fn forward_prefill_batch_v2(
                     quant_q4: false,
                     quant_int8: false,
                     quant_hfq8: false,
+                    quant_bf16: false,
                     f32_policy: hipfire_dispatch::families::kv_tier::F32AttnPolicy::Simple,
                     v_mode_bits: kv_full.v_mode_bits(),
                     pos: start_pos + n_batch - 1,
@@ -4655,6 +4659,7 @@ fn forward_prefill_batch_v2(
                             quant_q4: false,
                             quant_int8: false,
                             quant_hfq8: false,
+                            quant_bf16: false,
                             f32_policy: hipfire_dispatch::families::kv_tier::F32AttnPolicy::Simple,
                             v_mode_bits: kv_full.v_mode_bits(),
                             pos,
@@ -5562,6 +5567,7 @@ impl<'a> ForwardBindings for Gemma4Bindings<'a> {
                     quant_q4: false,
                     quant_int8: false,
                     quant_hfq8: false,
+                    quant_bf16: false,
                     f32_policy: hipfire_dispatch::families::kv_tier::F32AttnPolicy::Simple,
                     v_mode_bits: kv.v_mode_bits(),
                     pos,
@@ -5677,6 +5683,7 @@ impl<'a> ForwardBindings for Gemma4Bindings<'a> {
                     quant_q4: false,
                     quant_int8: false,
                     quant_hfq8: false,
+                    quant_bf16: false,
                     f32_policy: hipfire_dispatch::families::kv_tier::F32AttnPolicy::Simple,
                     v_mode_bits: kv.v_mode_bits(),
                     pos,

crates/hipfire-arch-maple/examples/maple_coherence.rs

@@ -12,7 +12,14 @@
 //! produce a model that loads, runs at full speed, and emits garbage.
 //!
 //! Usage:
-//!   maple_coherence --model <model.hfq> [--prompt "..."] [--max-tokens N] [--raw]
+//!   maple_coherence --model <model.hfq> [--prompt "..."] [--max-tokens N]
+//!                   [--raw] [--kv-mode q8|bf16]
+//!                   [--temp T] [--top-p P] [--seed N]
+//!                   [--head <head-only.hfq>]
+//!
+//! `--kv-mode bf16` swaps the Q8_0 KV cache for the flat BF16 tier. Both run
+//! the same sliding-window kernels with the same dim mapping and FMA order, so
+//! a q8-vs-bf16 diff isolates KV storage precision from everything else.
 //!
 //! `HIPFIRE_MAPLE_PER_TOKEN_PREFILL=1` forces the per-token prefill path, so the
 //! batched path can be A/B'd against it from one binary on one machine.
@@ -22,7 +29,7 @@
 //! against the HF reference is a separate follow-up; it needs a capture hook
 //! inside `decode_step_body`, which this harness deliberately does not have.
 
-use hipfire_arch_maple::bundle::load_maple_from_hfq;
+use hipfire_arch_maple::bundle::load_maple_from_hfq_with_head;
 use hipfire_arch_maple::forward::decode_step;
 use hipfire_runtime::hfq::HfqFile;
 use std::path::Path;
@@ -32,6 +39,84 @@ struct Args {
     prompt: String,
     max_tokens: usize,
     raw: bool,
+    kv_mode: String,
+    /// Optional head-overlay `.hfq` (hipfire-quantize --head-only).
+    head: Option<String>,
+    /// 0.0 = greedy (default, unchanged behaviour). > 0 = sample.
+    temp: f32,
+    top_p: f32,
+    seed: u64,
+}
+
+/// SplitMix64 — a 64-bit mixer used here as the sampling RNG.
+///
+/// Deliberately self-contained and NOT the engine's sampler: this harness needs
+/// a stream that depends only on `--seed`, so two runs of the same arm are
+/// reproducible and two different seeds are genuinely independent draws. It is
+/// not trying to match production sampling numerics.
+struct SplitMix64(u64);
+impl SplitMix64 {
+    fn new(seed: u64) -> Self {
+        Self(seed.wrapping_add(0x9E3779B97F4A7C15))
+    }
+    fn next_u64(&mut self) -> u64 {
+        self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15);
+        let mut z = self.0;
+        z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9);
+        z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB);
+        z ^ (z >> 31)
+    }
+    /// Uniform in [0, 1). 53 bits of mantissa, so the quantisation is far finer
+    /// than any probability this is used to compare against.
+    fn next_f64(&mut self) -> f64 {
+        (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64
+    }
+}
+
+/// Temperature + top-p (nucleus) sampling.
+///
+/// Softmax is computed in f64 max-shifted so the exponentials cannot overflow;
+/// the vocab is 151,936 wide and the raw logit range is large enough that the
+/// naive form does overflow in f32.
+fn sample_top_p(logits: &[f32], temp: f32, top_p: f32, rng: &mut SplitMix64) -> u32 {
+    let mut idx: Vec<u32> = (0..logits.len() as u32).collect();
+    let max = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max) as f64;
+    let t = temp.max(1e-6) as f64;
+    let mut p: Vec<f64> = logits
+        .iter()
+        .map(|&v| ((v as f64 - max) / t).exp())
+        .collect();
+    let sum: f64 = p.iter().sum();
+    for v in p.iter_mut() {
+        *v /= sum;
+    }
+    // Descending by probability, then keep the smallest prefix whose mass
+    // reaches top_p. The prefix always keeps at least one token, so a
+    // degenerate top_p cannot produce an empty nucleus.
+    idx.sort_unstable_by(|&a, &b| {
+        p[b as usize]
+            .partial_cmp(&p[a as usize])
+            .unwrap_or(std::cmp::Ordering::Equal)
+    });
+    let mut cum = 0.0;
+    let mut cut = idx.len();
+    for (n, &i) in idx.iter().enumerate() {
+        cum += p[i as usize];
+        if cum >= top_p as f64 {
+            cut = n + 1;
+            break;
+        }
+    }
+    let nucleus = &idx[..cut.max(1)];
+    let mass: f64 = nucleus.iter().map(|&i| p[i as usize]).sum();
+    let mut r = rng.next_f64() * mass;
+    for &i in nucleus {
+        r -= p[i as usize];
+        if r <= 0.0 {
+            return i;
+        }
+    }
+    nucleus[nucleus.len() - 1]
 }
 
 fn parse_args() -> Args {
@@ -40,6 +125,12 @@ fn parse_args() -> Args {
     let mut prompt = "The capital of France is".to_string();
     let mut max_tokens = 64usize;
     let mut raw = false;
+    // "" = MAPLE_POLICY's default (bf16). "q8" selects the block-quantized tier.
+    let mut kv_mode = String::new();
+    let mut temp = 0.0f32;
+    let mut top_p = 0.95f32;
+    let mut seed = 0u64;
+    let mut head: Option<String> = None;
     let mut i = 1;
     while i < argv.len() {
         match argv[i].as_str() {
@@ -61,6 +152,30 @@ fn parse_args() -> Args {
                 raw = true;
                 i += 1;
             }
+            // KV storage tier: "q8" (default) or "bf16". Anything else warns
+            // and falls back to q8 via MAPLE_POLICY.
+            "--kv-mode" => {
+                kv_mode = argv[i + 1].clone();
+                i += 2;
+            }
+            "--temp" => {
+                temp = argv[i + 1].parse().expect("--temp");
+                i += 2;
+            }
+            "--top-p" => {
+                top_p = argv[i + 1].parse().expect("--top-p");
+                i += 2;
+            }
+            "--seed" => {
+                seed = argv[i + 1].parse().expect("--seed");
+                i += 2;
+            }
+            // Swap the lm_head without a second full model: point at a
+            // single-tensor .hfq from `hipfire-quantize --head-only`.
+            "--head" => {
+                head = Some(argv[i + 1].clone());
+                i += 2;
+            }
             other => panic!("unknown arg {other}"),
         }
     }
@@ -69,6 +184,11 @@ fn parse_args() -> Args {
         prompt,
         max_tokens,
         raw,
+        kv_mode,
+        temp,
+        top_p,
+        seed,
+        head,
     }
 }
 
@@ -93,14 +213,32 @@ fn main() {
         args.prompt.clone()
     } else {
         format!(
-            "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
+            // The trailing "<think>\n" is REQUIRED and was missing until
+            // 2026-08-31. Maple's embedded jinja template ends its generation
+            // prompt with `'<|im_start|>assistant\n<think>\n'`, and the vendor's
+            // llama.cpp README calls out `--jinja` as applying the template
+            // "exactly, including its thinking prefix".
+            //
+            // Without it the model has to emit the opening <think> itself, so
+            // every generation starts off-distribution INSIDE the reasoning
+            // block — which is exactly where this model's degenerate loops
+            // occur. Any loop-rate measurement taken without this prefix is
+            // measuring a prompt frame the model was never trained on.
+            "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n<think>\n",
             args.prompt
         )
     };
     let prompt_toks = tokenizer.encode(&text);
 
     let max_seq = prompt_toks.len() + args.max_tokens + 64;
-    let mut b = load_maple_from_hfq(&mut hfq, &mut gpu, max_seq).expect("load maple bundle");
+    let mut b = load_maple_from_hfq_with_head(
+        &mut hfq,
+        &mut gpu,
+        max_seq,
+        &args.kv_mode,
+        args.head.as_deref().map(std::path::Path::new),
+    )
+    .expect("load maple bundle");
     eprintln!(
         "maple: hidden={} layers={} experts={}/{} moe_inter={} vocab={} eos={} max_seq={}",
         b.config.hidden_size,
@@ -157,12 +295,19 @@ fn main() {
         prompt_toks.len() as f64 / prefill_s
     );
 
-    // Greedy decode.
+    // Decode. `--temp 0` (the default) is greedy and bit-for-bit reproduces the
+    // previous behaviour; `--temp > 0` samples with top-p and an EXPLICIT seed.
+    //
+    // The seed is what makes a loop-rate measurement possible at all: greedy
+    // gives exactly ONE draw per (prompt, model), so sample size can only grow
+    // with the prompt set and prompt dominates the variance. With a seed, the
+    // same prompt can be redrawn N times and the arms compared on equal terms.
     let mut out = String::new();
     let t1 = std::time::Instant::now();
     let mut n_gen = 0usize;
+    let mut rng = SplitMix64::new(args.seed);
     for _ in 0..args.max_tokens {
-        let (best, _) =
+        let tok = if args.temp <= 0.0 {
             logits
                 .iter()
                 .enumerate()
@@ -172,8 +317,11 @@ fn main() {
                     } else {
                         acc
                     }
-                });
-        let tok = best as u32;
+                })
+                .0 as u32
+        } else {
+            sample_top_p(&logits, args.temp, args.top_p, &mut rng)
+        };
         if tok == b.eos_tok {
             eprintln!("[eos]");
             break;

crates/hipfire-arch-maple/examples/maple_decode_profile.rs

@@ -288,7 +288,7 @@ fn main() {
     // from the headline token counts alone silently under-allocates and the
     // first symptom is an illegal-access fault in an unrelated kernel.
     let max_seq = prompt_toks.len() + args.warmup + args.gen + args.profile_gen + 160 + 64;
-    let mut b = load_maple_from_hfq(&mut hfq, &mut gpu, max_seq).expect("load maple bundle");
+    let mut b = load_maple_from_hfq(&mut hfq, &mut gpu, max_seq, "").expect("load maple bundle");
     eprintln!(
         "maple: hidden={} layers={} experts={}/{} moe_inter={} vocab={} max_seq={}",
         b.config.hidden_size,

crates/hipfire-arch-maple/examples/maple_kld.rs

@@ -54,7 +54,8 @@ use std::path::Path;
 use std::time::Instant;
 
 const USAGE: &str = "usage: maple_kld --model <model.hfq> --tokens <tokens.json> \
-                     --ref <ref_logits.bin> [--dump <out.bin>] [--per-pos <out.csv>] [--limit N]";
+                     --ref <ref_logits.bin> [--dump <out.bin>] [--per-pos <out.csv>] [--limit N] \\
+                     [--kv-mode q8|bf16]";
 
 struct Args {
     model: String,
@@ -63,6 +64,11 @@ struct Args {
     dump: Option<String>,
     per_pos: Option<String>,
     limit: Option<usize>,
+    /// KV storage tier request, resolved through MAPLE_POLICY. "" = q8.
+    /// This is what lets the Q8-KV contribution to the measured KL be
+    /// SUBTRACTED rather than assumed: run the same tokens and the same
+    /// reference under q8 and bf16 and diff the results.
+    kv_mode: String,
 }
 
 fn parse_args() -> Args {
@@ -74,6 +80,7 @@ fn parse_args() -> Args {
         dump: None,
         per_pos: None,
         limit: None,
+        kv_mode: String::new(),
     };
     let mut i = 1;
     while i < argv.len() {
@@ -90,6 +97,7 @@ fn parse_args() -> Args {
             "--dump" => a.dump = Some(val()),
             "--per-pos" => a.per_pos = Some(val()),
             "--limit" => a.limit = Some(val().parse().expect("--limit")),
+            "--kv-mode" => a.kv_mode = val(),
             other => panic!("unknown arg {other}\n{USAGE}"),
         }
         i += 2;
@@ -187,7 +195,8 @@ fn main() {
 
     eprintln!("Loading weights from {}...", args.model);
     let t_load = Instant::now();
-    let mut b = load_maple_from_hfq(&mut hfq, &mut gpu, n).expect("load maple bundle");
+    let mut b =
+        load_maple_from_hfq(&mut hfq, &mut gpu, n, &args.kv_mode).expect("load maple bundle");
     eprintln!("Loaded in {:.1}s", t_load.elapsed().as_secs_f64());
     eprintln!(
         "maple: hidden={} layers={} experts={}/{} vocab={}",

crates/hipfire-arch-maple/examples/maple_perplexity.rs

@@ -128,7 +128,7 @@ fn main() {
 
     eprintln!("Loading weights from {}...", args.model);
     let t_load = Instant::now();
-    let mut b = load_maple_from_hfq(&mut hfq, &mut gpu, args.ctx).expect("load maple bundle");
+    let mut b = load_maple_from_hfq(&mut hfq, &mut gpu, args.ctx, "").expect("load maple bundle");
     eprintln!("Loaded in {:.1}s", t_load.elapsed().as_secs_f64());
     eprintln!(
         "maple: hidden={} layers={} experts={}/{} vocab={}",

crates/hipfire-arch-maple/examples/maple_prefill_parity.rs

@@ -371,7 +371,7 @@ fn main() {
     // default sweep and (b) LEAK every copy but the last: `GpuTensor` has no
     // `Drop` and this crate frees explicitly via `free_gpu`, so a dropped
     // bundle's device memory is simply gone until the process exits.
-    let mut bundle = load_maple_from_hfq(&mut hfq, &mut gpu, n_tokens + 64).expect("load");
+    let mut bundle = load_maple_from_hfq(&mut hfq, &mut gpu, n_tokens + 64, "").expect("load");
     let mut want = Vec::with_capacity(n_tokens);
     for (p, &t) in tokens.iter().enumerate() {
         want.push(