DeepSeek-V4.1: prefill segments, CED and long prompts

RUNBOOK.md

@@ -168,6 +168,12 @@ The server logs `heat_wait_s`, a cancel ends the wait, and the reply's tokens do
 On three GB10s serving GLM-5.3-Flash NVFP4 at TP=3, a 1,035,295-token prompt without the bands kept one host's hottest
 zone at 94 C or more for most of the prompt (97.7 C at the top); `TF_HEAT_HIGH=92` and `TF_HEAT_LOW=88` held all three
 at 93.0 C or under, and the first token came 10% later.
+`TF_GRAPH_CACHE=0` (or `off`, any case) runs every step eagerly in any family that installs the keyed CUDA graph cache,
+without a rebuild. Set it alike on every tensor-parallel rank: an off rank never joins the cache's per-miss vote, so a
+rank left on would wait at its first miss, and only the off rank logs the switch. Ranks that share a cache compare
+`Settings.digest` of their effective settings at start, before building it, and refuse to start when they differ.
+A tensor-parallel family passes its cache settings in the TP config; the bootstrap then compares them and refuses to
+start on every rank, naming TF_GRAPH_CACHE among the settings that must match.
 Archives whose names contain `host-only` are CPU verification artifacts and cannot serve CUDA inference.
 
 ## Flash Next and paired Metal serving

THIRD_PARTY_NOTICES.md

@@ -183,6 +183,12 @@ the one-shot all-gather protocol of [b12x](https://github.com/local-inference-la
 sequence parity and a timeout that fails the transport. Copyright 2026 Luke Alonso and the b12x contributors,
 Apache License 2.0 (see [the license text](LICENSES/Apache-2.0.txt)); b12x ships no NOTICE file.
 
+The Zig tensor-parallel fabric's RoCE path (`zig/src/fabric/tp/mailbox.cu`, `roce.zig`, `roce_verbs.zig`) adapts
+the one-shot all-gather protocol of [b12x](https://github.com/local-inference-lab/b12x)'s RoCEnante
+(`b12x/comm/roce/`, PRs #295 and #315): a device-resident epoch, sequence flags after each payload, two slots by
+sequence parity and a timeout that fails the transport. Copyright 2026 Luke Alonso and the b12x contributors,
+Apache License 2.0 (see [the license text](LICENSES/Apache-2.0.txt)); b12x ships no NOTICE file.
+
 ## Vendored code and weights
 
 In the Python engine, frozen at 0.6.6, `src/tensorfold/drafters/vendor/z_lab_dflash/model_mlx.py` is the unmodified

build.zig

@@ -19,6 +19,10 @@ const golden_build = @import("zig/build/golden.zig");
 const contract_build = @import("zig/build/kernel_contract.zig");
 const video_build = @import("zig/build/video.zig");
 const sys_build = @import("zig/build/sys.zig");
+const tp_build = @import("zig/build/tp.zig");
+const dsv41_build = @import("zig/build/dsv41.zig");
+const sessions_build = @import("zig/build/sessions.zig");
+const kv_build = @import("zig/build/kv.zig");
 
 comptime {
     const required = std.mem.trim(u8, @embedFile(".zig-version"), "\r\n");
@@ -202,6 +206,13 @@ pub fn build(b: *std.Build) void {
     b.step("test-learn-warning", "CPU production-log refusal, backoff and recovery test").dependOn(&warning_run.step);
     grammar_build.steps(b, target, xgrammar);
     contract_build.hostTests(b, gemma4_contract_host);
+    tp_build.add(b, target, optimize, test_step);
+    sessions_build.add(b, test_step);
+    // DeepSeek-V4.1's host code reads POSIX and Linux APIs (pthreads, clock_gettime, io_uring) that Windows lacks
+    if (b.graph.host.result.os.tag != .windows) {
+        dsv41_build.add(b, test_step);
+        kv_build.add(b, target, optimize, test_step); // the family's KV over sessions and the TP fabric
+    }
     sys_build.wire(b);
     @import("zig/build/checkpoint_config.zig").wire(b);
 }

docs/recipes/README.md

@@ -18,6 +18,7 @@ For agent-assisted contributions, read the [TensorFold contributor skill](SKILL.
 | Gemma 4 31B | [Native Metal, affine 4-bit](gemma-4-native.md) |
 | Gemma 4 12B | [Native Metal, affine 4-bit, MTP assistant](gemma-4-12b-native.md) |
 | DeepSeek-V4-Flash | [MLX on a 256 GB Mac, DSpark and MTP drafts](deepseek-v4-flash.md) |
+| DeepSeek-V4.1-Flash | [CUDA (Zig): tools and checks](deepseek-v4.1-cuda.md) |
 | Qwen3.6-35B-A3B | [MLX with MTP drafts, one-GPU CUDA](qwen3.6-moe.md) |
 | Step-5-Preview | [Native Metal on one 256 GB Mac: 2-bit experts, MTP drafts, images](step-5-preview.md) |
 

docs/recipes/deepseek-v4.1-cuda.md

@@ -0,0 +1,41 @@
+# DeepSeek-V4.1-Flash on CUDA (Zig): tools and checks
+
+The `deepseek_v41` family runs on the native Zig engine with its own CUDA kernels (`zig/kernels/cuda/deepseek_v41/`).
+Its development tools are Python and live outside this repository, in jayleaton/deepseek-v41-tensorfold-spark,
+pinned to commit c1cb91840fb8:
+[`tools/tensorfold-dsv41/`](https://github.com/jayleaton/deepseek-v41-tensorfold-spark/tree/c1cb91840fb8ff5db0bd27418fc7cc766a222cbc/tools/tensorfold-dsv41).
+That folder's README lists each tool's dependencies (the TensorFold Python engine, PyTorch, Triton, a GPU where
+noted) and `MAP.tsv` names the TensorFold path each file came from; its `check.py` runs the tools' own checks without
+a GPU.
+
+Nothing in `zig build` or `zig build test` runs Python. The checks below read fixtures committed here; references,
+captures, AOT compiles, SASS identity and GPU measurements are separate receipts made with the tools on real hardware.
+
+## SASS identity of the kernel copies
+
+The family's kernel copies (line 1 of each names its source) must compile to the reference extensions' SASS. With
+Docker and `nvcr.io/nvidia/pytorch:26.07-py3`, from a TensorFold checkout:
+
+```
+OUT=/scratch/sass SM=121 TF_DSV41_KERNELS=$PWD/zig/kernels/cuda/deepseek_v41 TF_DSV41_REF_SRC=/path/to/reference \
+  /path/to/tools/tensorfold-dsv41/kernels/sass_check.sh [name ...]
+```
+
+`TF_DSV41_REF_SRC` (and optionally `TF_DSV41_REF_SRC_R1` / `_X3GM2` with `TF_DSV41_REF_REV*`) names the reference
+repository and revision; the check stops before building without them.
+`-Dnvcc=zig/kernels/cuda/deepseek_v41/nvcc-docker.sh` builds the images themselves through the same container.
+
+## Stored fixtures checked natively
+
+Paths under `zig/src/families/deepseek_v41/`. Host suites (`zig build test-dsv41`, part of `zig build test`; no GPU):
+- `fixtures/widths-exl3-2p8.txt` and `fixtures/config.json`: a 2.8 bpw pack's expert widths and release config
+  (`host_test.zig`).
+- `fixtures/lanes-ref.bin`: the run-time lane layout (`prepare.zig`).
+- `fixtures/engram-hash-ref.bin` and `fixtures/engram-rows-ref.txt`: Engram's hasher and rows (`engram_host.zig`,
+  `engram_rows_test.zig`).
+- Not stored: the device kernel fixtures (`cuda_tests/oracle.py`, `cuda_tests/dsv41_capture.py` in the tools)
+  that `tf-dsv41-test all DIR` and `tf-dsv41-test replay DIR` compare on a GPU.
+- `kv/sched_golden.json`, `kv/session_slots_golden.json`, `kv/session_slots_split_golden.json`: the scheduler and
+  session slots against the reference (`kv/sched_test.zig`, `kv/session_slots_test.zig`; `zig build test-kv`).
+- `fixtures/dense-k2.txt`, `fixtures/expert-k2.txt`: the K2 width listings the emitters and plans are tested on;
+  `fixtures/chain-op-cap16.json`: the drafting chain's op cap (`slot_drafts_chain_test.zig`).

zig/build/cuda.zig

@@ -105,6 +105,7 @@ pub fn targets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.bu
     runner.addImport("glm_cuda", glm_cuda);
     const runner_exe = b.addExecutable(.{ .name = "tf-cuda-test", .root_module = runner });
     b.installArtifact(runner_exe);
+    b.installArtifact(@import("dsv41.zig").runner(b, target, optimize, cuda, strip));
     // tf-tp-watch-check: the ranks' lines with no GPU or NCCL; a failing or dying rank stops every rank
     b.installArtifact(b.addExecutable(.{ .name = "tf-tp-watch-check", .root_module = b.createModule(.{ .root_source_file = b.path("zig/tests/cuda/tp_watch_check.zig"), .target = target, .optimize = optimize, .link_libc = true, .strip = strip, .imports = &.{.{ .name = "cuda", .module = cuda }} }) }));
     flashnextChecks(b, target, optimize, cuda, mods.core, mods.lanes, mods.qwen_vision);

zig/build/dsv41.zig

@@ -0,0 +1,89 @@
+//! DeepSeek-V4.1's build: the family's host side (zig/src/families/deepseek_v41) and its host tests on any OS.
+
+const std = @import("std");
+const tp = @import("tp.zig");
+const kv = @import("kv.zig");
+
+/// The backend-neutral core (the safetensors reader the pack headers use) for the family's host side.
+fn core(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) *std.Build.Module {
+    const tokenizer = b.createModule(.{ .root_source_file = b.path("zig/src/core/tokenizer/tokenizer.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    const m = b.createModule(.{ .root_source_file = b.path("zig/src/core/root.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    m.addImport("tokenizer", tokenizer);
+    return m;
+}
+
+/// The CUDA runtime without kernel images: the family's launch bindings and device loads compile on any host.
+fn runtime(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) *std.Build.Module {
+    const options = b.addOptions();
+    options.addOption(bool, "with_kernels", false);
+    const cuda = b.createModule(.{ .root_source_file = b.path("zig/src/cuda/root.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    cuda.addOptions("kernel_options", options);
+    cuda.addImport("stagger", b.createModule(.{ .root_source_file = b.path("zig/src/core/stagger.zig"), .target = target, .optimize = optimize }));
+    return cuda;
+}
+
+/// The family's kernel bindings and loader (`dsv41_kernels`, kernels.zig) over `cuda`, without fatbins (host-only).
+pub fn kernels(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, cuda: *std.Build.Module) *std.Build.Module {
+    const options = b.addOptions();
+    options.addOption(bool, "with_kernels", false);
+    options.addOption(usize, "dsv41_images", 0); // also keeps this options file apart from the runtime's
+    const m = b.createModule(.{ .root_source_file = b.path("zig/src/families/deepseek_v41/kernels.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    m.addImport("cuda", cuda);
+    m.addOptions("dsv41_kernel_options", options);
+    // the optional images (keyed sampler, two-tile linears) are embedded as empty files
+    const empty = b.addWriteFiles().add("empty.fatbin", "");
+    for ([_][]const u8{ "sampling", "lin2" }) |n| m.addAnonymousImport(b.fmt("dsv41_fatbin_dsv41_{s}", .{n}), .{ .root_source_file = empty });
+    return m;
+}
+
+/// The family's host side (config, pack headers, weight plan, kernel bindings, the forward over `tp` and `kv`).
+pub fn module(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) *std.Build.Module {
+    return rooted(b, target, optimize, "deepseek_v41.zig");
+}
+
+/// The family's sources with their imports, rooted at `file` (a file of zig/src/families/deepseek_v41).
+fn rooted(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, file: []const u8) *std.Build.Module {
+    const m = b.createModule(.{ .root_source_file = b.path(b.fmt("zig/src/families/deepseek_v41/{s}", .{file})), .target = target, .optimize = optimize, .link_libc = true });
+    const cuda = runtime(b, target, optimize);
+    m.addImport("core", core(b, target, optimize));
+    m.addImport("cuda", cuda);
+    m.addImport("dsv41_kernels", kernels(b, target, optimize, cuda));
+    const tp_module = tp.module(b, target, optimize, cuda);
+    m.addImport("tp", tp_module);
+    m.addImport("kv", kv.module(b, target, optimize, cuda, tp_module));
+    return m;
+}
+
+/// `tf-dsv41-test`, the device test runner (zig/tests/cuda/dsv41_main.zig), over `cuda` and host-only family kernels.
+pub fn runner(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, cuda: *std.Build.Module, strip: bool) *std.Build.Step.Compile {
+    const m = b.createModule(.{ .root_source_file = b.path("zig/tests/cuda/dsv41_main.zig"), .target = target, .optimize = optimize, .link_libc = true, .strip = strip });
+    m.addImport("cuda", cuda);
+    m.addImport("dsv41_kernels", kernels(b, target, optimize, cuda));
+    return b.addExecutable(.{ .name = "tf-dsv41-test", .root_module = m });
+}
+
+pub fn add(b: *std.Build, test_step: *std.Build.Step) void {
+    const host = b.graph.host;
+    const step = b.step("test-dsv41", "DeepSeek-V4.1's host-side unit tests (no GPU work)");
+    const family = b.addRunArtifact(b.addTest(.{ .root_module = module(b, host, .debug) }));
+    const bindings = b.addRunArtifact(b.addTest(.{ .root_module = kernels(b, host, .debug, runtime(b, host, .debug)) }));
+    for ([_]*std.Build.Step.Run{ family, bindings }) |run| {
+        step.dependOn(&run.step);
+        test_step.dependOn(&run.step);
+    }
+    // the prefill emitter's own suites, each its own test root (also in test-dsv41 and `zig build test`)
+    const suites = [_][3][]const u8{
+        .{ "test-dsv41-gm2pf", "gm2pf_test.zig", "DeepSeek-V4.1 x3gm v2 (TF_DSV41_GM_V2) on the real prefill emitter, host only" },
+        .{ "test-dsv41-longpf", "longpf_test.zig", "DeepSeek-V4.1 long prompts (stream top-k, split KV's row blocks) on the real prefill emitter, host only" },
+    };
+    for (suites) |x| {
+        const run = b.addRunArtifact(b.addTest(.{ .root_module = rooted(b, host, .debug, x[1]) }));
+        b.step(x[0], x[2]).dependOn(&run.step);
+        step.dependOn(&run.step);
+        test_step.dependOn(&run.step);
+    }
+    // the device runner compiles on any host; running it needs a GPU and every family image
+    const compiled = &runner(b, host, .debug, runtime(b, host, .debug), false).step;
+    step.dependOn(compiled);
+    test_step.dependOn(compiled);
+}

zig/build/kv.zig

@@ -0,0 +1,65 @@
+//! DeepSeek-V4.1's KV memory build: the family's `kv` module over `sessions`, `cuda` and `tp`, host tests on any OS.
+
+const std = @import("std");
+const sessions = @import("sessions.zig");
+const tp = @import("tp.zig");
+
+/// The CUDA runtime without the engine's kernels: kv needs only the driver, memory and streams.
+fn runtime(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) *std.Build.Module {
+    const options = b.addOptions();
+    options.addOption(bool, "with_kernels", false);
+    const cuda = b.createModule(.{ .root_source_file = b.path("zig/src/cuda/root.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    cuda.addOptions("kernel_options", options);
+    cuda.addImport("stagger", b.createModule(.{ .root_source_file = b.path("zig/src/core/stagger.zig"), .target = target, .optimize = optimize }));
+    return cuda;
+}
+
+/// The `kv` module: imports sessions, cuda and tp (an engine passes its own runtime and tp modules).
+pub fn module(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, cuda: *std.Build.Module, tp_module: *std.Build.Module) *std.Build.Module {
+    const m = b.createModule(.{ .root_source_file = b.path("zig/src/families/deepseek_v41/kv/kv.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    m.addImport("sessions", sessions.module(b, target, optimize));
+    m.addImport("cuda", cuda);
+    m.addImport("tp", tp_module);
+    return m;
+}
+
+/// `tf-kv-split-test`, split KV's exchange on two GPUs (kv/split_gpu.zig) over `cuda`, `sessions` and `tp`.
+fn splitRunner(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) *std.Build.Step.Compile {
+    const rt = runtime(b, target, optimize);
+    const m = b.createModule(.{ .root_source_file = b.path("zig/src/families/deepseek_v41/kv/split_gpu.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    m.addImport("sessions", sessions.module(b, target, optimize));
+    m.addImport("cuda", rt);
+    m.addImport("tp", tp.module(b, target, optimize, rt));
+    return b.addExecutable(.{ .name = "tf-kv-split-test", .root_module = m });
+}
+
+/// `tf-kv-gpu-test`, the device pool's runner (kv/gpu_main.zig) over `cuda` and `sessions`.
+fn gpuRunner(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) *std.Build.Step.Compile {
+    const m = b.createModule(.{ .root_source_file = b.path("zig/src/families/deepseek_v41/kv/gpu_main.zig"), .target = target, .optimize = optimize, .link_libc = true });
+    m.addImport("sessions", sessions.module(b, target, optimize));
+    m.addImport("cuda", runtime(b, target, optimize));
+    return b.addExecutable(.{ .name = "tf-kv-gpu-test", .root_module = m });
+}
+
+pub fn add(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, test_step: *std.Build.Step) void {
+    const host = b.graph.host;
+    const cuda = runtime(b, host, .debug);
+    const run = b.addRunArtifact(b.addTest(.{ .root_module = module(b, host, .debug, cuda, tp.module(b, host, .debug, cuda)) }));
+    const step = b.step("test-kv", "DeepSeek-V4.1 KV's host tests: families, sessions, scheduler, split KV on in-process ranks (no GPU)");
+    step.dependOn(&run.step);
+    test_step.dependOn(&run.step);
+    // the device runners compile on any host; running them needs one or two GPUs
+    for ([_]*std.Build.Step{ &gpuRunner(b, host, .debug).step, &splitRunner(b, host, .debug).step }) |compiled| {
+        step.dependOn(compiled);
+        test_step.dependOn(compiled);
+    }
+    const bench = b.createModule(.{ .root_source_file = b.path("zig/src/families/deepseek_v41/kv/bench.zig"), .target = target, .optimize = .ReleaseFast, .link_libc = true });
+    bench.addImport("sessions", sessions.module(b, target, .ReleaseFast));
+    const exe = b.addExecutable(.{ .name = "tf-kv-bench", .root_module = bench });
+    b.step("tf-kv-bench", "Page ops and a 1M-token session parked to / restored from a real NVMe tier (host pool)").dependOn(&b.addInstallArtifact(exe, .{}).step);
+    if (target.result.os.tag != .linux) return;
+    const gexe = gpuRunner(b, target, optimize);
+    b.step("tf-kv-gpu-test", "The device pool on one GPU: tables, a 1M-token session through device memory and the NVMe tier, copy on write").dependOn(&b.addInstallArtifact(gexe, .{}).step);
+    const sexe = splitRunner(b, target, optimize);
+    b.step("tf-kv-split-test", "Split KV's exchange on two GPUs (one process a rank): dense, graph replays, unions vs the replicated rows").dependOn(&b.addInstallArtifact(sexe, .{}).step);
+}

zig/build/sessions.zig

@@ -0,0 +1,20 @@
+//! Session memory: portable bookkeeping tests everywhere, NVMe tier tests on Linux.
+
+const std = @import("std");
+
+pub fn module(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) *std.Build.Module {
+    return b.createModule(.{ .root_source_file = b.path("zig/src/sessions/sessions.zig"), .target = target, .optimize = optimize, .link_libc = true });
+}
+
+pub fn add(b: *std.Build, test_step: *std.Build.Step) void {
+    const host = b.graph.host;
+    const tests = if (host.result.os.tag == .linux) module(b, host, .debug) else b.createModule(.{
+        .root_source_file = b.path("zig/src/sessions/portable_test.zig"),
+        .target = host,
+        .optimize = .debug,
+        .link_libc = true,
+    });
+    const run = b.addRunArtifact(b.addTest(.{ .root_module = tests }));
+    b.step("test-sessions", "Session memory's host tests: pool, prefix trie, RAM tier, NVMe tier on Linux (no GPU)").dependOn(&run.step);
+    test_step.dependOn(&run.step);
+}