DeepSeek V4.1: the Engram token map built in Zig, byte-identical to the release's

build.zig

@@ -425,8 +425,9 @@ fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b
     const pack_cache_run = b.addRunArtifact(pack_cache_cases);
     b.step("test-flashnext-pack-cache", "CPU-only atomic Flash Next pack cache lifecycle").dependOn(&pack_cache_run.step);
     test_step.dependOn(&pack_cache_run.step);
-    const engine_programs = [_]struct { name: []const u8, path: []const u8, about: []const u8, c_source: ?[]const u8 = null }{
+    const engine_programs = [_]struct { name: []const u8, path: []const u8, about: []const u8, c_source: ?[]const u8 = null, host_tests: bool = false }{
         .{ .name = "tf-qwen35-check", .path = "zig/tests/qwen35_check.zig", .about = "Qwen3.5-2B checkpoint and native operations" },
+        .{ .name = "tf-dsv41-token-map", .path = "zig/dsv41_token_map.zig", .about = "Build the Engram token map from a tokenizer and inference config, byte-identical to the release's", .host_tests = true },
         .{ .name = "tf-qwen35-forward", .path = "zig/tests/qwen35_forward.zig", .about = "Qwen3.5-2B native teacher-forced logits" },
         .{ .name = "tf-qwen35-exact", .path = "zig/tests/qwen35_exact.zig", .about = "Qwen3.5-2B forward and committed-state exactness" },
         .{ .name = "tf-lfm2-forward", .path = "zig/tests/lfm2_forward.zig", .about = "LFM2-MoE native teacher-forced logits" },
@@ -479,6 +480,7 @@ fn metalTargets(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b
         const exe = b.addExecutable(.{ .name = p.name, .root_module = mod });
         b.installArtifact(exe);
         b.step(p.name, p.about).dependOn(&b.addInstallArtifact(exe, .{}).step);
+        if (p.host_tests) test_step.dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = mod })).step);
     }
 
     glm_checks.sampler(b, target, optimize, metal, lanes, sources);

tools/zig/lean_check.py

@@ -12,9 +12,9 @@
            ".sh": re.compile(r"^\s*#(?!!)")}
 BLOCK = re.compile(r"/\*")
 DOCSTRING = re.compile(r'^\s*(?:[rubf]{0,2})("""|\'\'\')')
-GENERATED = ("unicode_data.zig",)
+GENERATED = ("unicode_data.zig", "nfkd_data.zig")
 HEX_METAL = re.compile(r"_[0-9a-f]{16}\.metal$")
-# Generator outputs are skipped entirely. unicode_data.zig stays exempt from the length check only.
+# Generator outputs are skipped entirely. The Unicode table files stay exempt from the length check only.
 GENERATOR_OUTPUT = (
     "zig/src/families/flashnext/roles_gen.zig",
     "zig/kernels/metal/flashnext/sources_gen.zig",

tools/zig/nfkd_tables.py

@@ -0,0 +1,96 @@
+"""Emit the Engram token-map normalizer's NFKD, mark, lowercase and combining-class tables as Zig source."""
+import argparse
+import subprocess
+import sys
+import tempfile
+import unicodedata
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+OUT = ROOT / "zig" / "src" / "core" / "tokenizer" / "nfkd_data.zig"
+VERSION = "16.0.0"
+
+
+def rows(values, per_line, fmt):
+    """Comma-terminated lines of at most `per_line` values, four-space indented."""
+    for i in range(0, len(values), per_line):
+        yield "    " + " ".join(fmt(v) + "," for v in values[i:i + per_line])
+
+
+def render():
+    """The generated Zig source before the compiler's formatter aligns the columns."""
+    flat, decompositions = [], []
+    for cp in range(0x110000):
+        decomposed = unicodedata.normalize("NFKD", chr(cp))
+        if decomposed != chr(cp):
+            decompositions.append((cp, len(flat), len(decomposed)))
+            flat.extend(ord(c) for c in decomposed)
+    marks = [cp for cp in range(0x110000) if unicodedata.category(chr(cp)) in ("Mn", "Mc", "Me")]
+    lower_flat, lowercases = [], []
+    for cp in range(0x110000):
+        low = chr(cp).lower()
+        if low != chr(cp):
+            lowercases.append((cp, len(lower_flat), len(low)))
+            lower_flat.extend(ord(c) for c in low)
+    cccs = []
+    for cp in range(0x110000):
+        ccc = unicodedata.combining(chr(cp))
+        if not ccc:
+            continue
+        if cccs and cccs[-1][2] == ccc and cccs[-1][1] + 1 == cp:
+            cccs[-1][1] = cp
+        else:
+            cccs.append([cp, cp, ccc])
+    decomp = lambda e: ".{ .cp = 0x%04X, .start = %d, .len = %d }" % e
+    ccc = lambda t: ".{ 0x%04X, 0x%04X, 0x%04X }" % tuple(t)
+    return "\n".join([
+        "// Generated by tools/zig/nfkd_tables.py from the Unicode %s character database; do not edit." % VERSION,
+        'pub const version = "%s";' % VERSION,
+        "",
+        "pub const Decomp = struct { cp: u21, start: u32, len: u8 };",
+        "",
+        "// All NFKD sequences concatenated, indexed by Decomp.start/.len.",
+        "pub const sequences = [_]u21{", *rows(flat, 8, lambda v: "0x%04X" % v), "};",
+        "",
+        "pub const decompositions = [_]Decomp{", *rows(decompositions, 1, decomp), "};",
+        "",
+        "// General-category Mn/Mc/Me codepoints, sorted; the Rust tokenizers' StripAccents removes all of category M*.",
+        "pub const marks = [_]u21{", *rows(marks, 8, lambda v: "0x%04X" % v), "};",
+        "",
+        "pub const Lower = struct { cp: u21, start: u32, len: u8 };",
+        "",
+        "// Full lowercase mappings concatenated (Rust char::to_lowercase semantics).",
+        "pub const lower_sequences = [_]u21{", *rows(lower_flat, 8, lambda v: "0x%04X" % v), "};",
+        "",
+        "pub const lowercases = [_]Lower{", *rows(lowercases, 1, decomp), "};",
+        "",
+        "// Canonical combining class ranges [first, last, ccc], unicodedata %s." % VERSION,
+        "pub const cccs = [_][3]u21{", *rows(cccs, 4, ccc), "};",
+    ]) + "\n"
+
+
+def formatted(text, zig):
+    """`text` grown through the Zig compiler's formatter, which is how the committed file is shaped."""
+    with tempfile.TemporaryDirectory() as tmp:
+        path = Path(tmp) / "nfkd_data.zig"
+        path.write_text(text)
+        subprocess.run([zig, "fmt", str(path)], check=True, capture_output=True)
+        return path.read_text()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--check", action="store_true", help="fail if the committed file differs")
+    parser.add_argument("--zig", default="zig", help="the Zig compiler whose formatter shapes the output")
+    args = parser.parse_args()
+    if unicodedata.unidata_version != VERSION:
+        sys.exit("Python has Unicode %s; the Engram tables need %s" % (unicodedata.unidata_version, VERSION))
+    text = formatted(render(), args.zig)
+    if args.check:
+        sys.exit(0 if OUT.read_text() == text else "%s is stale; rerun tools/zig/nfkd_tables.py" % OUT.relative_to(ROOT))
+    OUT.write_text(text)
+    print("Wrote %s (Unicode %s)" % (OUT.relative_to(ROOT), VERSION))
+
+
+if __name__ == "__main__":
+    main()