From b8702006e01f6f4c45a95c5aa85b5923026166de Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 10:00:50 +0000 Subject: [PATCH] GLM CUDA copies may draft up to 15 tokens. TF_GLM_COPY_MAX defaults to 15, and 7 restores the 8-row window. Reply copies need a 16-token match. Shared rounds use the same ramp. TF_GLM_COPY_EXTEND=1 opts into context copies, root copies, and a long copy after a 16-token match. Co-authored-by: GarupanOjisan --- RUNBOOK.md | 27 ++++ docs/recipes/glm-5.3-flash-cuda.md | 16 +++ zig/src/core/lanes/config.zig | 9 ++ zig/src/core/lanes/copy_exact_test.zig | 168 ++++++++++++++++++++++++ zig/src/core/lanes/copy_limit.zig | 43 ++++++ zig/src/core/lanes/engine.zig | 10 +- zig/src/core/lanes/lanes.zig | 2 + zig/src/core/lanes/root_copy.zig | 4 +- zig/src/core/lanes/windows.zig | 10 +- zig/src/families/glm/cuda.zig | 1 + zig/src/families/glm/cuda_copy.zig | 91 +++++++++++++ zig/src/families/glm/cuda_engine.zig | 14 +- zig/src/families/glm/cuda_forward.zig | 9 +- zig/src/families/glm/cuda_lanes.zig | 16 ++- zig/src/families/glm/cuda_native.zig | 2 +- zig/src/families/glm/cuda_prompts.zig | 2 +- zig/src/families/glm/cuda_shared.zig | 6 +- zig/src/families/glm/cuda_snapshots.zig | 4 +- zig/src/families/glm/cuda_state.zig | 14 +- 19 files changed, 405 insertions(+), 43 deletions(-) create mode 100644 zig/src/core/lanes/copy_exact_test.zig create mode 100644 zig/src/core/lanes/copy_limit.zig create mode 100644 zig/src/families/glm/cuda_copy.zig diff --git a/RUNBOOK.md b/RUNBOOK.md index 5c20d035f8..a216c60971 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -212,6 +212,33 @@ path needs `LD_LIBRARY_PATH` set to that `lib` directory on both ranks. First launch. The first start can stay silent for several minutes while it reads the checkpoint digest. Wait for `checkpoint digest:` before treating it as stuck. A later start reports the cached files immediately. +### GLM CUDA copy windows + +Rank 0 reads these. Give both ranks the same values. The decode buffers, sampler and KDA save always hold 16 rows. +DFlash stays on its 8-row block (7 drafts). The MTP head still absorbs in 8-row chunks. Native verify launches +kernels directly: there is no captured CUDA graph to widen, and `WIDE_GRAPHS` is not read. The KDA wide step already +walks rows in tiles of 16. + +`TF_GLM_COPY_MAX` is the most tokens a copy may draft. The default is 15 (a 16-row window). `7` is the previous cap +(an 8-row window). Values outside 1..15 clamp into that range; text that is not an integer keeps 15. The first copy +is still 3 tokens and doubles plus one after a fully accepted copy, including in a shared or multi-stream round. +That growth fits: each stream's KDA save holds 16 rows, a shared segment may be 16 rows, and one shared forward +still holds at most 32 rows, so the allocator trims a round that would overflow. + +`TF_GLM_COPY_REPLY_MATCH` is the match a copy needs when its continuation starts in the reply. The default is 16. +A copy taken from the prompt still needs 8. The recipe's 16-token reply bar moved code from 55.4 to 56.2 tok/s and +edit from 120 to 125, with no prose regression called out for that bar. `TF_GLM_COPY_MAX=15` is what moved edit +from 84 to 119 tok/s with prose and code unchanged. Those figures are the recipe's, not a measurement on this tree. + +`TF_GLM_COPY_EXTEND=1` is off by default. It turns on context copies and root copies, and a match of at least 16 +tokens may take the whole `TF_GLM_COPY_MAX` in that round instead of waiting for the ramp. Root copies then skip +the head for an admitted copy. Use it for an A/B against the default. Unset or any value other than `1` leaves it off. + +A 16-row save is twice the KDA window of an 8-row save, and the shared scratch sizes its per-stream chain for 16 +rows. Watch free memory on both GPUs, especially at 4 streams. Drafted token ids must match `"draft": false` +(`token_sha256`). Prose and prefill should not regress. The on-hardware plan is in the pull request that added +these knobs. + ### GB10 CPU pinning for long CUDA runs In [#1033](https://github.com/ashhart/TensorFold/issues/1033), @Bizuayeu reports two CPU threads spinning per CUDA rank, diff --git a/docs/recipes/glm-5.3-flash-cuda.md b/docs/recipes/glm-5.3-flash-cuda.md index a887a291bb..ebb6d98ab4 100644 --- a/docs/recipes/glm-5.3-flash-cuda.md +++ b/docs/recipes/glm-5.3-flash-cuda.md @@ -51,6 +51,22 @@ left after a 12.2 GiB reserve` and a 5.8 GiB prompt cache (`TF_GLM_PREFILL_REDUCE`) and the two-rank RoCE gathers (`TF_ROCE`) are in [placement](../multi-rank-receipts.md#the-settings-the-ranks-agreed-on). +## Copy windows + +Rank 0 plans copies. The buffers below are allocated on every rank. + +| Variable | Default | Effect | +| --- | --- | --- | +| `TF_GLM_COPY_MAX` | 15 | Drafted tokens a copy may hold. `7` is the previous 8-row window. Clamped to 1..15. | +| `TF_GLM_COPY_REPLY_MATCH` | 16 | Matching tokens required when the copied continuation starts in the reply. Prompt copies stay at 8. | +| `TF_GLM_COPY_EXTEND` | off | `1` enables context copies, root copies, and a 16-token match that may use the whole copy cap at once. | + +DFlash remains a 7-draft block. The MTP head still chunks its absorb at 8 rows. A copy may grow past its first 3 +tokens after a fully accepted copy, in a lone round and in a shared round. The shared forward still admits at most +32 rows. Native GLM verify does not capture CUDA graphs, so there is no 16-row graph set; the KDA wide kernel +already accepts a 16-row window. Set the same values on every rank. Compare `token_sha256` with `"draft": false`, +and watch memory: the KDA save and the shared per-stream chain are sized for 16 rows. + ## Long runs on GB10 Long prompts and long runs of replies heat a GB10. The runbook's [heat bands](../../RUNBOOK.md#linux-and-cuda) (with a diff --git a/zig/src/core/lanes/config.zig b/zig/src/core/lanes/config.zig index cb2159b03a..8b276c2d26 100644 --- a/zig/src/core/lanes/config.zig +++ b/zig/src/core/lanes/config.zig @@ -45,6 +45,9 @@ pub const Model = struct { lone_cost_tile: u32 = 0, // live lone-window pricing probes at these target row boundaries (0: off) context_copies: bool = false, // a lone head window shares its agreeing copy path and appends the rest copy_match: i64 = 8, // matching history tokens a copy needs + reply_match: i64 = 0, // extra match when the continuation starts in the reply (0: copy_match covers it) + strong_match: i64 = 0, // 0 off; a match at least this long may copy strong_copy tokens in one round + strong_copy: u32 = 0, // that longer cap, still within max_copy (0: the ramp only) lone_copies: bool = true, // a lone stream proposes copies (false: only streams in shared rounds do) }; @@ -90,6 +93,9 @@ pub const Config = struct { lone_cost_tile: u32 = 0, // Model.lone_cost_tile context_copies: bool = false, // Model.context_copies, within the existing verification width enter_match: i64 = 8, // matching tokens a copy needs (rejects coincidental indentation) + reply_match: i64 = 0, // Model.reply_match, stricter than enter_match for a copy out of the reply + strong_match: i64 = 0, // Model.strong_match + strong_copy: u32 = 0, // Model.strong_copy copy_rate: f64 = 0.94, // a copied token's chance of landing in allocation lone_copies: bool = true, // Model.lone_copies k: Constants = .{}, @@ -130,6 +136,9 @@ pub const Config = struct { .root_copies = m.root_copies and m.copy_drafts, .context_copies = m.context_copies and m.copy_drafts, .enter_match = @max(1, m.copy_match), + .reply_match = if (m.reply_match > m.copy_match) m.reply_match else 0, + .strong_match = if (m.strong_copy > 0) m.strong_match else 0, + .strong_copy = if (m.copy_drafts) @min(m.strong_copy, max_copy) else 0, .lone_copies = m.lone_copies, .lone_cost_tile = m.lone_cost_tile, .family_width = width, diff --git a/zig/src/core/lanes/copy_exact_test.zig b/zig/src/core/lanes/copy_exact_test.zig new file mode 100644 index 0000000000..71fae8304d --- /dev/null +++ b/zig/src/core/lanes/copy_exact_test.zig @@ -0,0 +1,168 @@ +//! GLM-shaped copy windows: drafted tokens match a `"draft": false` run, greedy and sampled. +const std = @import("std"); +const Config = @import("config.zig").Config; +const Model = @import("config.zig").Model; +const Engine = @import("engine.zig").Engine; +const Stream = @import("stream.zig").Stream; +const Sampling = @import("sampling.zig").Sampling; +const SuffixLookup = @import("proposer.zig").SuffixLookup; +const fake = @import("fake.zig"); + +const gpa = std.testing.allocator; +const eos = [_]u32{96}; +const sampled: Sampling = .{ .seed = 11, .temperature = 0.7, .top_k = 0, .top_p = 0.9 }; + +fn glmModel() Model { + return .{ + .exact_width = 16, + .first_copy_rows = 4, + .mtp = true, + .speculate = true, + .speculate_early = false, + .drafts = 7, + .hidden_rows = true, + .batch_rows = 32, + .max_streams = 16, + .copy_match = 8, + .reply_match = 16, + .streams_exact = true, + }; +} + +const Out = struct { tokens: []u32, drafted: u64 }; + +fn run(prompts: []const []const u32, sampling: ?Sampling, drafts: bool) ![]Out { + var cfg = try Config.init(gpa, glmModel(), 16, 15); + defer cfg.deinit(gpa); + var target: fake.Fake = .{ .gpa = gpa }; + defer target.deinit(); + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, target.backend(), clock.clock()); + defer engine.deinit(); + const streams = try gpa.alloc(Stream, prompts.len); + defer gpa.free(streams); + const looks = try gpa.alloc(SuffixLookup, prompts.len); + defer gpa.free(looks); + for (prompts, streams, looks) |p, *s, *look| { + look.* = try SuffixLookup.init(gpa, .{ .ngram = 3, .min_match = 6 }); + s.* = try Stream.init(gpa, .{ + .id = "s", + .prompt = p, + .max_new = 24, + .eos = &eos, + .sampling = sampling, + .drafts = drafts, + .proposer = look.proposer(), + }); + } + defer for (streams, looks) |*s, *look| { + s.deinit(gpa); + look.deinit(); + }; + for (streams) |*s| try engine.addStream(s); + while (engine.activeCount() > 0) try engine.step(); + const out = try gpa.alloc(Out, prompts.len); + for (out, streams) |*o, *s| o.* = .{ .tokens = try gpa.dupe(u32, s.emitted()), .drafted = s.drafted }; + return out; +} + +fn freeOut(rows: []Out) void { + for (rows) |o| gpa.free(o.tokens); + gpa.free(rows); +} + +/// Two copies of a 16-token block, then the block's first 8, so a prompt copy has a continuation. +const repeat = blk: { + var p: [40]u32 = undefined; + for (&p, 0..) |*t, i| t.* = @as(u32, @intCast(i % 16)); + break :blk p; +}; +const other = [_]u32{ 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 3, 5, 7, 9, 11 }; + +test "drafted GLM copy windows match draft false, greedy and sampled" { + var cfg = try Config.init(gpa, glmModel(), 16, 15); + defer cfg.deinit(gpa); + try std.testing.expectEqual(@as(u32, 15), cfg.max_copy); + try std.testing.expectEqual(@as(u32, 3), cfg.first_copy); + try std.testing.expectEqual(@as(u32, 7), cfg.most_drafts); + for ([_]?Sampling{ null, sampled }) |sampling| { + const drafted = try run(&.{&repeat}, sampling, true); + defer freeOut(drafted); + const plain = try run(&.{&repeat}, sampling, false); + defer freeOut(plain); + try std.testing.expectEqualSlices(u32, plain[0].tokens, drafted[0].tokens); + try std.testing.expectEqual(@as(u64, 0), plain[0].drafted); + if (sampling == null) try std.testing.expect(drafted[0].drafted > 0); + } +} + +fn propose(extend: bool, alone: bool, width: ?u32, prompt: []const u32, extra: []const u32) ![]const u32 { + var cfg = try Config.init(gpa, .{ + .exact_width = 16, + .first_copy_rows = 4, + .mtp = true, + .speculate = true, + .drafts = 7, + .copy_match = 8, + .reply_match = 16, + .strong_match = if (extend) 16 else 0, + .strong_copy = if (extend) 15 else 0, + }, 16, 15); + var target: fake.Fake = .{ .gpa = gpa }; + var clock: fake.FixedClock = .{}; + var engine = Engine.init(gpa, &cfg, target.backend(), clock.clock()); + engine.alone = alone; + var look = try SuffixLookup.init(gpa, .{ .ngram = 3, .min_match = 6 }); + var s = try Stream.init(gpa, .{ .id = "p", .prompt = prompt, .max_new = 64, .proposer = look.proposer() }); + if (extra.len > 0) try s.context.appendSlice(gpa, extra); + s.copy_width = width; + const got = try @import("windows.zig").copyProposal(&engine, &s); + const kept = try gpa.dupe(u32, got.tokens); + s.deinit(gpa); + look.deinit(); + engine.deinit(); + target.deinit(); + cfg.deinit(gpa); + return kept; +} + +test "reply copies need 16 tokens, and a shared ramp is the grown width" { + var block: [24]u32 = undefined; + for (&block, 0..) |*t, i| t.* = @intCast(i + 1); + const prompt = [_]u32{ 90, 91, 92, 93 }; + var reply: [34]u32 = undefined; + @memcpy(reply[0..24], &block); + @memcpy(reply[24..], block[0..10]); + const short = try propose(false, true, null, &prompt, &reply); + defer gpa.free(short); + try std.testing.expectEqual(@as(usize, 0), short.len); + @memcpy(reply[24..], block[0..10]); + var long_reply: [40]u32 = undefined; + @memcpy(long_reply[0..24], &block); + @memcpy(long_reply[24..], block[0..16]); + const allowed = try propose(false, false, 7, &prompt, &long_reply); + defer gpa.free(allowed); + try std.testing.expect(allowed.len >= 2 and allowed.len <= 7); + var prompt_hist: [40]u32 = undefined; + @memcpy(prompt_hist[0..24], &block); + @memcpy(prompt_hist[24..], block[0..16]); + const capped = try propose(false, true, null, &prompt_hist, &.{}); + defer gpa.free(capped); + try std.testing.expect(capped.len >= 2 and capped.len <= 3); + const extended = try propose(true, true, null, &prompt_hist, &.{}); + defer gpa.free(extended); + try std.testing.expect(extended.len > 3 and extended.len <= 15); +} + +test "two streams' drafted tokens match each stream's draft false run" { + for ([_]?Sampling{ null, sampled }) |sampling| { + const drafted = try run(&.{ &repeat, &other }, sampling, true); + defer freeOut(drafted); + const a = try run(&.{&repeat}, sampling, false); + defer freeOut(a); + const b = try run(&.{&other}, sampling, false); + defer freeOut(b); + try std.testing.expectEqualSlices(u32, a[0].tokens, drafted[0].tokens); + try std.testing.expectEqualSlices(u32, b[0].tokens, drafted[1].tokens); + } +} diff --git a/zig/src/core/lanes/copy_limit.zig b/zig/src/core/lanes/copy_limit.zig new file mode 100644 index 0000000000..6ff611a218 --- /dev/null +++ b/zig/src/core/lanes/copy_limit.zig @@ -0,0 +1,43 @@ +//! How many tokens a copy may draft, including a reply-only match and a strong-match extension. +const Engine = @import("engine.zig").Engine; +const Stream = @import("stream.zig").Stream; + +/// The ramp this round would use before a strong-match extension (shared rounds included). +pub fn ramp(e: *const Engine, s: *const Stream) u32 { + const width = s.copy_width orelse e.cfg.first_copy; + return width; +} + +/// Tokens to ask the proposer for. A configured strong cap asks for that, and `take` shortens a weak match. +pub fn ask(e: *const Engine, s: *const Stream) u32 { + const base = ramp(e, s); + return if (e.cfg.strong_copy > base) e.cfg.strong_copy else base; +} + +/// The ramp after this copy: a full accept doubles plus one, up to max_copy; a miss returns to the first width. +pub fn nextWidth(current: u32, first: u32, max_copy: u32, full: bool) u32 { + return if (full) @min(max_copy, 2 * current + 1) else first; +} + +/// The copy to keep. A continuation that starts in the reply needs `reply_match`. A short match stays on the ramp. +pub fn take(e: *const Engine, s: *const Stream, match: i64, end: ?usize, copied: []const u32) []const u32 { + const least: usize = if (e.alone and e.cfg.context_copies) 1 else 2; + if (copied.len < least or match < e.cfg.enter_match) return &.{}; + if (e.cfg.reply_match > e.cfg.enter_match) { + if (end) |at| if (at >= s.prompt_len and match < e.cfg.reply_match) return &.{}; + } + if (e.cfg.strong_match > 0 and match >= e.cfg.strong_match) return copied; + if (e.cfg.strong_copy == 0) return copied; + const n = @min(copied.len, @as(usize, ramp(e, s))); + return if (n >= least) copied[0..n] else &.{}; +} + +const std = @import("std"); + +test "a full accept grows the ramp, and a miss returns to the first width" { + try std.testing.expectEqual(@as(u32, 7), nextWidth(3, 3, 15, true)); + try std.testing.expectEqual(@as(u32, 15), nextWidth(7, 3, 15, true)); + try std.testing.expectEqual(@as(u32, 15), nextWidth(15, 3, 15, true)); + try std.testing.expectEqual(@as(u32, 7), nextWidth(7, 3, 7, true)); + try std.testing.expectEqual(@as(u32, 3), nextWidth(7, 3, 15, false)); +} diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index a92b27ae7f..4b1d32c65e 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -8,6 +8,7 @@ const be = @import("backend.zig"); const sm = @import("stream.zig"); const ev = @import("events.zig"); const win = @import("windows.zig"); +const copy_limit = @import("copy_limit.zig"); const shape = @import("shape.zig"); const plan_lanes = @import("plan_lanes.zig"); const fill = @import("fill.zig"); @@ -46,7 +47,7 @@ pub const Engine = struct { drafted: u64 = 0, accepted: u64 = 0, shared_rounds: u64 = 0, - alone: bool = true, // this step has one live stream: its copies may take their ramp + alone: bool = true, // this step has one live stream lone_kept: [2][lone_costs.bands]?lone_costs.Kept = @splat(@splat(null)), // the newest lone calibrations, greedy then sampled, a band each steps: u64 = 0, arena: std.heap.ArenaAllocator, // a round's temporaries @@ -347,10 +348,9 @@ pub const Engine = struct { if (p.recalled) { if (s.recall) |r| r.observe(@intCast(own), @intCast(own_kept), s.rounds); } else if (s.proposer) |pr| pr.observe(@intCast(own), @intCast(own_kept)); - if (e.alone) { - const width = s.copy_width orelse e.cfg.first_copy; - s.copy_width = if (own_kept == own) @min(e.cfg.max_copy, 2 * width + 1) else e.cfg.first_copy; - } + // a shared round grows the same ramp: each stream's save holds the copy window + const width = s.copy_width orelse e.cfg.first_copy; + s.copy_width = copy_limit.nextWidth(width, e.cfg.first_copy, e.cfg.max_copy, own_kept == own); } else if (p.kind == .head) { if (p.lanes) |l| if (s.odds != null) s.observeLanes(l, path); const ds = try accept.depths(a, parents); diff --git a/zig/src/core/lanes/lanes.zig b/zig/src/core/lanes/lanes.zig index ff09b58fb2..df9fcea54e 100644 --- a/zig/src/core/lanes/lanes.zig +++ b/zig/src/core/lanes/lanes.zig @@ -65,6 +65,8 @@ test { _ = @import("fill_test.zig"); _ = @import("fill_trim_test.zig"); _ = @import("fair_turns_test.zig"); + _ = @import("copy_limit.zig"); + _ = @import("copy_exact_test.zig"); } pub const learned_disk = @import("learned_disk.zig"); diff --git a/zig/src/core/lanes/root_copy.zig b/zig/src/core/lanes/root_copy.zig index c3bc5accd4..6da3355d17 100644 --- a/zig/src/core/lanes/root_copy.zig +++ b/zig/src/core/lanes/root_copy.zig @@ -2,6 +2,7 @@ const Engine = @import("engine.zig").Engine; const be = @import("backend.zig"); const win = @import("windows.zig"); +const limit = @import("copy_limit.zig"); pub fn prepare(e: *Engine, requests: []const be.DraftRequest) ![]const be.DraftRequest { if (!e.cfg.root_copies) return requests; @@ -15,8 +16,7 @@ pub fn prepare(e: *Engine, requests: []const be.DraftRequest) ![]const be.DraftR const pending = try e.readFeed(r.first orelse continue); const most = @min(@as(i64, e.cfg.max_copy), s.draftRoom() - 1); const got = p.proposeAfter(s.context.items, &.{pending}, most) catch continue; - const least: usize = if (e.alone and e.cfg.context_copies) 1 else 2; - break :blk got.len >= least and p.lastMatch() >= e.cfg.enter_match; + break :blk limit.take(e, s, p.lastMatch(), p.lastEnd(), got).len > 0; } else (try win.copyProposal(e, s)).tokens.len > 0; if (!copies) continue; if (out == null) out = e.arena.allocator().dupe(be.DraftRequest, requests) catch return requests; diff --git a/zig/src/core/lanes/windows.zig b/zig/src/core/lanes/windows.zig index e8e48be2ad..3d8cd52e52 100644 --- a/zig/src/core/lanes/windows.zig +++ b/zig/src/core/lanes/windows.zig @@ -12,6 +12,7 @@ const shape = @import("shape.zig"); const ev = @import("events.zig"); const trail = @import("trail.zig"); const Engine = @import("engine.zig").Engine; +const limit = @import("copy_limit.zig"); const Stream = @import("stream.zig").Stream; const Value = ev.Value; const f = ev.f; @@ -195,17 +196,16 @@ fn headTrunk(e: *Engine, s: *Stream, p: Plan, levels: usize) !?fill.Trunk { return .{ .tokens = chain, .others = others }; } -/// A copy from the context backed by `enter_match` ids (Python `_copy_proposal`), else Recall's when it should pay. +/// A context copy backed by enter_match (Python `_copy_proposal`), else Recall; shared rounds use this ramp too. pub fn copyProposal(e: *Engine, s: *Stream) !Copy { if (e.cfg.max_copy == 0 or s.force.items.len > 0 or (e.cfg.root_copies and s.root_copy_miss)) return .{}; - const width: u32 = if (e.alone) (s.copy_width orelse e.cfg.first_copy) else e.cfg.first_copy; - const most = @min(@as(i64, width), s.draftRoom() - 1); + const most = @min(@as(i64, limit.ask(e, s)), s.draftRoom() - 1); if (s.proposer) |p| if (!e.alone or e.cfg.lone_copies) { // a priced proposer's drafts are as it gives them, its none too: no width, length or match gate, no Recall if (p.vtable.priced) return .{ .tokens = p.propose(s.context.items, @min(@as(i64, e.cfg.max_copy), s.draftRoom() - 1)) catch &.{} }; const copied = p.proposeAfter(s.context.items, &.{}, most) catch &.{}; - const least: usize = if (e.alone and e.cfg.context_copies) 1 else 2; - if (copied.len >= least and p.lastMatch() >= e.cfg.enter_match) return .{ .tokens = copied }; + const kept = limit.take(e, s, p.lastMatch(), p.lastEnd(), copied); + if (kept.len > 0) return .{ .tokens = kept }; }; const r = s.recall orelse return .{}; if (most < 2) return .{}; diff --git a/zig/src/families/glm/cuda.zig b/zig/src/families/glm/cuda.zig index 7fe9c1a198..3b62f90778 100644 --- a/zig/src/families/glm/cuda.zig +++ b/zig/src/families/glm/cuda.zig @@ -67,6 +67,7 @@ test { _ = engine; _ = follow; _ = lanes; + _ = @import("cuda_copy.zig"); _ = native; _ = @import("cuda_qpack.zig"); _ = @import("cuda_tile.zig"); diff --git a/zig/src/families/glm/cuda_copy.zig b/zig/src/families/glm/cuda_copy.zig new file mode 100644 index 0000000000..83dc5143a9 --- /dev/null +++ b/zig/src/families/glm/cuda_copy.zig @@ -0,0 +1,91 @@ +//! GLM CUDA copy-window knobs. DFlash and the MTP head stay on the 8-row window; copies may use 16. +const std = @import("std"); +const lanes = @import("lanes"); +const state = @import("cuda_state.zig"); + +/// Rows the decode buffers, sampler and KDA save hold. Head windows stay at `window_rows`. +pub const copy_rows = state.copy_rows; +/// Drafted tokens a copy window holds at most. +pub const max_copy: u32 = copy_rows - 1; +/// A copy whose continuation starts in the reply needs this many matching tokens (the recipe's measured bar). +pub const reply_default: i64 = 16; +/// With the opt-in flag, a match at least this long may take the whole copy cap in one round. +pub const strong_match: i64 = 16; + +pub const Knobs = struct { + copy_max: u32 = max_copy, + reply_match: i64 = reply_default, + extend: bool = false, +}; + +/// `TF_GLM_COPY_MAX` defaults to 15 (7 is the old cap). Reply match defaults to 16. Extend is `1`. +pub fn read() Knobs { + return .{ + .copy_max = clampCopy(envInt("TF_GLM_COPY_MAX")), + .reply_match = clampReply(envInt("TF_GLM_COPY_REPLY_MATCH")), + .extend = envOne("TF_GLM_COPY_EXTEND"), + }; +} + +/// Values outside 1..15 clamp into that range. Unparseable text keeps the default. +pub fn clampCopy(v: ?i64) u32 { + const n = v orelse return max_copy; + if (n < 1) return 1; + if (n > max_copy) return max_copy; + return @intCast(n); +} + +/// Values outside 1..64 clamp into that range. Unparseable text keeps the default. +pub fn clampReply(v: ?i64) i64 { + const n = v orelse return reply_default; + if (n < 1) return 1; + if (n > 64) return 64; + return n; +} + +/// Drafting facts with the copy cap applied. Head `drafts` is left as the caller set it (DFlash 7, MTP 3). +pub fn apply(m: lanes.Model, k: Knobs) lanes.Model { + var out = m; + if (!m.mtp) return out; + out.exact_width = k.copy_max + 1; + out.reply_match = k.reply_match; + if (!k.extend) return out; + out.context_copies = true; + out.root_copies = true; + out.strong_match = strong_match; + out.strong_copy = k.copy_max; + return out; +} + +fn envInt(name: [*:0]const u8) ?i64 { + const v = std.c.getenv(name) orelse return null; + return std.fmt.parseInt(i64, std.mem.span(v), 10) catch null; +} + +fn envOne(name: [*:0]const u8) bool { + const v = std.c.getenv(name) orelse return false; + return std.mem.eql(u8, std.mem.span(v), "1"); +} + +test "copy caps clamp, and the head draft count is not the copy window" { + try std.testing.expectEqual(@as(u32, 15), clampCopy(null)); + try std.testing.expectEqual(@as(u32, 7), clampCopy(7)); + try std.testing.expectEqual(@as(u32, 1), clampCopy(0)); + try std.testing.expectEqual(@as(u32, 15), clampCopy(40)); + try std.testing.expectEqual(@as(i64, 16), clampReply(null)); + try std.testing.expectEqual(@as(i64, 64), clampReply(100)); + const head = lanes.Model{ .exact_width = 8, .mtp = true, .speculate = true, .drafts = 7, .first_copy_rows = 4 }; + const wide = apply(head, .{}); + try std.testing.expectEqual(@as(u32, 16), wide.exact_width); + try std.testing.expectEqual(@as(u32, 7), wide.drafts); + try std.testing.expect(!wide.context_copies and !wide.root_copies); + try std.testing.expectEqual(@as(u32, 0), wide.strong_copy); + const old = apply(head, .{ .copy_max = 7 }); + try std.testing.expectEqual(@as(u32, 8), old.exact_width); + const ext = apply(head, .{ .extend = true }); + try std.testing.expect(ext.context_copies and ext.root_copies); + try std.testing.expectEqual(strong_match, ext.strong_match); + try std.testing.expectEqual(@as(u32, 15), ext.strong_copy); + const idle = apply(.{ .exact_width = 1 }, .{}); + try std.testing.expectEqual(@as(u32, 1), idle.exact_width); +} diff --git a/zig/src/families/glm/cuda_engine.zig b/zig/src/families/glm/cuda_engine.zig index b51c8366bc..44b9be1072 100644 --- a/zig/src/families/glm/cuda_engine.zig +++ b/zig/src/families/glm/cuda_engine.zig @@ -58,7 +58,7 @@ pub const Engine = struct { images: ?*images_mod.Images = null, // --vision: a prompt pass's image rows (rank 0 encodes, the others receive) packs: mm_mod.Packed = .{}, // the linears packed at load and their bits (TF_GLM_DENSE_BITS, _HEAD_BITS) drafter: ?*dflash.Drafter = null, // --drafter: DFlash2 (every rank runs its passes; rank 0 selects the drafts) - profs: [fw.window_rows]?*prof_mod.Prof = @splat(null), // TF_GLM_PROF=1: verify sections' GPU time, by rows + profs: [fw.copy_rows]?*prof_mod.Prof = @splat(null), // TF_GLM_PROF=1: verify sections' GPU time, by rows profiling_prompt: ?*prof_mod.Prof = null, prompt_profs: [4]?*prof_mod.Prof = @splat(null), // and prompt chunks': to 64, 256, 1024, 2048 rows checkpoint_digest: [32]u8, @@ -172,16 +172,16 @@ pub const Engine = struct { } e.bp = try fw.Buffers.init(d, &e.w, fw.prefill_rows, true, o.capacity); errdefer e.bp.deinit(); - e.bd = try fw.Buffers.init(d, &e.w, fw.window_rows, false, o.capacity); + e.bd = try fw.Buffers.init(d, &e.w, fw.copy_rows, false, o.capacity); errdefer e.bd.deinit(); - e.bm = try fw.Buffers.init(d, &e.w, fw.window_rows, false, o.capacity); + e.bm = try fw.Buffers.init(d, &e.w, fw.window_rows, false, o.capacity); // MTP absorbs in head-sized chunks errdefer e.bm.deinit(); if (o.mtp) e.bs = try fw.Buffers.initShared(d, &e.w, shared_rows, o.capacity); errdefer if (e.bs) |*b| b.deinit(); e.pairs = try cuda.DeviceBuffer.alloc(d, pair_bytes * world); errdefer e.pairs.free(); try d.check(d.api.cuMemsetD8_v2(e.pairs.ptr, 0, pair_bytes * world), "cuMemsetD8"); // each pair's padding stays zero - e.smp = try sampler.Sampler.init(d, c.vocab, fw.window_rows); + e.smp = try sampler.Sampler.init(d, c.vocab, fw.copy_rows); errdefer e.smp.deinit(); errdefer e.freeProfs(); if (prof_mod.Prof.wanted()) for (&e.profs) |*p| { @@ -307,7 +307,7 @@ pub const Engine = struct { /// Where a prompt pass of `len` ids starts: 0, or `from`'s position with its kept state restored. fn prefillStart(e: *Engine, len: usize, from: ?Resume) !usize { - if (len == 0 or len + fw.window_rows >= e.cur.st.capacity) return error.PromptTooLong; + if (len == 0 or len + fw.copy_rows >= e.cur.st.capacity) return error.PromptTooLong; const r = from orelse return 0; if (r.at == 0 or r.at >= len or r.at % fw.prefill_rows != 0) return error.BadResume; try e.restoreState(r.id); @@ -449,7 +449,7 @@ pub const Engine = struct { /// A verify window: `ids` (pending token, then drafts) at the committed position; `out[r]` drawn at pos + 1 + r. pub fn verify(e: *Engine, ids: []const u32, out: []u32, bits: u64) !void { - if (ids.len == 0 or ids.len > fw.window_rows) return error.WindowTooWide; + if (ids.len == 0 or ids.len > fw.copy_rows) return error.WindowTooWide; var one: [1]fw.Rank = undefined; var f = e.view(&e.bd, &one); const at = e.cur.st.pos; @@ -472,7 +472,7 @@ pub const Engine = struct { var ids: [shared_rows]u32 = undefined; var rows: usize = 0; for (parts, 0..) |p, i| { - if (p.ids.len == 0 or p.ids.len > fw.window_rows or rows + p.ids.len > shared_rows) return error.WindowTooWide; + if (p.ids.len == 0 or p.ids.len > fw.copy_rows or rows + p.ids.len > shared_rows) return error.WindowTooWide; for (parts[0..i]) |q| if (q.slot == p.slot) return error.SlotTwice; try e.bind(p.slot); segs[i] = .{ .st = &e.cur.st, .row0 = rows, .rows = p.ids.len }; diff --git a/zig/src/families/glm/cuda_forward.zig b/zig/src/families/glm/cuda_forward.zig index aa87369737..1e34f8b00c 100644 --- a/zig/src/families/glm/cuda_forward.zig +++ b/zig/src/families/glm/cuda_forward.zig @@ -26,6 +26,7 @@ const shared = @import("cuda_shared.zig"); pub const prefill_rows = state.prefill_rows; pub const window_rows = state.window_rows; +pub const copy_rows = state.copy_rows; pub const State = state.State; pub const Buffers = state.Buffers; @@ -239,11 +240,11 @@ pub const Forward = struct { s.p = if (b.prefill) b.kproj else st.proj + li * st.proj_layer * 2; s.saves.state_out = st.recAt(1 - cur, li); if (!b.prefill) { // a window's saved rows stay with the layer until its commit - const kv = li * window_rows * st.heads * kda.dk; + const kv = li * copy_rows * st.heads * kda.dk; s.saves.k = st.save_k + kv * 4; s.saves.v = st.save_v + kv * 2; s.saves.g = st.save_g + kv * 4; - s.saves.b = st.save_b + li * window_rows * st.heads * 4; + s.saves.b = st.save_b + li * copy_rows * st.heads * 4; } const conv = st.conv + li * 3 * st.conv_c * 2; try kda_block.bodyProfile(f.set(), &f.m.kda, f.stream, k, s, b.normed, c.hidden, rows, conv, st.recAt(cur, li), !b.prefill and f.kda_wide, b.prefill, f.prof); @@ -517,8 +518,8 @@ pub const Forward = struct { a.add(st.recAt(cur, 0)); a.add(@as(usize, st.rec_layer / 4)); for ([_]u64{ st.save_k, st.save_v, st.save_g, st.save_b }) |x| a.add(x); - a.add(@as(usize, window_rows * h * kda.dk)); - a.add(@as(usize, window_rows * h)); + a.add(@as(usize, copy_rows * h * kda.dk)); + a.add(@as(usize, copy_rows * h)); a.add(@as(c_int, @intCast(keep))); a.add(st.recAt(1 - cur, 0)); try cuda.launch.launch(f.m.kda.replay_layers, .{ .grid = .{ .x = @intCast(st.kda_layers * h) }, .block = .{ .x = 1024 } }, f.stream, &a); diff --git a/zig/src/families/glm/cuda_lanes.zig b/zig/src/families/glm/cuda_lanes.zig index f4599356c1..7c0a0729cf 100644 --- a/zig/src/families/glm/cuda_lanes.zig +++ b/zig/src/families/glm/cuda_lanes.zig @@ -10,16 +10,17 @@ const prompt_lp = @import("cuda_prompt_logprobs.zig"); const cuda = @import("cuda"); const be = lanes.backend; +const copy = @import("cuda_copy.zig"); /// First tokens a handle names. const ring = 1024; -/// The widest verify window: the pending token and a copy (the lane core proposes them), as the decode buffers hold. -pub const max_rows = fw.window_rows; +/// The widest verify window the buffers hold. DFlash stays at fw.window_rows; copies use this. +pub const max_rows = fw.copy_rows; const mtp_rows = 1 + engine.max_drafts; const max_held = dflash.BLOCK - 1; // a DFlash2 block's drafts (the MTP head's chains stop at max_drafts) -/// With DFlash2: a window's ms by rows on two GB10s (EXL3, TF_GLM_PROF: 86 to 107 ms at 5 to 8), shared as glm_cuda's. +/// Window ms by rows on two GB10s (EXL3, TF_GLM_PROF: 86 to 107 ms at 5 to 8); past 8 rows the same line. const gb10_window_costs = costTable(max_rows, 52.0, 6.8); const gb10_shared_costs = costTable(engine.shared_rows, 76.0, 4.1); @@ -99,12 +100,12 @@ pub const Lanes = struct { break :vt v; }; - /// Round loop facts: copies up to max_rows, head chains to mtp_rows, shared windows unless TF_GLM_SHARED=0. + /// Round loop facts: copies up to TF_GLM_COPY_MAX, head chains to mtp_rows, shared unless TF_GLM_SHARED=0. pub fn facts(self: *const Lanes) lanes.Model { const drafting = self.x.e.mtp; const block = self.x.e.drafter != null; const sharing = drafting and !self.alone; - return .{ + return copy.apply(.{ .exact_width = if (drafting) max_rows else 1, .first_copy_rows = mtp_rows, // a first copy as wide as a head round; it grows as its tokens land .gpu_tokens = false, @@ -120,7 +121,7 @@ pub const Lanes = struct { .hidden_rows = sharing, .batch_rows = if (sharing) engine.shared_rows else 0, .max_streams = if (sharing) engine.max_shared else 1, - }; + }, copy.read()); } fn of(ptr: *anyopaque) *Lanes { @@ -517,7 +518,8 @@ pub const Lanes = struct { }; comptime { - std.debug.assert(max_rows <= fw.window_rows); + std.debug.assert(max_rows == fw.copy_rows); + std.debug.assert(dflash.BLOCK <= fw.window_rows); // DFlash's block stays on the head window } test "a lane's slot is chosen before its map entry exists" { diff --git a/zig/src/families/glm/cuda_native.zig b/zig/src/families/glm/cuda_native.zig index d46b4284f9..590592368a 100644 --- a/zig/src/families/glm/cuda_native.zig +++ b/zig/src/families/glm/cuda_native.zig @@ -90,7 +90,7 @@ pub fn weightShare(a: std.mem.Allocator, io: std.Io, dir: []const u8, total: u64 /// Slots for `context` plus a verify window, a multiple of 64 so the sparse pool keeps the AOT set. pub fn capacity(context: usize) usize { - return std.mem.alignForward(usize, context + fw.window_rows, 64); + return std.mem.alignForward(usize, context + fw.copy_rows, 64); } const snapshots = @import("cuda_snapshots.zig"); diff --git a/zig/src/families/glm/cuda_prompts.zig b/zig/src/families/glm/cuda_prompts.zig index b66392f44c..f40d95da55 100644 --- a/zig/src/families/glm/cuda_prompts.zig +++ b/zig/src/families/glm/cuda_prompts.zig @@ -10,7 +10,7 @@ pub fn fits(parts: []const engine.Prompt, capacity: usize) bool { if (parts.len == 0 or parts.len > engine.max_together) return false; var rows: usize = 0; for (parts, 0..) |p, i| { - if (p.ids.len == 0 or p.ids.len > fw.prefill_rows or p.ids.len + fw.window_rows >= capacity) return false; + if (p.ids.len == 0 or p.ids.len > fw.prefill_rows or p.ids.len + fw.copy_rows >= capacity) return false; for (parts[0..i]) |q| if (q.slot == p.slot) return false; rows += p.ids.len; } diff --git a/zig/src/families/glm/cuda_shared.zig b/zig/src/families/glm/cuda_shared.zig index 1899ed31b2..01a4715ade 100644 --- a/zig/src/families/glm/cuda_shared.zig +++ b/zig/src/families/glm/cuda_shared.zig @@ -13,7 +13,7 @@ const b16 = @import("cuda_b16.zig"); /// Checks a shared window's segments (a prompt buffer's: whole prompts) against its `rows` rows; each holds its own. pub fn begin(r: fw.Rank, rows: usize, stream: cuda.Stream) !void { - const most: usize = if (r.b.prefill) fw.prefill_rows else fw.window_rows; + const most: usize = if (r.b.prefill) fw.prefill_rows else fw.copy_rows; if (r.b.prefill and r.segs.len > @import("cuda_state.zig").prompt_heads) return error.BadWindow; var at: usize = 0; for (r.segs) |g| { @@ -65,11 +65,11 @@ pub fn mixer(f: *const fw.Forward, r: fw.Rank, l: weights.Layer, li: usize, di: } v.p = st.proj + li * st.proj_layer * 2; // the window's rows stay with the layer until its commit try f.d.check(f.d.api.cuMemcpyDtoDAsync_v2(v.p, b.kproj + g.row0 * w * 2, g.rows * w * 2, f.stream.handle), "cuMemcpyDtoDAsync"); - const kv = li * fw.window_rows * st.heads * kda.dk; + const kv = li * fw.copy_rows * st.heads * kda.dk; v.saves.k = st.save_k + kv * 4; v.saves.v = st.save_v + kv * 2; v.saves.g = st.save_g + kv * 4; - v.saves.b = st.save_b + li * fw.window_rows * st.heads * 4; + v.saves.b = st.save_b + li * fw.copy_rows * st.heads * 4; try kda_block.chainRows(&f.m.kda, f.stream, k, v, g.rows, conv, st.recAt(cur, li), f.kda_wide); } if (out) |o| try kda_block.fill(f.set(), f.stream, k, s, 0, rows, o); diff --git a/zig/src/families/glm/cuda_snapshots.zig b/zig/src/families/glm/cuda_snapshots.zig index 3aa68235ad..4c814270b0 100644 --- a/zig/src/families/glm/cuda_snapshots.zig +++ b/zig/src/families/glm/cuda_snapshots.zig @@ -25,7 +25,7 @@ pub fn agree(e: anytype, budget: u64) !void { } pub fn bytes(e: anytype, at: usize) u64 { - const head: ?usize = if (e.mtp) @min(e.capacity, std.math.add(usize, at, @import("cuda_state.zig").window_rows) catch return std.math.maxInt(u64)) else null; + const head: ?usize = if (e.mtp) @min(e.capacity, std.math.add(usize, at, @import("cuda_state.zig").copy_rows) catch return std.math.maxInt(u64)) else null; const p = layout.Plan.init(&e.cur.st, &e.c, at, head) catch return std.math.maxInt(u64); return pages.rounded(p.len) catch std.math.maxInt(u64); } @@ -41,7 +41,7 @@ pub fn keep(e: anytype) !u64 { pub fn keepAs(e: anytype, id: u64) !void { if (e.snaps.contains(id)) return error.StateKept; - if (e.cur.st.pending != 0 or e.cur.st.mtp_len > e.cur.st.pos + @import("cuda_state.zig").window_rows) + if (e.cur.st.pending != 0 or e.cur.st.mtp_len > e.cur.st.pos + @import("cuda_state.zig").copy_rows) return error.NotAtMark; const pool = if (e.snap_pages) |*p| p else return error.NoSnapshotCache; const plan = try layout.Plan.init(&e.cur.st, &e.c, e.cur.st.pos, if (e.mtp) e.cur.st.mtp_len else null); diff --git a/zig/src/families/glm/cuda_state.zig b/zig/src/families/glm/cuda_state.zig index 37131e94f1..689be8e37d 100644 --- a/zig/src/families/glm/cuda_state.zig +++ b/zig/src/families/glm/cuda_state.zig @@ -17,8 +17,10 @@ const kda_block = @import("cuda_kda_block.zig"); /// A prompt chunk's rows (decode.PREFILL_ROWS). pub const prefill_rows = 2048; -/// A decode window's rows (Engine's max_rows). +/// A DFlash block and the MTP head's window. Copy verifies use `copy_rows` and do not widen this. pub const window_rows = 8; +/// Rows a copy verify may run (15 drafted tokens). Buffers, the sampler and the KDA save hold this many. +pub const copy_rows = 16; /// Prompts one prompt pass takes together (cuda_prompts): a prompt buffer's logits rows, one a prompt. pub const prompt_heads = 16; /// The context the buffers' dense attention partials reach (Buffers' min(capacity, 2560)). @@ -115,9 +117,9 @@ pub const Buffers = struct { return make(d, w, rows, prefill, capacity, rows); } - /// A shared window's scratch: `rows` rows of several streams, each one's attention and KDA chain on `window_rows`. + /// A shared window's scratch: `rows` rows of several streams, each one's attention and KDA chain on `copy_rows`. pub fn initShared(d: *const cuda.Driver, w: *const weights.Weights, rows: usize, capacity: usize) !Buffers { - return make(d, w, rows, false, capacity, window_rows); + return make(d, w, rows, false, capacity, copy_rows); } /// `seg`: the most rows one stream's attention takes (a decode window's rows, or a shared window's segment). @@ -299,8 +301,8 @@ pub const State = struct { const pk_rows = capacity / sparse.pool + 2; var sizes: std.ArrayList(usize) = .empty; defer sizes.deinit(gpa); - try sizes.appendSlice(gpa, &.{ 4, nk * 3 * 3 * n * 2, 2 * nk * s.kda.len() * kda.dk * kda.dv * 4, nk * window_rows * kw * 2 }); - try sizes.appendSlice(gpa, &.{ nk * window_rows * n * 4, nk * window_rows * n * 2, nk * window_rows * n * 4, nk * window_rows * s.kda.len() * 4 }); + try sizes.appendSlice(gpa, &.{ 4, nk * 3 * 3 * n * 2, 2 * nk * s.kda.len() * kda.dk * kda.dv * 4, nk * copy_rows * kw * 2 }); + try sizes.appendSlice(gpa, &.{ nk * copy_rows * n * 4, nk * copy_rows * n * 2, nk * copy_rows * n * 4, nk * copy_rows * s.kda.len() * 4 }); const ring_rows = @min(capacity, index_ring); for (0..nd + 1) |_| try sizes.appendSlice(gpa, &.{ lc_bytes, ring_rows * c.i_dim * 2, ring_rows * c.i_dim * 2, pk_rows * c.i_dim * 2 }); try sizes.appendSlice(gpa, &.{ 4, c.hidden * 2 }); // MTP position and sequence-owned first-draft row @@ -316,7 +318,7 @@ pub const State = struct { for (caches, 0..) |*x, j| x.* = .{ .lc = p[8 + 4 * j], .ls = if (kv8) p[8 + 4 * j] + capacity * c.kv_lora else 0, .ik = p[9 + 4 * j], .ig = p[10 + 4 * j], .pk = p[11 + 4 * j], .pk_rows = pk_rows }; const rings = try gpa.alloc(u64, 2 * (nd + 1)); for (0..nd + 1) |j| rings[2 * j ..][0..2].* = .{ p[9 + 4 * j], p[10 + 4 * j] }; - return .{ .ring_rows = ring_rows, .ring_row = c.i_dim * 2, .rings = rings, .arena = a, .gpa = gpa, .capacity = capacity, .pos_dev = p[0], .conv = p[1], .rec = p[2], .proj = p[3], .save_k = p[4], .save_v = p[5], .save_g = p[6], .save_b = p[7], .cur = cur, .heads = s.kda.len(), .mtp_caches = .{ .lc = p[8 + 4 * nd], .ls = if (kv8) p[8 + 4 * nd] + capacity * c.kv_lora else 0, .ik = p[9 + 4 * nd], .ig = p[10 + 4 * nd], .pk = p[11 + 4 * nd], .pk_rows = pk_rows }, .mtp_pos_dev = p[12 + 4 * nd], .prompt_hidden = p[13 + 4 * nd], .kda_layers = nk, .caches = caches, .conv_c = 3 * n, .rec_layer = s.kda.len() * kda.dk * kda.dv * 4, .proj_layer = window_rows * kw, .proj_row = kw }; + return .{ .ring_rows = ring_rows, .ring_row = c.i_dim * 2, .rings = rings, .arena = a, .gpa = gpa, .capacity = capacity, .pos_dev = p[0], .conv = p[1], .rec = p[2], .proj = p[3], .save_k = p[4], .save_v = p[5], .save_g = p[6], .save_b = p[7], .cur = cur, .heads = s.kda.len(), .mtp_caches = .{ .lc = p[8 + 4 * nd], .ls = if (kv8) p[8 + 4 * nd] + capacity * c.kv_lora else 0, .ik = p[9 + 4 * nd], .ig = p[10 + 4 * nd], .pk = p[11 + 4 * nd], .pk_rows = pk_rows }, .mtp_pos_dev = p[12 + 4 * nd], .prompt_hidden = p[13 + 4 * nd], .kda_layers = nk, .caches = caches, .conv_c = 3 * n, .rec_layer = s.kda.len() * kda.dk * kda.dv * 4, .proj_layer = copy_rows * kw, .proj_row = kw }; } pub fn deinit(st: *State) void {