From 71f188f0449de9f4fc9640b135dbde011a251896 Mon Sep 17 00:00:00 2001
From: BobClawblaw <BobClawblaw@users.noreply.github.com>
Date: Sat, 17 Oct 2026 15:24:56 +0000
Subject: [PATCH 1/4] glm cuda: fresh text prompts admitted together
 (prefill_many): one prompt pass over every prompt's rows, chains and
 attention per prompt, the head on each prompt's last row; the lane host
 batches short prompts beside chunked fills (many_rows)

---
 zig/src/core/lane_host_admit.zig      |  7 ++--
 zig/src/core/lanes/backend.zig        |  2 +
 zig/src/core/lanes/engine.zig         |  1 +
 zig/src/core/lanes/open.zig           |  6 +++
 zig/src/families/glm/cuda_engine.zig  | 23 +++++++----
 zig/src/families/glm/cuda_follow.zig  | 35 ++++++++++++++++-
 zig/src/families/glm/cuda_forward.zig |  6 ++-
 zig/src/families/glm/cuda_lanes.zig   | 43 +++++++++++++++++++--
 zig/src/families/glm/cuda_prompts.zig | 55 +++++++++++++++++++++++++++
 zig/src/families/glm/cuda_shared.zig  | 43 +++++++++++++++------
 zig/src/families/glm/cuda_state.zig   |  4 +-
 11 files changed, 196 insertions(+), 29 deletions(-)
 create mode 100644 zig/src/families/glm/cuda_prompts.zig

diff --git a/zig/src/core/lane_host_admit.zig b/zig/src/core/lane_host_admit.zig
index fde33f7010..09b3e6bd2d 100644
--- a/zig/src/core/lane_host_admit.zig
+++ b/zig/src/core/lane_host_admit.zig
@@ -23,9 +23,10 @@ fn fits(a: api.Admit, job: *const Job) bool {
     return a.fits(a.ctx, @intCast(job.request.prompt.len), job.request.max_tokens);
 }
 
-/// Whether to admit several at once: prompts batch, none fill by chunks, and two or more wait with two lanes free.
+/// Whether to admit several at once: prompts batch (beside chunked fills up to many_rows), two wait, two lanes free.
 pub fn batches(h: *LaneHost) bool {
-    if (!h.core.batchesPrompts() or h.core.fills()) return false;
+    if (!h.core.batchesPrompts()) return false;
+    if (h.core.fills() and h.core.backend.vtable.many_rows == 0) return false;
     h.lock();
     defer h.unlock();
     const lanes_now = @min(h.info_.lanes, h.memory_lanes orelse h.info_.lanes);
@@ -49,7 +50,7 @@ pub fn many(h: *LaneHost) bool {
         };
         took = true;
         const at = h.setup(job) orelse continue;
-        if (h.startsAlone(job)) {
+        if (h.startsAlone(job) or (h.core.fills() and !h.core.batchable(&job.stream))) { // long, image, resumed: filled
             _ = h.launch(job, at);
             continue;
         }
diff --git a/zig/src/core/lanes/backend.zig b/zig/src/core/lanes/backend.zig
index 31cf0f4786..f138e86a7d 100644
--- a/zig/src/core/lanes/backend.zig
+++ b/zig/src/core/lanes/backend.zig
@@ -70,6 +70,8 @@ pub const Backend = struct {
         prefill_step: ?*const fn (ptr: *anyopaque, s: *Stream) anyerror!bool = null,
         /// Several prompts in one pass, each as after `prefill`; false or an error leaves none prefilled.
         prefill_many: ?*const fn (ptr: *anyopaque, ss: []const *Stream) anyerror!bool = null,
+        /// Prompts of at most this many tokens prefill_many takes while prompts otherwise fill by chunks (0: none).
+        many_rows: u32 = 0,
         /// Draw the first token from the prompt's last row at `position`; a handle to it.
         first: *const fn (ptr: *anyopaque, s: *Stream, position: u64) anyerror!u64,
         /// Feed one token and queue the draw of the next at `position` (a one-token round); a handle to it.
diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig
index 7e47830941..ed46752c10 100644
--- a/zig/src/core/lanes/engine.zig
+++ b/zig/src/core/lanes/engine.zig
@@ -80,6 +80,7 @@ pub const Engine = struct {
 
     pub const addStream = open.add;
     pub const addStreams = open.addStreams;
+    pub const batchable = open.batchable;
     pub const max_together = open.max_together;
 
     /// Whether the backend prefills several prompts in one pass (addStreams batches them).
diff --git a/zig/src/core/lanes/open.zig b/zig/src/core/lanes/open.zig
index e0b0ba45e8..a0087231a1 100644
--- a/zig/src/core/lanes/open.zig
+++ b/zig/src/core/lanes/open.zig
@@ -40,6 +40,12 @@ pub fn addStreams(e: *Engine, ss: []const *Stream, errs: []?anyerror) void {
     };
 }
 
+/// Whether `s` may join a batched prompt pass beside chunked fills: short, text only, nothing kept to resume from.
+pub fn batchable(e: *const Engine, s: *const Stream) bool {
+    const n = e.backend.vtable.many_rows;
+    return n > 0 and s.prompt_len <= n and s.images.len == 0 and s.reuse.saved == null;
+}
+
 fn addEach(e: *Engine, ss: []const *Stream, errs: []?anyerror) void {
     for (ss, errs) |s, *x| add(e, s) catch |err| {
         x.* = err;
diff --git a/zig/src/families/glm/cuda_engine.zig b/zig/src/families/glm/cuda_engine.zig
index f9e92556e0..a9d67e3889 100644
--- a/zig/src/families/glm/cuda_engine.zig
+++ b/zig/src/families/glm/cuda_engine.zig
@@ -32,6 +32,10 @@ const pair_bytes = 16;
 
 /// A prompt pass's chunk rows (decode.prefill's).
 pub const prompt_rows = fw.prefill_rows;
+/// Whole prompts one prompt pass takes together (cuda_prompts).
+pub const max_together = @import("cuda_state.zig").prompt_heads;
+/// One of them: its stream's slot and its ids.
+pub const Prompt = struct { slot: u32, ids: []const u32 };
 
 /// A shared verify's max rows and streams: under hc_pre's 64-row block change, 8 streams of 4 rows.
 pub const shared_rows = 32;
@@ -49,6 +53,7 @@ const Seq = struct {
     sampling: ?sampler.Sampling = null, // null: greedy
     in_head: usize = 0, // drafted rows the MTP caches hold past the kept ones (st.mtp_drafted)
     rows_at: u64 = 0, // its last verify's final-norm rows (bf16 [rows][hidden]), which the next draft reads
+    held_at: u64 = 0, // its prompt's last row's hidden (a row of `hold`), which the first draft reads
 };
 
 /// One rank's copy of a kept prompt state: the state arena's bytes and the host fields beside them.
@@ -82,7 +87,7 @@ pub const Engine = struct {
     bd: fw.Buffers, // verify windows
     bm: fw.Buffers, // the MTP head's rows
     bs: ?fw.Buffers = null, // shared verify windows (with the MTP head, as the lane core shares only then)
-    hold: cuda.DeviceBuffer, // the prompt's last row's hidden (decode's e.last_hidden), the first draft's source
+    hold: cuda.DeviceBuffer, // prompts' last rows' hidden (decode's e.last_hidden), a row a prompt pass's prompt
     pairs: cuda.DeviceBuffer, // every rank's (value, id) of a greedy draw
     smp: sampler.Sampler,
     seqs: std.ArrayList(?*Seq) = .empty, // every stream's sequence by slot (the lane backend's), slot 0 made at init
@@ -161,7 +166,7 @@ pub const Engine = struct {
         errdefer e.bm.deinit();
         if (o.mtp) e.bs = try fw.Buffers.initShared(d, &e.w, shared_rows, o.capacity);
         errdefer if (e.bs) |*b| b.deinit();
-        e.hold = try cuda.DeviceBuffer.alloc(d, c.hidden * 2);
+        e.hold = try cuda.DeviceBuffer.alloc(d, max_together * c.hidden * 2);
         errdefer e.hold.free();
         e.pairs = try cuda.DeviceBuffer.alloc(d, pair_bytes * world);
         errdefer e.pairs.free();
@@ -205,7 +210,7 @@ pub const Engine = struct {
         gpa.destroy(e);
     }
 
-    fn exchange(e: *const Engine) fw.Exchange {
+    pub fn exchange(e: *const Engine) fw.Exchange {
         return .{ .nccl = .{ .lib = e.lib, .comm = e.comm, .group = e.group } };
     }
 
@@ -310,6 +315,7 @@ pub const Engine = struct {
         if (f.prof) |p| p.mark(e.stream, .draw);
         if (e.mtp) {
             if (last) try e.copy(e.hold.ptr, e.bp.fnormed + (rows - 1) * e.c.hidden * 2, e.c.hidden * 2);
+            if (last) e.cur.held_at = e.hold.ptr;
             const nxt = as_i32[1..];
             if (nxt.len > 0) { // decode._absorb_rows
                 const mp = f.prof;
@@ -326,7 +332,7 @@ pub const Engine = struct {
     }
 
     /// The chunk gate on every rank: a kept state at the mark, the heat wait, rank 0's cancel; the id exchanged first.
-    fn gate(e: *Engine, cancel: ?Cancel) !void {
+    pub fn gate(e: *Engine, cancel: ?Cancel) !void {
         const kept = try e.shared(@floatFromInt(e.announce));
         e.announce = 0;
         if (kept > 0 and e.share.rank != 0) try e.keepAs(@intFromFloat(kept));
@@ -476,6 +482,9 @@ pub const Engine = struct {
         }
     }
 
+    /// Whole prompts on their slots in one prompt pass (cuda_prompts); `firsts[k]`: prompt k's first token.
+    pub const prefillMany = @import("cuda_prompts.zig").prefillMany;
+
     /// forward.commit of the last window's first `n` rows (the KDA replay when fewer than it ran).
     pub fn keep(e: *Engine, n: usize) !void {
         var one: [1]fw.Rank = undefined;
@@ -490,7 +499,7 @@ pub const Engine = struct {
         var one: [1]fw.Rank = undefined;
         const f = e.view(&e.bm, &one);
         const D = e.c.hidden;
-        const hidden = if (from_prompt) e.hold.ptr else e.cur.rows_at;
+        const hidden = if (from_prompt) e.cur.held_at else e.cur.rows_at;
         try e.cur.st.setMtpLen(e.stream, e.cur.st.mtp_len - e.cur.in_head);
         e.cur.in_head = 0;
         const next: []const i32 = @ptrCast(follow);
@@ -511,7 +520,7 @@ pub const Engine = struct {
     }
 
     /// `rows` logits rows of this rank's vocabulary -> the tokens at positions first, first + 1, ...
-    fn draw(e: *Engine, logits: u64, rows: usize, first: u64, out: []u32) !void {
+    pub fn draw(e: *Engine, logits: u64, rows: usize, first: u64, out: []u32) !void {
         if (e.cur.sampling) |s| {
             try e.smp.draw(e.stream, e.exchange(), e.spans, &.{.{ .rank = e.share.rank, .logits = logits }}, rows, first, sampler.rule(s), out);
             return e.checkGathers(); // the draw read rows the forward's gathers made
@@ -560,7 +569,7 @@ pub const Engine = struct {
         return if (s.len == 0) heat.default_root else s;
     }
 
-    fn copy(e: *const Engine, dst: u64, src: u64, bytes: usize) !void {
+    pub fn copy(e: *const Engine, dst: u64, src: u64, bytes: usize) !void {
         try e.d.check(e.d.api.cuMemcpyDtoDAsync_v2(dst, src, bytes, e.stream.handle), "cuMemcpyDtoDAsync");
     }
 };
diff --git a/zig/src/families/glm/cuda_follow.zig b/zig/src/families/glm/cuda_follow.zig
index 1f31bad634..6c80df6651 100644
--- a/zig/src/families/glm/cuda_follow.zig
+++ b/zig/src/families/glm/cuda_follow.zig
@@ -6,9 +6,10 @@ const engine = @import("cuda_engine.zig");
 const images_mod = @import("cuda_images.zig");
 const vcfg = @import("vision_config.zig");
 const sampler = @import("cuda_sampler.zig");
+const prompts = @import("cuda_prompts.zig");
 const Engine = engine.Engine;
 
-pub const Kind = enum(u32) { begin = 1, prefill, verify, keep, draft, end, drop, release, chunk, shared, image };
+pub const Kind = enum(u32) { begin = 1, prefill, verify, keep, draft, end, drop, release, chunk, shared, image, many };
 
 /// One step as it crosses: 64 bytes, then `n` u32 ids.
 pub const Header = extern struct {
@@ -210,6 +211,22 @@ pub const Leader = struct {
         try x.e.verifyShared(parts);
     }
 
+    /// Whole prompts in one prompt pass on every rank (Engine.prefillMany): their slots and lengths, then the ids.
+    pub fn prefillMany(x: Leader, parts: []const engine.Prompt, firsts: []u32) !void {
+        if (!prompts.fits(parts, x.e.capacity)) return error.BadPrompts;
+        try x.flush();
+        var buf: [2 * engine.max_together + engine.prompt_rows]u32 = undefined;
+        var n: usize = 2 * parts.len;
+        for (parts, 0..) |p, i| {
+            buf[2 * i] = p.slot;
+            buf[2 * i + 1] = @intCast(p.ids.len);
+            @memcpy(buf[n..][0..p.ids.len], p.ids);
+            n += p.ids.len;
+        }
+        try x.send(.{ .kind = @backingInt(Kind.many), .depth = @intCast(parts.len) }, buf[0..n]);
+        try x.e.prefillMany(parts, firsts);
+    }
+
     pub fn keep(x: Leader, n: usize) !void {
         try x.flush();
         try x.send(.{ .kind = @backingInt(Kind.keep), .a = n }, &.{});
@@ -236,7 +253,7 @@ pub fn follow(e: *Engine, link: *Link) !void {
         const step = try link.receive();
         const h = step.h;
         const kind: Kind = @fromBackingInt(@intCast(h.kind));
-        if (kind != .end and kind != .drop and kind != .shared) try e.bind(h.slot);
+        if (kind != .end and kind != .drop and kind != .shared and kind != .many) try e.bind(h.slot);
         switch (kind) {
             .begin => try e.beginFollowing(if (h.flags & 1 != 0) .{ .seed = h.seed, .temperature = h.temperature, .top_p = h.top_p, .min_p = h.min_p, .top_k = h.top_k } else null),
             .prefill => {
@@ -274,6 +291,20 @@ pub fn follow(e: *Engine, link: *Link) !void {
                 }
                 try e.verifyShared(parts[0..k]);
             },
+            .many => {
+                const k = h.depth;
+                if (k == 0 or k > engine.max_together or step.ids.len < 2 * k) return error.BadStep;
+                var parts: [engine.max_together]engine.Prompt = undefined;
+                var at: usize = 2 * k;
+                for (parts[0..k], 0..) |*p, i| {
+                    const n = step.ids[2 * i + 1];
+                    if (at + n > step.ids.len) return error.BadStep;
+                    p.* = .{ .slot = step.ids[2 * i], .ids = step.ids[at..][0..n] };
+                    at += n;
+                }
+                var firsts: [engine.max_together]u32 = undefined;
+                try e.prefillMany(parts[0..k], &firsts);
+            },
             .keep => try e.keep(@intCast(h.a)),
             .draft => {
                 const ids = try e.gpa.dupe(u32, step.ids);
diff --git a/zig/src/families/glm/cuda_forward.zig b/zig/src/families/glm/cuda_forward.zig
index bbc5a92942..f851254bf9 100644
--- a/zig/src/families/glm/cuda_forward.zig
+++ b/zig/src/families/glm/cuda_forward.zig
@@ -441,7 +441,7 @@ pub const Forward = struct {
         var li: usize = 0;
         var di: usize = 0;
         for (layers, 0..) |_, j| {
-            for (f.ranks) |r| try f.mixer(r, r.w.layers[j], li, di, R, @min(latent.chunksFor(r.st.pos + R), r.b.nch), null);
+            for (f.ranks) |r| if (r.segs.len > 0) try shared.mixer(f, r, r.w.layers[j], li, di, R, null) else try f.mixer(r, r.w.layers[j], li, di, R, @min(latent.chunksFor(r.st.pos + R), r.b.nch), null);
             try f.exchangePieces(p, j, .mixer, cut);
             try f.gluePieces(p, j, .mixer, cut);
             for (f.ranks) |r| try f.ffn(r, r.w.layers[j], R, null);
@@ -516,6 +516,10 @@ pub const Forward = struct {
         for (f.ranks) |r| {
             const b = r.b;
             try glue.rmsnorm(f.set(), f.stream, b.hidden, c.hidden, r.w.norm, b.fnormed, c.hidden, b.fxs, c.eps, R, c.hidden);
+            if (b.prefill and r.segs.len > 0) {
+                try shared.heads(f, r);
+                continue;
+            }
             const head_rows: usize = if (b.prefill) 1 else R;
             const first = if (b.prefill) R - 1 else 0;
             try b16.matmul(f.set(), f.stream, b.fnormed + first * c.hidden * 2, c.hidden, r.w.head, b.logits, b.kda.mm_part, head_rows, r.w.share.vocab.len(), c.hidden, false);
diff --git a/zig/src/families/glm/cuda_lanes.zig b/zig/src/families/glm/cuda_lanes.zig
index c257fca5ab..2486951ef9 100644
--- a/zig/src/families/glm/cuda_lanes.zig
+++ b/zig/src/families/glm/cuda_lanes.zig
@@ -22,6 +22,7 @@ const Lane = struct {
     held: [engine.max_drafts]u32 = undefined,
     held_n: usize = 0,
     fill: ?usize = null, // a prompt pass going a chunk at a time: where its next chunk starts
+    first: ?u64 = null, // its first token's handle once its prompt pass ends
 };
 
 pub const Lanes = struct {
@@ -43,6 +44,7 @@ pub const Lanes = struct {
 
     const whole_vt: be.Backend.VTable = .{
         .prefill = prefillFn,
+        .prefill_many = prefillManyFn,
         .first = firstFn,
         .queue = queueFn,
         .read = readFn,
@@ -54,6 +56,7 @@ pub const Lanes = struct {
     const steps_vt: be.Backend.VTable = vt: {
         var v = whole_vt;
         v.prefill_step = prefillStepFn;
+        v.many_rows = engine.prompt_rows; // short fresh prompts go together; longer ones fill by chunks
         break :vt v;
     };
 
@@ -124,7 +127,39 @@ pub const Lanes = struct {
         const self = of(ptr);
         const from = try self.opening(s);
         try self.images(s, from);
-        _ = self.take(try self.x.prefill(s.prompt(), from, .{ .ptr = s, .check = cancelled }, marksOf(s)));
+        const first = try self.x.prefill(s.prompt(), from, .{ .ptr = s, .check = cancelled }, marksOf(s));
+        self.lanes.getPtr(s).?.first = self.take(first);
+    }
+
+    /// Fresh text prompts together, in passes of at most prompt_rows rows; false: any image or kept state (each alone).
+    fn prefillManyFn(ptr: *anyopaque, ss: []const *lanes.Stream) anyerror!bool {
+        const self = of(ptr);
+        if (ss.len > engine.max_together) return false;
+        for (ss) |s| if (s.images.len > 0 or s.reuse.saved != null or s.prompt_len > engine.prompt_rows) return false;
+        var opened: usize = 0;
+        errdefer for (ss[0..opened]) |s| releaseFn(ptr, s);
+        var parts: [engine.max_together]engine.Prompt = undefined;
+        var firsts: [engine.max_together]u32 = undefined;
+        var lo: usize = 0;
+        var rows: usize = 0;
+        for (ss, 0..) |s, i| {
+            if (rows + s.prompt_len > engine.prompt_rows) { // this pass is full: run it, the next starts here
+                try self.many(ss[lo..i], parts[lo..i], firsts[lo..i]);
+                lo = i;
+                rows = 0;
+            }
+            if (try self.opening(s) != null) return error.BadPrompts; // no kept state (checked above)
+            opened += 1;
+            parts[i] = .{ .slot = self.lanes.getPtr(s).?.slot, .ids = s.prompt() };
+            rows += s.prompt_len;
+        }
+        try self.many(ss[lo..], parts[lo..ss.len], firsts[lo..ss.len]);
+        return true;
+    }
+
+    fn many(self: *Lanes, ss: []const *lanes.Stream, parts: []const engine.Prompt, firsts: []u32) !void {
+        try self.x.prefillMany(parts, firsts);
+        for (ss, firsts) |s, t| self.lanes.getPtr(s).?.first = self.take(t);
     }
 
     /// One chunk of the stream's prompt pass (fillOne runs other rounds between); true once the first token is drawn.
@@ -138,7 +173,7 @@ pub const Lanes = struct {
         const at = going orelse if (from) |r| r.at else 0;
         errdefer l.fill = null;
         if (try self.x.chunk(s.prompt(), at, from, .{ .ptr = s, .check = cancelled }, marksOf(s))) |first| {
-            _ = self.take(first);
+            l.first = self.take(first);
             l.fill = null;
             return true;
         }
@@ -155,6 +190,7 @@ pub const Lanes = struct {
         gop.value_ptr.pending_rows = null;
         gop.value_ptr.held_n = 0;
         gop.value_ptr.fill = null;
+        gop.value_ptr.first = null;
         try self.x.bind(gop.value_ptr.slot);
         if (self.x.e.images) |v| v.clear(); // no earlier request's rows splice into this one
         try self.x.begin(s.sampling);
@@ -195,7 +231,8 @@ pub const Lanes = struct {
     fn firstFn(ptr: *anyopaque, s: *lanes.Stream, position: u64) anyerror!u64 {
         const self = of(ptr);
         if (position != s.prompt_len) return error.PositionMismatch;
-        return self.next - 1;
+        const l = self.lanes.getPtr(s) orelse return error.UnknownStream;
+        return l.first orelse error.NoFirstToken;
     }
 
     fn queueFn(ptr: *anyopaque, s: *lanes.Stream, feed: be.Feed, position: u64) anyerror!u64 {
diff --git a/zig/src/families/glm/cuda_prompts.zig b/zig/src/families/glm/cuda_prompts.zig
new file mode 100644
index 0000000000..312752d9ca
--- /dev/null
+++ b/zig/src/families/glm/cuda_prompts.zig
@@ -0,0 +1,55 @@
+//! GLM-5.3-Flash's prefill_many: fresh streams' whole prompts in one prompt pass, each left as its own pass leaves it.
+
+const std = @import("std");
+const fw = @import("cuda_forward.zig");
+const engine = @import("cuda_engine.zig");
+const Engine = engine.Engine;
+
+/// Whether `parts` go in one pass: 1 to max_together fresh prompts, prompt_rows in all, each under the capacity.
+pub fn fits(parts: []const engine.Prompt, capacity: usize) bool {
+    if (parts.len == 0 or parts.len > engine.max_together) return false;
+    var rows: usize = 0;
+    for (parts, 0..) |p, i| {
+        if (p.ids.len == 0 or p.ids.len + fw.window_rows >= capacity) return false;
+        for (parts[0..i]) |q| if (q.slot == p.slot) return false;
+        rows += p.ids.len;
+    }
+    return rows <= fw.prefill_rows;
+}
+
+/// The prompts' pass on their slots (each begun, at position 0); `firsts[k]`: prompt k's first token.
+pub fn prefillMany(e: *Engine, parts: []const engine.Prompt, firsts: []u32) !void {
+    if (!fits(parts, e.capacity) or firsts.len < parts.len) return error.BadPrompts;
+    var segs: [engine.max_together]fw.Seg = undefined;
+    var ids: [fw.prefill_rows]u32 = undefined;
+    var rows: usize = 0;
+    for (parts, 0..) |p, k| {
+        try e.bind(p.slot);
+        if (e.cur.st.pos != 0 or e.cur.st.mtp_len != 0) return error.BadPrompts;
+        segs[k] = .{ .st = &e.cur.st, .row0 = rows, .rows = p.ids.len };
+        @memcpy(ids[rows..][0..p.ids.len], p.ids);
+        rows += p.ids.len;
+    }
+    try e.gate(null);
+    var one: [1]fw.Rank = .{.{ .w = &e.w, .st = segs[0].st, .b = &e.bp, .segs = segs[0..parts.len] }};
+    var f: fw.Forward = .{ .d = e.d, .m = &e.m, .stream = e.stream, .ranks = &one, .exchange = e.exchange(), .pieces = &e.pieces, .split = e.settings.split, .kda_wide = e.settings.kda_wide };
+    try f.run(@ptrCast(ids[0..rows]));
+    const D = e.c.hidden;
+    for (parts, segs[0..parts.len], 0..) |p, g, k| {
+        try e.bind(p.slot);
+        try e.draw(e.bp.logits + k * e.share.vocab.len() * 2, 1, p.ids.len, firsts[k..][0..1]);
+        if (!e.mtp) continue;
+        try e.copy(e.hold.ptr + k * D * 2, e.bp.fnormed + (g.row0 + g.rows - 1) * D * 2, D * 2);
+        e.cur.held_at = e.hold.ptr + k * D * 2;
+    }
+    f.pieces = null;
+    for (parts, segs[0..parts.len]) |p, g| {
+        one[0] = .{ .w = &e.w, .st = g.st, .b = &e.bp };
+        // decode._absorb_rows; the head's pass writes final-norm row 0, which the hold rows above already copied
+        if (e.mtp and g.rows > 1) {
+            try f.mtp(@ptrCast(p.ids[1..]), &.{e.bp.fnormed + g.row0 * D * 2});
+            try g.st.setMtpLen(e.stream, g.st.mtp_len + g.rows - 1);
+        }
+        try f.commit(g.rows);
+    }
+}
diff --git a/zig/src/families/glm/cuda_shared.zig b/zig/src/families/glm/cuda_shared.zig
index 86b19c640d..2b540a2e5e 100644
--- a/zig/src/families/glm/cuda_shared.zig
+++ b/zig/src/families/glm/cuda_shared.zig
@@ -1,4 +1,4 @@
-//! GLM-5.3-Flash's shared verify windows: several streams' rows in one forward, chains per segment.
+//! GLM-5.3-Flash's shared windows: several streams' verify windows or whole prompts in one forward, chains per segment.
 
 const std = @import("std");
 const cuda = @import("cuda");
@@ -8,25 +8,36 @@ const kda = @import("cuda_kda.zig");
 const latent = @import("cuda_latent.zig");
 const kda_block = @import("cuda_kda_block.zig");
 const dsa_block = @import("cuda_dsa_block.zig");
+const glue = @import("cuda_glue.zig");
+const b16 = @import("cuda_b16.zig");
 
-/// Checks a shared window's segments against its `rows` rows, then each sequence holds its rows until its commit.
+/// Checks a shared window's segments (a prompt buffer's: whole prompts) against its `rows` rows; each holds its own.
 pub fn begin(r: fw.Rank, rows: usize) !void {
-    if (r.b.prefill) return error.BadWindow;
+    const most: usize = if (r.b.prefill) fw.prefill_rows else fw.window_rows;
+    if (r.b.prefill and r.segs.len > @import("cuda_state.zig").prompt_heads) return error.BadWindow;
     var at: usize = 0;
     for (r.segs) |g| {
-        if (g.row0 != at or g.rows == 0 or g.rows > fw.window_rows) return error.BadWindow;
+        if (g.row0 != at or g.rows == 0 or g.rows > most) return error.BadWindow;
         if (g.st.pos + g.rows > g.st.capacity) return error.PastCapacity;
         at += g.rows;
     }
     if (at != rows) return error.BadWindow;
     for (r.segs) |g| {
         g.st.pending = g.rows;
-        g.st.prompt = false;
+        g.st.prompt = r.b.prefill;
     }
 }
 
-/// Forward.mixer on a shared window: all rows' projections, each segment's chain or attention alone, all o_proj.
-pub fn mixer(f: *const fw.Forward, r: fw.Rank, l: weights.Layer, li: usize, di: usize, rows: usize, out: u64) !void {
+/// Forward.finish's head on a prompt buffer: each segment's last final-norm row, logits row k for segment k.
+pub fn heads(f: *const fw.Forward, r: fw.Rank) !void {
+    const D = r.w.config.hidden;
+    const b = r.b;
+    for (r.segs, 0..) |g, k| try f.d.check(f.d.api.cuMemcpyDtoDAsync_v2(b.normed + k * D * 2, b.fnormed + (g.row0 + g.rows - 1) * D * 2, D * 2, f.stream.handle), "cuMemcpyDtoDAsync");
+    try b16.matmul(f.set(), f.stream, b.normed, D, r.w.head, b.logits, b.kda.mm_part, r.segs.len, r.w.share.vocab.len(), D, false);
+}
+
+/// Forward.mixer on a shared window: all rows' projections, each segment's chain or attention, o_proj (null: pieced).
+pub fn mixer(f: *const fw.Forward, r: fw.Rank, l: weights.Layer, li: usize, di: usize, rows: usize, out: ?u64) !void {
     const c = &r.w.config;
     const b = r.b;
     switch (l.mixer) {
@@ -40,20 +51,28 @@ pub fn mixer(f: *const fw.Forward, r: fw.Rank, l: weights.Layer, li: usize, di:
                 const st = g.st;
                 const cur = st.cur[li];
                 var v = s;
-                v.p = st.proj + li * st.proj_layer * 2; // the window's rows stay with the layer until its commit
-                try f.d.check(f.d.api.cuMemcpyDtoDAsync_v2(v.p, b.kproj + g.row0 * w * 2, g.rows * w * 2, f.stream.handle), "cuMemcpyDtoDAsync");
                 v.ka += g.row0 * n * 2;
                 v.kg += g.row0 * n * 2;
                 v.saves.out += g.row0 * n * 2;
                 v.saves.state_out = st.recAt(1 - cur, li);
+                const conv = st.conv + li * 3 * st.conv_c * 2;
+                if (b.prefill) { // a whole prompt: its rows in place, the layer committed now (as Forward.mixer)
+                    v.p = b.kproj + g.row0 * w * 2;
+                    try kda_block.chainRows(&f.m.kda, f.stream, k, v, g.rows, conv, st.recAt(cur, li), false);
+                    try glue.convShift(f.set(), f.stream, conv, v.p, g.rows, g.rows * w, w, 1, 3 * n);
+                    st.cur[li] = 1 - cur;
+                    continue;
+                }
+                v.p = st.proj + li * st.proj_layer * 2; // the window's rows stay with the layer until its commit
+                try f.d.check(f.d.api.cuMemcpyDtoDAsync_v2(v.p, b.kproj + g.row0 * w * 2, g.rows * w * 2, f.stream.handle), "cuMemcpyDtoDAsync");
                 const kv = li * fw.window_rows * st.heads * kda.dk;
                 v.saves.k = st.save_k + kv * 4;
                 v.saves.v = st.save_v + kv * 2;
                 v.saves.g = st.save_g + kv * 4;
                 v.saves.b = st.save_b + li * fw.window_rows * st.heads * 4;
-                try kda_block.chainRows(&f.m.kda, f.stream, k, v, g.rows, st.conv + li * 3 * st.conv_c * 2, st.recAt(cur, li), f.kda_wide);
+                try kda_block.chainRows(&f.m.kda, f.stream, k, v, g.rows, conv, st.recAt(cur, li), f.kda_wide);
             }
-            try kda_block.fill(f.set(), f.stream, k, s, 0, rows, out);
+            if (out) |o| try kda_block.fill(f.set(), f.stream, k, s, 0, rows, o);
         },
         .mla => |a| {
             var sparse_any = false;
@@ -64,7 +83,7 @@ pub fn mixer(f: *const fw.Forward, r: fw.Rank, l: weights.Layer, li: usize, di:
                 try dsa_block.attend(.{ .set = f.set(), .ops = f.m.ops() }, f.stream, a, g.st.caches[di], b.dsa, g.row0, g.rows, g.st.pos_dev, g.st.pos, nch, b.part_rows);
             }
             try latent.expandV(f.set(), f.stream, b.dsa.ol, a.wv, b.dsa.vn, rows, a.heads, a.dv, a.lw);
-            try dsa_block.fill(f.set(), f.stream, a, b.dsa, 0, rows, out);
+            if (out) |o| try dsa_block.fill(f.set(), f.stream, a, b.dsa, 0, rows, o);
         },
     }
 }
diff --git a/zig/src/families/glm/cuda_state.zig b/zig/src/families/glm/cuda_state.zig
index b90123aa15..eba2038fc4 100644
--- a/zig/src/families/glm/cuda_state.zig
+++ b/zig/src/families/glm/cuda_state.zig
@@ -19,6 +19,8 @@ const kda_block = @import("cuda_kda_block.zig");
 pub const prefill_rows = 2048;
 /// A decode window's rows (Engine's max_rows).
 pub const window_rows = 8;
+/// Prompts one prompt pass takes together (cuda_prompts): a prompt buffer's logits rows, one a prompt.
+pub const prompt_heads = 16;
 /// The context the buffers' dense attention partials reach (Buffers' min(capacity, 2560)).
 pub const dense_reach = 2560;
 
@@ -161,7 +163,7 @@ pub const Buffers = struct {
             rows * sh * 2,
             rows * (sh / 64) * 4, rows * D * 4, // sxs, sy
             rows * D * 2,         rows * D * 2,
-            rows * (D / 64) * 4,  (if (prefill) 1 else rows) * s.vocab.len() * 2,
+            rows * (D / 64) * 4,  (if (prefill) prompt_heads else rows) * s.vocab.len() * 2,
             rows * D * 2, rows * D * 2, rows * 2 * D * 2, rows * D * 2, // hin, me, mcat, mx
             rows * D * 4, // part: this rank's fp32 partial rows a split exchange sends from
             x3[0], x3[1], x3[2], x3[3], // EXL3 experts: 8-slot picks, rotated inputs, a decode window's partials

From 5598a05e5efd43db26f8f11a8ce0774885007c7c Mon Sep 17 00:00:00 2001
From: BobClawblaw <BobClawblaw@users.noreply.github.com>
Date: Sat, 17 Oct 2026 15:53:57 +0000
Subject: [PATCH 2/4] lanes: the CUDA wrapper forwards many_rows (the
 forwarding test now checks integer fields, with a backend that both steps and
 batches); a failed batched pass logs its error before each goes alone; host
 test: short prompts batch beside chunked fills

---
 zig/src/core/lane_host_many_test.zig | 16 ++++++++++++++--
 zig/src/core/lanes/fake.zig          |  5 +++++
 zig/src/core/lanes/open.zig          |  6 +++++-
 zig/src/native/cuda_bound.zig        |  5 +++--
 4 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/zig/src/core/lane_host_many_test.zig b/zig/src/core/lane_host_many_test.zig
index 25b6993b0c..0cee744337 100644
--- a/zig/src/core/lane_host_many_test.zig
+++ b/zig/src/core/lane_host_many_test.zig
@@ -46,13 +46,19 @@ fn expectOwn(prompt: []const u32, tokens: []const u32) !void {
 }
 
 /// `n` requests queued before the host starts (`sampled`: one the fake refuses), replies checked, batched passes.
-fn serve(n: usize, memory_lanes: ?usize, sampled: ?usize) !struct { passes: usize, streams: usize } {
+const Passes = struct { passes: usize, streams: usize };
+
+fn serve(n: usize, memory_lanes: ?usize, sampled: ?usize) !Passes {
+    return serveOn(n, memory_lanes, sampled, false);
+}
+
+fn serveOn(n: usize, memory_lanes: ?usize, sampled: ?usize, steps: bool) !Passes {
     var cfg = try lanes.Config.init(gpa, .{ .exact_width = 8, .gpu_tokens = true, .hidden_rows = true }, 8, 7);
     defer cfg.deinit(gpa);
     var target: lanes.fake.Fake = .{ .gpa = gpa, .memory_lanes = memory_lanes, .refuse_sampled = sampled != null };
     defer target.deinit();
     var clock: lanes.fake.FixedClock = .{};
-    var core = lanes.Engine.init(gpa, &cfg, target.batched(), clock.clock());
+    var core = lanes.Engine.init(gpa, &cfg, if (steps) target.batchedSteps() else target.batched(), clock.clock());
     defer core.deinit();
     var host = LaneHost.init(gpa, std.testing.io, &core, .{ .lanes = 4 });
     var prompts: [6][5]u32 = undefined;
@@ -86,6 +92,12 @@ test "queued requests share one prompt pass, each reply the target's own continu
     try std.testing.expectEqual(@as(usize, 4), r.streams);
 }
 
+test "a backend that also fills by chunks: short queued prompts still share one pass" {
+    const r = try serveOn(4, null, null, true);
+    try std.testing.expectEqual(@as(usize, 1), r.passes);
+    try std.testing.expectEqual(@as(usize, 4), r.streams);
+}
+
 test "more requests than lanes: the free lanes' prompts in one pass, the rest after" {
     const r = try serve(6, null, null);
     try std.testing.expect(r.passes >= 1);
diff --git a/zig/src/core/lanes/fake.zig b/zig/src/core/lanes/fake.zig
index 234fe8d63e..cc5e26a648 100644
--- a/zig/src/core/lanes/fake.zig
+++ b/zig/src/core/lanes/fake.zig
@@ -106,6 +106,11 @@ pub const Fake = struct {
         return .{ .ptr = x, .vtable = &.{ .prefill = prefill, .prefill_step = prefillStep, .first = first, .queue = queue, .read = read, .verify = verify, .keep = keep, .draft = draft, .features = features, .release = release, .first_row = firstRow, .reports_fill = true } };
     }
 
+    /// Both: prompts a chunk a call, and short ones (up to 8 tokens) several in one pass (`many_rows`).
+    pub fn batchedSteps(x: *Fake) be.Backend {
+        return .{ .ptr = x, .vtable = &.{ .prefill = prefill, .prefill_step = prefillStep, .prefill_many = prefillMany, .many_rows = 8, .first = first, .queue = queue, .read = read, .verify = verify, .keep = keep, .draft = draft, .features = features, .release = release, .first_row = firstRow, .reports_fill = true } };
+    }
+
     fn self(ptr: *anyopaque) *Fake {
         return @ptrCast(@alignCast(ptr));
     }
diff --git a/zig/src/core/lanes/open.zig b/zig/src/core/lanes/open.zig
index a0087231a1..1d4589a98b 100644
--- a/zig/src/core/lanes/open.zig
+++ b/zig/src/core/lanes/open.zig
@@ -1,4 +1,5 @@
 //! A stream's way into the rounds: its prompt pass whole or a chunk a round, then its first token and drafts.
+const std = @import("std");
 const Engine = @import("engine.zig").Engine;
 const sm = @import("stream.zig");
 const be = @import("backend.zig");
@@ -33,7 +34,10 @@ pub fn addStreams(e: *Engine, ss: []const *Stream, errs: []?anyerror) void {
     if (ss.len < 2 or ss.len > max_together) return addEach(e, ss, errs);
     _ = e.arena.reset(.retain_capacity);
     for (ss) |s| trail.event(e, &.{ f("ev", str("add")), f("stream", str(s.id)) }) catch {};
-    const together = many(e.backend.ptr, ss) catch false; // one stream's refusal is its own: each goes alone
+    const together = many(e.backend.ptr, ss) catch |err| blk: { // one stream's refusal is its own: each goes alone
+        std.log.warn("{d} prompts together: {s}; each goes alone", .{ ss.len, @errorName(err) });
+        break :blk false;
+    };
     if (!together) return addEach(e, ss, errs);
     for (ss, errs) |s, *x| opened(e, s) catch |err| { // a stream cancelled in the pass gets no first token
         x.* = err;
diff --git a/zig/src/native/cuda_bound.zig b/zig/src/native/cuda_bound.zig
index c825d65f4b..8992a81b82 100644
--- a/zig/src/native/cuda_bound.zig
+++ b/zig/src/native/cuda_bound.zig
@@ -24,6 +24,7 @@ pub fn vtable(comptime H: type, v: be.Backend.VTable) be.Backend.VTable {
                 return x.inner.vtable.prefill_many.?(x.inner.ptr, ss);
             }
         }.f else null,
+        .many_rows = v.many_rows,
         .first = struct {
             fn f(p: *anyopaque, s: *lanes.Stream, position: u64) anyerror!u64 {
                 const x = H.bind(p);
@@ -134,12 +135,12 @@ test "every entry the family's backend gives is forwarded, and none it leaves ou
     };
     var fake: lanes.fake.Fake = .{ .gpa = std.testing.allocator, .prefill_chunks = 3 };
     defer fake.deinit();
-    for ([_]be.Backend{ fake.backend(), fake.stepped() }) |b| {
+    for ([_]be.Backend{ fake.backend(), fake.stepped(), fake.batchedSteps() }) |b| {
         const w = vtable(H, b.vtable.*);
         const info = @typeInfo(be.Backend.VTable).@"struct";
         inline for (info.field_names, info.field_types) |name, T| {
             if (@typeInfo(T) == .optional) try std.testing.expectEqual(@field(b.vtable, name) != null, @field(w, name) != null);
-            if (T == bool) try std.testing.expectEqual(@field(b.vtable, name), @field(w, name));
+            if (T == bool or @typeInfo(T) == .int) try std.testing.expectEqual(@field(b.vtable, name), @field(w, name)); // e.g. many_rows
         }
     }
     try std.testing.expect(vtable(H, fake.stepped().vtable.*).prefill_step != null);

From 44b59e7f02d3f51bf84737144db27362f17ed975 Mon Sep 17 00:00:00 2001
From: BobClawblaw <BobClawblaw@users.noreply.github.com>
Date: Sat, 17 Oct 2026 16:22:20 +0000
Subject: [PATCH 3/4] glm cuda: a prompt pass's linears take glm_mm's one-slice
 blocks at any rows, so a prompt's sums are the same alone or batched with
 others (split-K under 128 rows summed in another order)

---
 zig/src/families/glm/cuda_b16.zig     | 2 +-
 zig/src/families/glm/cuda_forward.zig | 2 +-
 zig/src/families/glm/cuda_glue.zig    | 8 ++++++--
 zig/src/families/glm/cuda_mm.zig      | 6 +++---
 4 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/zig/src/families/glm/cuda_b16.zig b/zig/src/families/glm/cuda_b16.zig
index 0509b459d2..d6fa9bc199 100644
--- a/zig/src/families/glm/cuda_b16.zig
+++ b/zig/src/families/glm/cuda_b16.zig
@@ -56,7 +56,7 @@ pub fn partBytes(m: usize, n: usize, k: usize) usize {
 /// qmm.matmul for a B16 weight: out [m][n] bf16 (fp32 with `wide`) = x . w; 128+ rows sum in registers, else `part`.
 pub fn matmul(set: glue.Set, stream: cuda.Stream, x: u64, x_stride: usize, w: u64, out: u64, part: u64, m: usize, n: usize, k: usize, wide: bool) !void {
     if (k % bk != 0) return error.KNotMultipleOf64;
-    if (set.mm) |mk| return mm.matmul(mk, stream, x, x_stride, .{ .w = w, .n = n, .k = k }, out, part, m, wide); // tiled weights
+    if (set.mm) |mk| return mm.matmul(mk, stream, x, x_stride, .{ .w = w, .n = n, .k = k }, out, part, m, wide, set.prompt); // tiled weights
     const sk = splitK(n, k);
     var a: cuda.Args = .{};
     a.add(x);
diff --git a/zig/src/families/glm/cuda_forward.zig b/zig/src/families/glm/cuda_forward.zig
index f851254bf9..583b91428e 100644
--- a/zig/src/families/glm/cuda_forward.zig
+++ b/zig/src/families/glm/cuda_forward.zig
@@ -176,7 +176,7 @@ pub const Forward = struct {
     prof: ?*prof_mod.Prof = null, // TF_GLM_PROF=1: events between the sections
 
     pub fn set(f: *const Forward) glue.Set {
-        return .{ .glue = &f.m.glue, .mm = if (f.m.tiled) &f.m.mm else null };
+        return .{ .glue = &f.m.glue, .mm = if (f.m.tiled) &f.m.mm else null, .prompt = f.ranks[0].b.prefill };
     }
 
     /// reduce.sums' gather: every rank's partial of `rows` rows into every rank's gathered buffer, rank order.
diff --git a/zig/src/families/glm/cuda_glue.zig b/zig/src/families/glm/cuda_glue.zig
index 53414396d3..1c31d6935d 100644
--- a/zig/src/families/glm/cuda_glue.zig
+++ b/zig/src/families/glm/cuda_glue.zig
@@ -4,8 +4,12 @@ const std = @import("std");
 const cuda = @import("cuda");
 const mm_mod = @import("cuda_mm.zig");
 
-/// What the forward's kernels launch from: our glue's functions.
-pub const Set = struct { glue: *const Kernels, mm: ?*const mm_mod.Kernels = null }; // mm: bf16 linears tiled at load
+/// What the forward's kernels launch from: our glue's functions, glm_mm's (tiled weights), a prompt pass's flag.
+pub const Set = struct {
+    glue: *const Kernels,
+    mm: ?*const mm_mod.Kernels = null,
+    prompt: bool = false, // glm_mm's one-slice blocks at any rows: a prompt's sums alike alone or batched
+};
 
 /// Our glm_glue.cu's functions.
 pub const Kernels = struct {
diff --git a/zig/src/families/glm/cuda_mm.zig b/zig/src/families/glm/cuda_mm.zig
index 0abdfd7e39..1113ae7767 100644
--- a/zig/src/families/glm/cuda_mm.zig
+++ b/zig/src/families/glm/cuda_mm.zig
@@ -50,12 +50,12 @@ pub fn partBytes(m: usize, w: Tiled) usize {
     return if (sk > 1) sk * @min(m, 64) * w.n * 4 else 0;
 }
 
-/// x [m][k] (rows `ldx` apart) by `w` -> out [m][n], bf16 or fp32 (`wide`); `part`: partBytes(m, w).
-pub fn matmul(k: *const Kernels, s: cuda.Stream, x: u64, ldx: usize, w: Tiled, out: u64, part: u64, m: usize, wide: bool) !void {
+/// x [m][k] (`ldx` apart) by `w` -> out [m][n] (`wide`: fp32); `one`: prompt blocks at any rows, sums per row alike.
+pub fn matmul(k: *const Kernels, s: cuda.Stream, x: u64, ldx: usize, w: Tiled, out: u64, part: u64, m: usize, wide: bool, one: bool) !void {
     if (w.k % 64 != 0 or w.n % 8 != 0 or x % 16 != 0 or ldx % 8 != 0) return error.BadShape;
     var done: usize = 0;
     while (done < m) { // windows to 64 rows a launch with K slices; prompts in one launch of 128-row blocks, one slice
-        const big = m - done >= 128;
+        const big = one or m - done >= 128;
         const rows = if (big) m - done else @min(m - done, 64);
         const i: usize = if (big) 3 else if (rows <= 16) 0 else if (rows <= 32) 1 else 2;
         const bm = bucket(i);

From aaa0bef865b32ba4f63de9bf11f59ad908d69d3b Mon Sep 17 00:00:00 2001
From: BobClawblaw <BobClawblaw@users.noreply.github.com>
Date: Sun, 18 Oct 2026 08:41:09 +0000
Subject: [PATCH 4/4] glm cuda: prefill_many's passes keep separate hold rows
 (stream i of the batch at row i, carried on the follow link), so a later pass
 no longer overwrites an earlier pass's first-draft rows before their drafts;
 the pass split is a host-tested helper

---
 zig/src/families/glm/cuda_follow.zig  | 12 ++---
 zig/src/families/glm/cuda_lanes.zig   | 69 ++++++++++++++++++++-------
 zig/src/families/glm/cuda_prompts.zig | 17 +++----
 3 files changed, 67 insertions(+), 31 deletions(-)

diff --git a/zig/src/families/glm/cuda_follow.zig b/zig/src/families/glm/cuda_follow.zig
index 6c80df6651..8a97da1977 100644
--- a/zig/src/families/glm/cuda_follow.zig
+++ b/zig/src/families/glm/cuda_follow.zig
@@ -211,9 +211,9 @@ pub const Leader = struct {
         try x.e.verifyShared(parts);
     }
 
-    /// Whole prompts in one prompt pass on every rank (Engine.prefillMany): their slots and lengths, then the ids.
-    pub fn prefillMany(x: Leader, parts: []const engine.Prompt, firsts: []u32) !void {
-        if (!prompts.fits(parts, x.e.capacity)) return error.BadPrompts;
+    /// Whole prompts in one pass on every rank (Engine.prefillMany): slots, lengths, ids; hold0 (field a).
+    pub fn prefillMany(x: Leader, parts: []const engine.Prompt, firsts: []u32, hold0: usize) !void {
+        if (!prompts.fits(parts, hold0, x.e.capacity)) return error.BadPrompts;
         try x.flush();
         var buf: [2 * engine.max_together + engine.prompt_rows]u32 = undefined;
         var n: usize = 2 * parts.len;
@@ -223,8 +223,8 @@ pub const Leader = struct {
             @memcpy(buf[n..][0..p.ids.len], p.ids);
             n += p.ids.len;
         }
-        try x.send(.{ .kind = @backingInt(Kind.many), .depth = @intCast(parts.len) }, buf[0..n]);
-        try x.e.prefillMany(parts, firsts);
+        try x.send(.{ .kind = @backingInt(Kind.many), .depth = @intCast(parts.len), .a = hold0 }, buf[0..n]);
+        try x.e.prefillMany(parts, firsts, hold0);
     }
 
     pub fn keep(x: Leader, n: usize) !void {
@@ -303,7 +303,7 @@ pub fn follow(e: *Engine, link: *Link) !void {
                     at += n;
                 }
                 var firsts: [engine.max_together]u32 = undefined;
-                try e.prefillMany(parts[0..k], &firsts);
+                try e.prefillMany(parts[0..k], &firsts, @intCast(h.a));
             },
             .keep => try e.keep(@intCast(h.a)),
             .draft => {
diff --git a/zig/src/families/glm/cuda_lanes.zig b/zig/src/families/glm/cuda_lanes.zig
index 2486951ef9..403b49c0c4 100644
--- a/zig/src/families/glm/cuda_lanes.zig
+++ b/zig/src/families/glm/cuda_lanes.zig
@@ -140,28 +140,22 @@ pub const Lanes = struct {
         errdefer for (ss[0..opened]) |s| releaseFn(ptr, s);
         var parts: [engine.max_together]engine.Prompt = undefined;
         var firsts: [engine.max_together]u32 = undefined;
-        var lo: usize = 0;
-        var rows: usize = 0;
-        for (ss, 0..) |s, i| {
-            if (rows + s.prompt_len > engine.prompt_rows) { // this pass is full: run it, the next starts here
-                try self.many(ss[lo..i], parts[lo..i], firsts[lo..i]);
-                lo = i;
-                rows = 0;
+        var lens: [engine.max_together]usize = undefined;
+        for (ss, lens[0..ss.len]) |s, *n| n.* = s.prompt_len;
+        var cut: [engine.max_together][2]usize = undefined;
+        for (cut[0..passes(lens[0..ss.len], engine.prompt_rows, &cut)]) |p| {
+            for (ss[p[0]..p[1]], p[0]..) |s, i| {
+                if (try self.opening(s) != null) return error.BadPrompts; // no kept state (checked above)
+                opened += 1;
+                parts[i] = .{ .slot = self.lanes.getPtr(s).?.slot, .ids = s.prompt() };
             }
-            if (try self.opening(s) != null) return error.BadPrompts; // no kept state (checked above)
-            opened += 1;
-            parts[i] = .{ .slot = self.lanes.getPtr(s).?.slot, .ids = s.prompt() };
-            rows += s.prompt_len;
+            // stream i's hold row is i: every pass's rows stay until the core asks each stream's first drafts
+            try self.x.prefillMany(parts[p[0]..p[1]], firsts[p[0]..p[1]], p[0]);
+            for (ss[p[0]..p[1]], firsts[p[0]..p[1]]) |s, t| self.lanes.getPtr(s).?.first = self.take(t);
         }
-        try self.many(ss[lo..], parts[lo..ss.len], firsts[lo..ss.len]);
         return true;
     }
 
-    fn many(self: *Lanes, ss: []const *lanes.Stream, parts: []const engine.Prompt, firsts: []u32) !void {
-        try self.x.prefillMany(parts, firsts);
-        for (ss, firsts) |s, t| self.lanes.getPtr(s).?.first = self.take(t);
-    }
-
     /// One chunk of the stream's prompt pass (fillOne runs other rounds between); true once the first token is drawn.
     fn prefillStepFn(ptr: *anyopaque, s: *lanes.Stream) anyerror!bool {
         const self = of(ptr);
@@ -340,6 +334,47 @@ pub const Lanes = struct {
     }
 };
 
+/// prefill_many's passes over prompts of `lens` tokens, in order, at most `rows` tokens a pass: [lo, hi) of each.
+fn passes(lens: []const usize, rows: usize, out: *[engine.max_together][2]usize) usize {
+    var n: usize = 0;
+    var lo: usize = 0;
+    var used: usize = 0;
+    for (lens, 0..) |len, i| {
+        if (i > lo and used + len > rows) { // this pass is full: the next starts here
+            out[n] = .{ lo, i };
+            n += 1;
+            lo = i;
+            used = 0;
+        }
+        used += len;
+    }
+    if (lens.len > lo) {
+        out[n] = .{ lo, lens.len };
+        n += 1;
+    }
+    return n;
+}
+
+test "prefill_many's passes: in order, each within the rows, every prompt once, hold rows distinct across passes" {
+    var cut: [engine.max_together][2]usize = undefined;
+    const lens = [_]usize{ 1500, 800, 300, 2048, 10, 10 };
+    const n = passes(&lens, 2048, &cut);
+    try std.testing.expectEqualSlices([2]usize, &.{ .{ 0, 1 }, .{ 1, 3 }, .{ 3, 4 }, .{ 4, 6 } }, cut[0..n]);
+    var seen: [engine.max_together]bool = @splat(false);
+    for (cut[0..n]) |p| {
+        var used: usize = 0;
+        for (p[0]..p[1]) |i| {
+            used += lens[i];
+            try std.testing.expect(!seen[i]); // a stream's hold row is its index in the batch: no two share one
+            seen[i] = true;
+        }
+        try std.testing.expect(used <= 2048);
+    }
+    for (seen[0..lens.len]) |s| try std.testing.expect(s);
+    try std.testing.expectEqual(@as(usize, 1), passes(&[_]usize{ 28, 28, 28, 28 }, 2048, &cut)); // the ruler's case
+    try std.testing.expectEqual(@as(usize, 0), passes(&[_]usize{}, 2048, &cut));
+}
+
 comptime {
     std.debug.assert(max_rows <= fw.window_rows);
 }
diff --git a/zig/src/families/glm/cuda_prompts.zig b/zig/src/families/glm/cuda_prompts.zig
index 312752d9ca..a312acf33b 100644
--- a/zig/src/families/glm/cuda_prompts.zig
+++ b/zig/src/families/glm/cuda_prompts.zig
@@ -5,9 +5,9 @@ const fw = @import("cuda_forward.zig");
 const engine = @import("cuda_engine.zig");
 const Engine = engine.Engine;
 
-/// Whether `parts` go in one pass: 1 to max_together fresh prompts, prompt_rows in all, each under the capacity.
-pub fn fits(parts: []const engine.Prompt, capacity: usize) bool {
-    if (parts.len == 0 or parts.len > engine.max_together) return false;
+/// Whether `parts` go in one pass: fresh prompts, prompt_rows in all, under the capacity, hold rows from hold0 fit.
+pub fn fits(parts: []const engine.Prompt, hold0: usize, capacity: usize) bool {
+    if (parts.len == 0 or hold0 + parts.len > engine.max_together) return false;
     var rows: usize = 0;
     for (parts, 0..) |p, i| {
         if (p.ids.len == 0 or p.ids.len + fw.window_rows >= capacity) return false;
@@ -17,9 +17,9 @@ pub fn fits(parts: []const engine.Prompt, capacity: usize) bool {
     return rows <= fw.prefill_rows;
 }
 
-/// The prompts' pass on their slots (each begun, at position 0); `firsts[k]`: prompt k's first token.
-pub fn prefillMany(e: *Engine, parts: []const engine.Prompt, firsts: []u32) !void {
-    if (!fits(parts, e.capacity) or firsts.len < parts.len) return error.BadPrompts;
+/// The prompts' pass (each begun at 0): prompt k's first token in firsts[k], draft source in hold row hold0 + k.
+pub fn prefillMany(e: *Engine, parts: []const engine.Prompt, firsts: []u32, hold0: usize) !void {
+    if (!fits(parts, hold0, e.capacity) or firsts.len < parts.len) return error.BadPrompts;
     var segs: [engine.max_together]fw.Seg = undefined;
     var ids: [fw.prefill_rows]u32 = undefined;
     var rows: usize = 0;
@@ -39,8 +39,9 @@ pub fn prefillMany(e: *Engine, parts: []const engine.Prompt, firsts: []u32) !voi
         try e.bind(p.slot);
         try e.draw(e.bp.logits + k * e.share.vocab.len() * 2, 1, p.ids.len, firsts[k..][0..1]);
         if (!e.mtp) continue;
-        try e.copy(e.hold.ptr + k * D * 2, e.bp.fnormed + (g.row0 + g.rows - 1) * D * 2, D * 2);
-        e.cur.held_at = e.hold.ptr + k * D * 2;
+        const row = e.hold.ptr + (hold0 + k) * D * 2;
+        try e.copy(row, e.bp.fnormed + (g.row0 + g.rows - 1) * D * 2, D * 2);
+        e.cur.held_at = row;
     }
     f.pieces = null;
     for (parts, segs[0..parts.len]) |p, g| {