From 66eec7f549f5120583a26939aed34894db90f6a2 Mon Sep 17 00:00:00 2001 From: mad-lab-kbando <69054773+kmbandy@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:33:44 -0400 Subject: [PATCH 01/19] MAD-305 DSWS: Phase A substrate (GPU-proven) + Phase B conversion design/plan Snapshot of the DSWS v2 claim-based split-K substrate and the Phase B runtime role-conversion design, after the static substrate passed its supervised GPU gate (all 3 role mixes x 2 split-K tiers, oracle bit-exact, zero bricks, 2026-07-01). Included (all new, isolated to spike/dvgpr_occ/): - occ_kernel_dsws.s: v2 claim-based split-K kernel (Phase A, GPU-green) - dsws_ctrl_model.cpp + test: control law (watermark/epoch/gate/reserve) - build_dsws.sh: offline assemble + RGA gate driver - SPEC/PLAN_DSWS_SUBSTRATE_V2.md: Phase A design + plan - SPEC/PLAN_DSWS_PHASEB_CONVERSION.md: Phase B conversion design + plan (per-epoch snapshot quiesce + bail-time commit; approved 2026-07-01) - SPEC/PLAN_DSWS_CONTROLLER.md, MAD305_DSWS_MASTER.md, RESULT_DSWS.md: context Deliberately excluded: pre-existing working-tree changes in occ_kernel_coop.s, occ_dispatch.cpp, fp8_oracle.*, mt_pagedattn* (entangled with unrelated run_mbcoop/CoopResult work) and build artifacts (*.bin/*.o/build/). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0132aDSBLwusCJ4KzHQTnvdu --- .../spike/dvgpr_occ/MAD305_DSWS_MASTER.md | 172 +++++ .../spike/dvgpr_occ/PLAN_DSWS_CONTROLLER.md | 320 ++++++++ .../dvgpr_occ/PLAN_DSWS_PHASEB_CONVERSION.md | 400 ++++++++++ .../spike/dvgpr_occ/PLAN_DSWS_SUBSTRATE_V2.md | 338 ++++++++ .../spike/dvgpr_occ/RESULT_DSWS.md | 353 +++++++++ .../spike/dvgpr_occ/SPEC_DSWS_CONTROLLER.md | 217 ++++++ .../dvgpr_occ/SPEC_DSWS_PHASEB_CONVERSION.md | 222 ++++++ .../spike/dvgpr_occ/SPEC_DSWS_SUBSTRATE_V2.md | 222 ++++++ .../spike/dvgpr_occ/build_dsws.sh | 51 ++ .../spike/dvgpr_occ/dsws_ctrl_model.cpp | 54 ++ .../spike/dvgpr_occ/occ_kernel_dsws.s | 726 ++++++++++++++++++ .../spike/dvgpr_occ/test_dsws_ctrl_model.cpp | 51 ++ 12 files changed, 3126 insertions(+) create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/MAD305_DSWS_MASTER.md create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_CONTROLLER.md create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_PHASEB_CONVERSION.md create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_SUBSTRATE_V2.md create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/RESULT_DSWS.md create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_CONTROLLER.md create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_PHASEB_CONVERSION.md create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_SUBSTRATE_V2.md create mode 100755 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_dsws.sh create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/dsws_ctrl_model.cpp create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/test_dsws_ctrl_model.cpp diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/MAD305_DSWS_MASTER.md b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/MAD305_DSWS_MASTER.md new file mode 100644 index 000000000000..ad2e1cf18c38 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/MAD305_DSWS_MASTER.md @@ -0,0 +1,172 @@ +# MAD-305 / DSWS — Master Pickup Doc (GPU kernel campaign) + +**Single source of truth for the dynamic-VGPR fp8 GEMM work.** Consolidated +2026-06-26 before a context compact. Read this first to resume cold. Detailed +history lives in the sibling docs + KG notes pointed to at the bottom. + +--- + +## 1. North star + the moat + +- **Goal:** 250–300 TF fp8 GEMM on **gfx1201** (AMD R9700 / Navi48, RDNA4, wave32). +- **The moat — dynamic VGPR (`s_alloc_vgpr`)** is armable on a compute queue **only + via raw PM4** writing **COMPUTE_PGM_RSRC2 bit 6** on a KFD queue (libhsakmt, + bypassing ROCr). HIP/the toolchain cannot emit it for compute. This is why the + whole harness dispatches via raw PM4 (`occ_dispatch.cpp` + `dvgpr_pm4/`), not HIP. + Confirmed: bit clear → reads 0; bit 6 set → `DYN_VGPR_EN` reads back. (MAD-304.) + +## 2. Current state (the numbers) + +- **fp8 WMMA ceiling = 307 TF** (15.9 WMMA/cyc); **272 TF** feed-free. One WMMA = + 16×16×16 = 8192 FLOP. R9700 = 64 CU × 2 SIMD32 = **128 SIMDs**. +- **Best real kernel ≈ 165.7 TF** (8×2 tile, KWINBPF double-buffer + s_setprio + + wide-A feed) on **square** shapes = **~52% of ceiling**, **8.34 WMMA/cyc**. + HIP 4×4-dbuf reference = 161 TF / 52.5% (VGPR 183/256, 16 frags = 128 acc VGPR, + occ 5 blocks = 20 waves). +- **On real ml8 dims = ~113 TF** (latest tile tuning, 3 verified fixes; commit + `99f063e9c`). ml8 shapes: **`down`** (ffn_down) M=2048 K=9216 N=2560; + **`down_pf`** (prefill) M=512. *Always bench on these, not throwaway squares.* + +## 3. THE WALL — it's the VALU issue port, NOT occupancy + +- Measured **~31 non-WMMA instructions issued per 32 WMMAs** (square shapes) → + caps throughput at **8.34 / 15.9 WMMA/cyc = 52%** of 307 TF. The issue port, + not memory and not occupancy, is the binding constraint. +- **Occupancy is FLAT**: minWaves 4→8 barely moves TF (~160 TF either way). So + raising occupancy is NOT the lever. +- **The lever = cut non-WMMA issues per WMMA** — via (a) more fragment reuse, or + (b) wave specialization so the math wave issues ~only WMMAs while other waves do + the address/feed VALU. This is what DSWS targets. +- Caveat: the 165 TF winner + the 31:32 ratio are **square** shapes. ml8 shapes + (tall-skinny `down`) differ — re-measure the issue mix ON ml8 with `--att`. + +## 4. SOLVED this week (both real, keep) + +### POOLTERM — pool≥2 teardown brick (FIXED, confirmed on silicon 2026-06-26) +- **Root cause:** the compute terminal in `occ_kernel_coop.s` was a **POOL=1-only + diagnostic stub** — compute counted tiles it processed (s57) and exited at + s57==TOTAL. Valid only when ONE WG owns all tiles. At pool≥2 the WGs SPLIT tiles + via the shared global atomic claim, so each compute gets ~1s) persistent dispatches starve + the compositor's gfx ring → brick. Fix: bound each dispatch to `ML8_COOP_CHUNK` + output tiles (claim `occW[5]=base`, `userdata[11]/s11=hi` terminal), drain on the + EOP fence, YIELD between chunks, `ML8_COOP_CHUNK_MAXS` guard. Sub-second bounded + dispatches run the real ml8 GEMM **imperceptibly** on the display GPU. + +## 5. DSWS — the design (the actual north-star architecture) + +**"Dynamic-Split Wave-Specialization" / adaptive wave-role economy.** ONE kernel +that re-balances its mix of **{fat compute waves (large VGPR, hold fp32 WMMA +accumulators) / lean A-feed waves / lean B-feed waves}** to the bottleneck **at +runtime**, using `s_alloc_vgpr` to make the VGPR budget **fungible across roles +asymmetrically** (shrink one fat compute ~112–256 VGPR → fund several lean feeders, +or vice-versa). Bottleneck sensed by the prod/cons ring counters already in +`occ_kernel_coop.s`. + +- **Split-K is the headroom-creator, NOT an occupancy play.** Full-K GEMM holds the + fp32 accumulators (~64 of 112 VGPR) at peak for the ENTIRE K-loop (~95% duty — + square wave, not trapezoid), so staggering buys nothing. Split-K creates brief + accumulator-peak windows = the room to reallocate VGPR between roles. +- **Why this beats static:** the issue-port wall (§3) is broken by letting the math + wave issue ~only WMMAs while feed waves absorb the address/feed VALU — and tuning + that ratio per-shape (and within a kernel) to whatever the runtime says is short. + +## 6. Prior art + the research — VERDICT (Murmur prior-art-scan, landed 2026-06-27) + +**Workflow `5ec8a958` (42 scouts on 6900xt + Qwen3.6-27B captain sift). Verdict: +the RUNTIME-ADAPTIVE piece is NOVEL; the lean/fat mechanism is established (borrow +it).** Ranked leads: + +1. **Runtime-adaptive producer:consumer wave-role rebalancing with dynamic per-wave + register reallocation is NOT in any known GPU GEMM/compute kernel** (HIGH conf — + unanimous negative across ~40 scouts: arXiv 2023–2026, AMD GPUOpen, NVIDIA + CUTLASS/CuTe, Triton, LLVM/ACO, forums). → DSWS's core (in-kernel ring-counter + sensing + runtime `s_alloc_vgpr` rebalancing) is **architecturally novel**. +2. **CUTLASS warp-spec = STATIC `setmaxnreg` / `warpgroup_reg_alloc` at launch** — + producer(DMA/TMA):consumer(MMA) split + register budgets fixed for the whole + kernel. The direct baseline to differentiate from; register fungibility is a + compile/launch-time optimization, not a runtime control loop. +3. **AMD `s_alloc_vgpr` (RDNA3.5/4) = static per-wave; no in-kernel sensing / runtime + shifting.** The primitive we repurpose; the ISA lacks native runtime-rebalancing + semantics → DSWS supplies its own coordination layer (= the contribution). +4. **Adjacent dynamic work (Stream-K, persistent kernels, WaveTune) balances at the + CTA/workgroup level, NOT intra-CTA wave roles or mid-flight register realloc** + (HIGH conf). Rules out the nearby paradigm a reviewer might conflate us with. + +**GAPS / must verify against primary sources before we stake the claim:** +- Exact CUTLASS `setmaxnreg` register ranges + granularity (the "24..256 step 8" + numbers) and documented HW restrictions — NOT extracted by any scout. +- Precise producer:consumer warp ratios in CUTLASS / CK-rocWMMA / Triton persistent. +- AMD `s_alloc_vgpr` HW rules re: barriers, occupancy, wave-launch sync in COMPUTE + shaders (vs the RT path) — needs ISA-level confirmation. (Ties to our own + barrier-vs-dyn-VGPR deadlock history — KG `8a9ce97f`/`17f209af`.) +- Novelty is partly absence-of-evidence; one citation (`WaveTune arXiv:2604.10187`) + is suspect (possible 8B-scout hallucination) — do not cite unverified. + +Full LEADS doc: handoff `141924b9` result in quantdb on mad-lab-2026. Captain banked +to claude__main KG. (This run also surfaced + fixed the engine bug that had been +truncating the captain's scout bundle — KG `22063f79`.) + +## 7. Key files + bins (all on mad-lab-main, committed to fork master) + +- `occ_kernel_coop.s` — cooperative kernel; **POOLTERM** fix; the prod/cons ring + counters DSWS will sense; SAFEPROBE bounds guard. +- `occ_kernel_wavespec.s` — static loader/compute role-split prototype (WS path). +- `occ_kernel_mbgemm.s`, `occ_kernel_wggemm2.s` — the 8×2 / KWINBPF winner lineage. +- `occ_dispatch.cpp` — raw-PM4 KFD harness; compositor-safe chunking; oracle gate. +- `build_coop.sh` / `build.sh` — defsym build matrices (FM,FN,P,RINGD,BATCH,DYNVGPR). +- `../dvgpr_pm4/` — PM4 packet defs + RSRC2 bit6 arming. +- **Detailed specs:** `SPEC_WAVESPEC.md` (DSWS v1+v2, wall numbers, prior art), + `MORNING_PICKUP.md` (day arc), `RESULTS_ml8_dynvgpr_gauntlet.md`, + `MAD305_LEVER_CATALOG.md`, `L4_LEAN_DESIGN.md`, the `RELATED_WORK_SWEEP` + + `NVIDIA_KERNEL_IDEAS_FOR_RDNA4` research docs. + +## 8. STANDING SAFETY (hard rules — do not violate) + +- **A GPU brick is a BUG, never an "accepted tax."** Recoverable-via-MODE1/reboot + ≠ acceptable. Root-cause and fix; don't route around. +- **THE USER decides EVERY GPU dispatch** — each individual dispatch, not one + blanket "go" for a session. A hang/timeout on an unproven kernel = full STOP + + report, NOT auto-fire the next variant. Do max work OFFLINE first (disasm, RGA, + static analysis); batch a diagnostic into ONE prepared dispatch. +- Only **sub-second bounded** dispatches are safe on the display GPU (R9700 drives + the monitors). Freeze dyn dispatch on any build known to leave the queue non-idle. +- **NEVER pass `--gl2c`** (MES-crash landmine). Keep SAFEPROBE clamp + bounds gate + + padding ON. Every run streams to disk (`ML8_COOP_STREAM=1`). +- Do NOT move displays to the eGPU 6900XT (can't init pre-login; also NOT the + single-GPU target user — the kernel must coexist with the compositor). + +## 9. NEXT STEPS (resume order for step 4) + +1. **Incorporate the Murmur prior-art leads** (research output) into the DSWS + novelty + setmaxnreg mechanics-to-copy. +2. **R0 occupancy attribution** — step pool up ONE at a time (gated, abort on any + non-clean exit); RGA showed compute peak-live ~81 vs HW-allocated 120 → trim NFV + toward ~96 = free occupancy. +3. **ML8_P=2** — one feed + TWO compute waves in one WG = the real reuse/throughput + lever (reuse 2.0/2.4/2.67 for P=2/3/4). P>1 needs its own terminal review. +4. **DSWS build** — brainstorm → spec → plan → TDD, on the POOLTERM substrate, + measured on ml8 `down`/`down_pf` with `--att` for the issue mix. + +## 10. Pointers + +- KG: `dac0bb8c` (POOLTERM root cause), `0a2cea44` (POOLTERM silicon confirm), + `63583120` (DSWS v2), `21827908` (compositor chunking), `34f29c00` (full-day + session summary), `58a41155` (GPU-dispatch discipline), `1630687a` (earlier + brick = OOB shader access root cause + the mandatory bounds-guard lesson). +- Jira epic: MAD-305 (under the 250-300 TF north star). MAD-304 = the PM4 dyn-VGPR + arming. MAD-300 = the WMMA ceiling + rocWMMA baseline. +- Branch/state: fork **master** (pushed `9ff961564`), all dvgpr work committed + (`a514395ec` snapshot + the upstream merge). diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_CONTROLLER.md b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_CONTROLLER.md new file mode 100644 index 000000000000..3a5fcba40011 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_CONTROLLER.md @@ -0,0 +1,320 @@ +# DSWS Adaptive Wave-Role Controller — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> +> **Domain note:** this is hand-written gfx1201 (RDNA4 wave32) assembly + a raw-PM4 KFD harness + a CPU fp8 oracle — NOT a pytest codebase. "Tests" are: the CPU fp8 e4m3 **oracle gate** (bit/tolerance match of stored C), the offline **RGA gate** (0 spills / 0 scratch + livereg peak), and **supervised GPU** oracle/perf runs. Assembly tasks are **gate-defined**: the step states the precise structural change and the exact gate command + expected output; the gate is the test. Host/logic tasks (oracle harness, CAS single-winner model, LDS sizing, watermark logic) get real code + commands. + +**Goal:** A single fp8 GEMM kernel on gfx1201 that senses its in-kernel bottleneck (ring pressure) and rebalances its mix of {compute / A-feed / B-feed} waves at runtime via `s_alloc_vgpr`, beating the 165 TF static winner on ml8 shapes while staying oracle-correct. + +**Architecture:** Build on `occ_kernel_coop.s` (dyn-VGPR + split-K + POOLTERM, raw-PM4 dispatch). Add a 3rd wave role (A-feed) to make a static 3-role substrate, then layer a lock-free, barrier-free controller: per-WG LDS role-count slots + ring-occupancy sensing + watermark hysteresis + an epoch-gated single-winner CAS ticket that serializes conversions (≤1 per direction per `2^EPOCH_SHIFT` segments), with sum-envelope reservation enforced at grow-time. + +**Tech Stack:** gfx1201 hand-asm (LLVM `clang` assembler), `occ_dispatch.cpp` (libhsakmt KFD PM4 harness), `fp8_oracle.cpp` CPU reference, `rga_check.sh` (Radeon GPU Analyzer), `rocprofv3 --att/--pmc`. Build via `build.sh` / `build_coop.sh` defsym matrices. + +## Global Constraints + +- **Target:** gfx1201 (AMD R9700 / Navi48, RDNA4, **wave32**). One WMMA = 16×16×16 = 8192 FLOP. fp8 e4m3 inputs, fp32 accumulate. fp8 WMMA ceiling = 307 TF. +- **Dispatch:** raw PM4 on a KFD compute queue only (dyn-VGPR armed via `COMPUTE_PGM_RSRC2` bit 6). Never HIP for the kernel. **Never pass `--gl2c`** (MES-crash landmine). SAFEPROBE clamp + bounds gate + padding stay ON. Every run streams to disk (`ML8_COOP_STREAM=1`). +- **GPU runs are SUPERVISED.** R9700 drives the displays — a hang resets monitors. THE USER greenlights EACH dispatch individually. A hang/timeout = full STOP + report, never auto-fire the next variant. Only sub-second bounded (compositor-safe chunked) dispatches. Do max work OFFLINE first (disasm, RGA, CPU oracle). +- **A GPU brick is a BUG, never a tax.** Freeze dyn dispatch on any build known to leave the queue non-idle. +- **Commits:** per the user's standing rule, **commit only when the user asks.** The `git` steps below are checkpoints — batch them and run when greenlit. Co-author/session trailers per repo convention. +- **Correctness invariant:** DSWS is a pure *performance* transform — the adaptive mix MUST NOT change the math. Every gate is "stored C matches the fp8 oracle for any (shape, mix, conversion schedule)." +- **ml8 bench shapes (always bench these, not throwaway squares):** `down` = M=2048 K=9216 N=2560; `down_pf` = M=512 K=9216 N=2560. +- **Baselines to beat:** static 3-role baseline (Phase 1) and the 165.7 TF 8×2 static winner. + +## File Structure + +- `occ_kernel_coop.s` — **modify.** The substrate. Add A-feed role + A-ring LDS; add controller state (role slots, gates, reservation, epoch counter); add sensing + conversion at boundaries; make the POOLTERM terminal role-agnostic. All new behavior behind defsyms so `DSWS=0` stays byte-identical to today's proven `coop` d0. +- `occ_dispatch.cpp` — **modify.** New `--dsws` mode: launch N waves, arm dyn-VGPR, program the launch mix, the per-chunk hang-abort, oracle gate, role-count/conversion snapshot decode. +- `fp8_oracle.cpp` / `test_fp8_oracle.cpp` — **reuse**, extend harness only (role-agnostic math). +- `dsws_ctrl_model.cpp` / `test_dsws_ctrl_model.cpp` — **create.** A pure-CPU model of the control-law logic (epoch/gate single-winner, watermark band decision, reservation envelope) so the *logic* is unit-tested offline before it goes into asm. +- `build_dsws.sh` — **create.** Defsym build matrix (NCOMP/NAFEED/NBFEED, RINGD, LOW/HIGH, EPOCH_SHIFT, DSWS, STORE) + oracle + RGA gates. +- `rga_check.sh` — **reuse** (re-point KSRC). +- `RESULT_DSWS.md` — **create** (Phase 4 outcome). + +--- + +## Phase 1 — Static 3-role substrate (Gate 1) + +### Task 1.1: Oracle harness accepts a 3-role fixed-mix config + +**Files:** +- Modify: `occ_dispatch.cpp` (the `--dsws` arg parse + config struct) +- Reuse: `fp8_oracle.cpp` (math unchanged — roles don't change C = A·B) + +**Interfaces:** +- Produces: a `DswsCfg{ uint32_t nComp, nAfeed, nBfeed, ringd, low, high, epochShift; bool dyn; }` parsed from env (`DSWS_NCOMP`, `DSWS_NAFEED`, `DSWS_NBFEED`, `DSWS_RINGD`, `DSWS_LOW`, `DSWS_HIGH`, `DSWS_EPOCHSHIFT`, `DSWS_DYN`) with `N = nComp+nAfeed+nBfeed`. + +- [ ] **Step 1: Add the config struct + env parse + a validation refuse-path** (mirror the existing WAVESPEC handler's bin-presence guard at `occ_dispatch.cpp` ~2811). Validate `nComp≥1, nAfeed≥1, nBfeed≥1` and that the requested `_gd` bin for `(N, ringd, dyn)` exists; if not, print refusal + `rc=4`, do NOT dispatch. +- [ ] **Step 2: Gate (host compile + refusal smoke, no GPU)** + +Run: `cd spike/dvgpr_occ && clang++ -std=c++17 -O2 -Wall -Wno-unused -I "$PM4/vendor/compat" -I "$PM4/vendor" -I "$PM4" -I /opt/rocm/include occ_dispatch.cpp fp8_oracle.cpp "$PM4/vendor/PM4Packet.cpp" "$PM4/vendor/BasePacket.cpp" /opt/rocm/lib/libhsakmt.a -ldrm_amdgpu -ldrm -lnuma -lpthread -ldl -lrt -o occ_dispatch` (PM4=../dvgpr_pm4) +Then: `DSWS_NCOMP=0 ./occ_dispatch --dsws` → Expected: prints refusal (`nComp≥1`), `rc=4`, no dispatch. +- [ ] **Step 3: Commit (when greenlit):** `git add occ_dispatch.cpp; git commit -m "dsws: 3-role fixed-mix config + validation refuse-path"` + +### Task 1.2: A-feed role + A-ring in the kernel (static mix) + +**Files:** +- Modify: `occ_kernel_coop.s` (LDS layout: add `ARING_OFF`, `PROD_A_OFF`, `CONS_A_OFF`; add the A-feed wave loop ported from `occ_kernel_wggemm2.s` A-LDS-share; switch compute's A source from direct-load to A-ring `ds_load`) + +**Interfaces:** +- Produces: defsyms `NCOMP`, `NAFEED`, `NBFEED` (static role counts); `RINGD_A` (A-ring depth, default = `RINGD`); `DSWS` (0 = byte-identical to current coop d0, 1 = 3-role). LDS symbols `ARING_OFF`, `PROD_A_OFF`, `CONS_A_OFF`, `min_cons_a` macro (mirror of B-side `PROD_OFF`/`CONS_OFF`/`min_cons`). + +- [ ] **Step 1: Add the A-ring LDS layout + the A-feed wave loop** behind `.if DSWS`. Port the proven cooperative A-fill/share addressing from `occ_kernel_wggemm2.s` (the KWIN A-LDS pattern); A-feed waves `global_load` A → `ds_store` into the A-ring → bump `prod_a`. Compute waves `ds_load` A from the A-ring (gated on `prod_a`) instead of `global_load`-direct. Keep `DSWS=0` path = the existing direct-A code, byte-identical. +- [ ] **Step 2: RGA gate (offline, no GPU)** + +Run: `KSRC=occ_kernel_coop.s DEFS="-defsym DSWS=1 -defsym NCOMP=4 -defsym NAFEED=2 -defsym NBFEED=2 -defsym RINGD=2" ./rga_check.sh` +Expected: assembles; **0 VGPR spills / 0 SGPR spills / 0 scratch**; prints livereg peak per role (record `V_fat` = compute peak-live, `V_lean` = feed peak-live — needed in Phase 3). +- [ ] **Step 3: Byte-identity gate for DSWS=0** + +Run: build the `DSWS=0` d0 bin and `cmp` against the committed `.clean_bins` coop d0. +Expected: **byte-identical** (the 1716B reference) — proves the A-feed addition is fully gated and the proven substrate is untouched. +- [ ] **Step 4: Commit (when greenlit):** `git add occ_kernel_coop.s; git commit -m "dsws: static A-feed role + A-ring LDS (DSWS=0 byte-identical)"` + +### Task 1.3: `--dsws` static dispatch + oracle gate wiring + +**Files:** +- Modify: `occ_dispatch.cpp` (the `--dsws` dispatch: WG = `N*32` threads, program role counts into `userdata`, load the `_gd` bin, STORE=1 oracle path) +- Create: `build_dsws.sh` (defsym matrix + oracle + RGA) + +**Interfaces:** +- Consumes: `DswsCfg` (1.1), kernel defsyms (1.2). +- Produces: `build_dsws.sh` emitting `occ_dsws_cab_r[_dyn][_st].bin`; dispatch grid = `pool * N * 32`. + +- [ ] **Step 1: Wire the `--dsws` dispatch** (clone the WAVESPEC/coop dispatch path): set `COMPUTE_START_X` dims for `N*32` threads/WG, write `NCOMP/NAFEED/NBFEED` into `userdata` slots the kernel reads at init, load the matching `_gd` bin, run STORE=1 then bit-check vs `fp8_oracle`. +- [ ] **Step 2: `build_dsws.sh` static matrix** — cells `{4c2a2b, 6c1a1b, 2c3a3b}` × `RINGD={2}` × `{st(STORE=1), perf(STORE=0)}`, each oracle-gated and RGA-gated offline. +- [ ] **Step 3: Gate (offline build + RGA, no GPU)** + +Run: `./build_dsws.sh static` +Expected: all cells assemble, RGA 0-spill, harness links. No GPU yet. +- [ ] **Step 4: Commit (when greenlit):** `git add occ_dispatch.cpp build_dsws.sh; git commit -m "dsws: static 3-role dispatch + build matrix + oracle wiring"` + +### Task 1.4 [SUPERVISED GPU]: Gate 1 — static 3-role oracle-green + +**Files:** none (run only). + +- [ ] **Step 1: Prepare ONE gated oracle dispatch.** Config `4c2a2b r2 STORE=1`, compositor-safe chunked, `ML8_COOP_STREAM=1`. Present the exact command to the user; **STOP for go.** +- [ ] **Step 2: On greenlight, run Gate 1** across `{down, down_pf, square}` × `{4c2a2b, 6c1a1b, 2c3a3b}`, one dispatch at a time. + +Run (example): `ML8_COOP_STREAM=1 DSWS_NCOMP=4 DSWS_NAFEED=2 DSWS_NBFEED=2 DSWS_RINGD=2 WG_M=2048 WG_K=9216 WG_N=2560 timeout 30 ./occ_dispatch --dsws` +Expected: `oracle CLEAN ok= bad=0`, fence FIRED, clean teardown, no dmesg faults, **user-confirmed no brick**. +- [ ] **Step 3:** If any cell bricks/hangs → STOP, capture last stream snapshot, report. Do not proceed to Phase 2 until all Gate-1 cells are clean. +- [ ] **Step 4: Commit the green substrate (when greenlit)** + note RGA `V_fat`/`V_lean`/`BUDGET` numbers in `RESULT_DSWS.md` (created here, appended through Phase 4). + +--- + +## Phase 2 — Sensing + role slots + reservation (no conversions yet) + +### Task 2.1: Controller LDS state + sizing assertion + +**Files:** +- Modify: `occ_kernel_coop.s` (LDS layout additions behind `.if DSWS`) + +**Interfaces:** +- Produces LDS symbols: `NCOMP_SLOT`, `NAFEED_SLOT`, `NBFEED_SLOT` (u32 each), `GATE_OFF` (4×u32, one per direction), `VRESV_OFF` (u32 `vgpr_reserved`), `SEGCNT_OFF` (u32 per-WG segments_processed). `LDS_TOTAL_DSWS` recomputed. + +- [ ] **Step 1: Add the state symbols + recompute `LDS_TOTAL`.** Add a build-time `.if (LDS_TOTAL_DSWS > 65536) .error` assertion (gfx1201 = 64 KB LDS/WG). +- [ ] **Step 2: Gate (RGA + LDS-size assert, offline)** + +Run: `KSRC=occ_kernel_coop.s DEFS="-defsym DSWS=1 -defsym NCOMP=4 -defsym NAFEED=2 -defsym NBFEED=2 -defsym RINGD=2" ./rga_check.sh` +Expected: assembles (LDS assertion passes for the swept tiles), 0-spill. If a large tile trips the 64 KB assert → record the max RINGD/tile that fits (feeds the "LDS sizing" open detail). +- [ ] **Step 3: Commit (when greenlit):** `git add occ_kernel_coop.s; git commit -m "dsws: controller LDS state (role slots, gates, reservation, segcnt) + 64KB assert"` + +### Task 2.2: CPU model of the control-law logic (TDD — this is genuinely unit-testable) + +**Files:** +- Create: `dsws_ctrl_model.cpp`, `test_dsws_ctrl_model.cpp` + +**Interfaces:** +- Produces (pure functions, mirrored 1:1 by the asm in Phase 3): + - `int watermark_decision(uint32_t occ, uint32_t low, uint32_t high)` → `-1` (over-served, occ>high), `+1` (starved, occ> shift`. + - `bool gate_try_win(std::atomic& gate, uint32_t E)` → CAS single-winner (read g; if g≥E false; else CAS(g→E)). + - `bool reserve_grow(std::atomic& resv, uint32_t delta, uint32_t budget)` → atomic_add; if over budget, atomic_sub back + false; else true. + +- [ ] **Step 1: Write failing tests** + +```cpp +// test_dsws_ctrl_model.cpp +#include "dsws_ctrl_model.cpp" +#include #include #include #include #include +int main(){ + // watermark bands + assert(watermark_decision(0,2,6)==+1); // empty -> starved + assert(watermark_decision(7,2,6)==-1); // full -> over-served + assert(watermark_decision(4,2,6)==0); // dead-zone + // epoch + assert(epoch_of(0,3)==0 && epoch_of(8,3)==1 && epoch_of(15,3)==1 && epoch_of(16,3)==2); + // gate: exactly ONE winner per epoch among many racers + for(uint32_t E=1;E<50;++E){ std::atomic g{E-1}; std::atomic wins{0}; + std::vector ts; for(int i=0;i<64;++i) ts.emplace_back([&]{ if(gate_try_win(g,E)) wins++; }); + for(auto&t:ts)t.join(); assert(wins.load()==1); } + // reservation: concurrent grows never exceed budget + { std::atomic r{0}; std::atomic ok{0}; + std::vector ts; for(int i=0;i<10;++i) ts.emplace_back([&]{ if(reserve_grow(r,30,100)) ok++; }); + for(auto&t:ts)t.join(); assert(r.load()<=100 && ok.load()==3); } // 3*30=90<=100, 4th would be 120>100 + printf("dsws_ctrl_model: ALL PASS\n"); return 0; } +``` +- [ ] **Step 2: Run to verify it fails** + +Run: `clang++ -std=c++17 -O2 -pthread test_dsws_ctrl_model.cpp -o t_ctrl && ./t_ctrl` +Expected: FAIL to compile (`dsws_ctrl_model.cpp` not written). +- [ ] **Step 3: Write `dsws_ctrl_model.cpp`** — the four pure functions exactly as specced in Interfaces. +- [ ] **Step 4: Run to verify pass** + +Run: `clang++ -std=c++17 -O2 -pthread test_dsws_ctrl_model.cpp -o t_ctrl && ./t_ctrl` +Expected: `dsws_ctrl_model: ALL PASS`. (This locks the control *logic* before it's transcribed to asm — the asm must match these semantics.) +- [ ] **Step 5: Commit (when greenlit):** `git add dsws_ctrl_model.cpp test_dsws_ctrl_model.cpp; git commit -m "dsws: CPU model + tests for watermark/epoch/gate-CAS/reservation"` + +### Task 2.3: Sensing reads + state init in the kernel (read-only; mix stays static) + +**Files:** +- Modify: `occ_kernel_coop.s` (leader zeroes/initializes role slots to the launch mix, `vgpr_reserved` to the launch footprint, gates to 0, segcnt to 0; add `occ_a`/`occ_b` reads + `watermark_decision`-equivalent at the boundaries — but **do not act** on them yet; stream `occ_a/occ_b` + role slots into the snapshot) +- Modify: `occ_dispatch.cpp` (decode the new snapshot fields) + +**Interfaces:** +- Consumes: LDS state (2.1), watermark semantics (2.2). + +- [ ] **Step 1: Add leader init** (extend the existing barrier-free INITFLAG publish): zero gates/segcnt, set role slots to `NCOMP/NAFEED/NBFEED`, set `vgpr_reserved = NCOMP*V_fat + (NAFEED+NBFEED)*V_lean`. +- [ ] **Step 2: Add read-only sensing** at the compute segment boundary and the feed inter-frag check: compute `occ_a`/`occ_b`, evaluate the band (no action), and the per-WG `segcnt` bump on segment completion. Stream `occ_a, occ_b, nComp, nAfeed, nBfeed` in the occ snapshot. +- [ ] **Step 3: Gate (RGA offline + oracle unchanged)** + +Run: `./build_dsws.sh static && KSRC=... ./rga_check.sh` +Expected: assembles, 0-spill; oracle STILL green (no behavior change — sensing is read-only). `DSWS=0` still byte-identical. +- [ ] **Step 4: Commit (when greenlit):** `git add occ_kernel_coop.s occ_dispatch.cpp; git commit -m "dsws: state init + read-only ring sensing + snapshot instrumentation"` + +### Task 2.4 [SUPERVISED GPU]: Sensors-report-sane gate + +**Files:** none (run only). + +- [ ] **Step 1: Prepare ONE gated dispatch** (`4c2a2b r2 STORE=1`, streamed). **STOP for go.** +- [ ] **Step 2: On greenlight, run** and inspect the streamed snapshots. + +Expected: oracle CLEAN; `occ_b` (and `occ_a`) visibly oscillate in `[0, RINGD]` as the rings fill/drain; role slots remain the static launch mix; no brick. This confirms the sensors read real pressure before we let them actuate anything. +- [ ] **Step 3:** Record a representative occ trace in `RESULT_DSWS.md`. If sensors look wrong (pinned at 0 or RINGD always) → STOP, the sensing addressing is off; fix offline before Phase 3. + +--- + +## Phase 3 — Conversion + epoch/ticket + role-agnostic terminal (Gate 2) + +### Task 3.1: Epoch + gate-CAS in asm (transcribe the verified model) + +**Files:** +- Modify: `occ_kernel_coop.s` (`epoch = segcnt >> EPOCH_SHIFT`; `gate_try_win` via `ds_cmpst_b32` (LDS compare-swap) on `GATE_OFF + dir*4`) + +**Interfaces:** +- Consumes: `dsws_ctrl_model.cpp` semantics (2.2) — the asm MUST match `gate_try_win`/`epoch_of` exactly. `EPOCH_SHIFT` defsym. + +- [ ] **Step 1: Implement `epoch_of` + `gate_try_win`** as an asm macro `try_gate dir` returning SCC/flag. Use `ds_cmpst_b32` for the LDS CAS (read g; branch if g≥E; else CAS(g,E); success iff returned-old==g). +- [ ] **Step 2: Gate (RGA offline)** — assembles, 0-spill. **Still no conversion wired** (macro defined + a unit call site that logs "would-win" into a snapshot slot, no actuation). +- [ ] **Step 3 [SUPERVISED GPU, optional micro-check]:** if cheap, one streamed run confirming the "would-win" counter increments ≤1 per epoch per direction. Else defer to 3.5. +- [ ] **Step 4: Commit (when greenlit):** `git add occ_kernel_coop.s; git commit -m "dsws: epoch + lock-free gate-CAS ticket (ds_cmpst), no actuation yet"` + +### Task 3.2: compute→feed conversion (shrink path) + +**Files:** +- Modify: `occ_kernel_coop.s` (at the compute segment boundary: if `occ_X1` → CAS-dec `nComp`, inc `nFeedX`, `atomic_sub vgpr_reserved`, `s_alloc_vgpr LEANREG`, jump into the feed loop) + +**Interfaces:** +- Consumes: gate macro (3.1), role slots + reservation (2.1), `V_fat`/`V_lean` (1.2 RGA). + +- [ ] **Step 1: Implement the shrink-conversion** at the segment boundary, floor-guarded (`nComp>1` via `ds_cmpst`/atomic dec-if-greater), epoch-gated. Shrink always succeeds. +- [ ] **Step 2: Gate (offline RGA + build)** — assembles, 0-spill. +- [ ] **Step 3: Oracle gate (offline-buildable, GPU-run deferred to 3.5)** — build STORE=1 bin. +- [ ] **Step 4: Commit (when greenlit):** `git add occ_kernel_coop.s; git commit -m "dsws: compute->feed conversion (shrink, floor+epoch gated)"` + +### Task 3.3: feed→compute conversion (grow path + envelope abort) + +**Files:** +- Modify: `occ_kernel_coop.s` (at the feed inter-frag check: if `occ_X>HIGH` & `try_gate(feedX→compute)` & `nFeedX>1` → `reserve_grow`; on success: CAS-dec `nFeedX`, inc `nComp`, `s_alloc_vgpr NFV`, enter compute loop; on reserve fail: abort, leave slots/reservation untouched, stay feed) + +**Interfaces:** +- Consumes: `reserve_grow` semantics (2.2), gate (3.1), `BUDGET` (per-SIMD VGPR budget, from RGA/ISA in 1.4). + +- [ ] **Step 1: Implement the grow-conversion** with the reservation-then-validate-then-maybe-undo envelope (transcribe `reserve_grow`). Order: reserve first (atomic_add), validate `≤BUDGET`, then commit slot move + `s_alloc_vgpr` grow; on over-budget atomic_sub + abort (NO slot change). +- [ ] **Step 2: Gate (offline RGA + build)** — assembles, 0-spill; `DSWS=0` still byte-identical. +- [ ] **Step 3: Commit (when greenlit):** `git add occ_kernel_coop.s; git commit -m "dsws: feed->compute conversion (grow + sum-envelope reservation/abort)"` + +### Task 3.4: Role-agnostic POOLTERM terminal + +**Files:** +- Modify: `occ_kernel_coop.s` (every role's loop checks the `ti≥TOTAL` terminal broadcast at the SAME boundary it checks for conversion; a converted wave re-checks immediately) + +**Interfaces:** +- Consumes: the existing POOLTERM `ti≥TOTAL` feed broadcast (KG `dac0bb8c`). + +- [ ] **Step 1: Hoist the terminal check** so it is evaluated by compute AND both feed roles at their decision boundaries, on the RAW ti (before SAFEPROBE clamp). A wave that just converted falls through to the terminal check before doing role work. +- [ ] **Step 2: Gate (offline)** — assembles; `DSWS=0` byte-identical. +- [ ] **Step 3: Commit (when greenlit):** `git add occ_kernel_coop.s; git commit -m "dsws: role-agnostic POOLTERM terminal (no wave stranded past drain)"` + +### Task 3.5 [SUPERVISED GPU]: Gate 2 — dynamic oracle-green + conversion-storm + +**Files:** none (run only). + +- [ ] **Step 1: Prepare ONE gated dispatch** — `DSWS_DYN=1 4c2a2b r2 STORE=1`, normal watermarks, streamed. **STOP for go.** +- [ ] **Step 2: On greenlight, dynamic oracle** across `{down, down_pf, square}`. Expected: `ok=N bad=0`, conversions visible in the snapshot (role slots move), fence FIRED, clean teardown, no brick. +- [ ] **Step 3: Conversion-storm stress** — `DSWS_LOW`/`DSWS_HIGH` tight + `DSWS_EPOCHSHIFT=0` (max conversion rate) + many repeats (oracle STORE=1, x10). Expected: STILL `bad=0` under the storm — proves the lock-free protocol has no race (the strong-oracle+repeats discipline that caught 136/552 before). Any `bad>0` → STOP, a conversion race exists; bisect offline. +- [ ] **Step 4:** Record Gate-2 results in `RESULT_DSWS.md`. Do not proceed to tuning until dynamic + storm are both clean. + +--- + +## Phase 4 — Adaptivity proof + tuning + +### Task 4.1: Role-count + conversion-counter instrumentation + +**Files:** +- Modify: `occ_kernel_coop.s` (per-direction conversion counters into snapshot), `occ_dispatch.cpp` (decode + print a compact mix-over-time table) + +- [ ] **Step 1: Add 4 conversion counters** (one per direction) + emit role slots each snapshot; harness prints `t, nComp, nAfeed, nBfeed, convs[4]`. +- [ ] **Step 2: Gate (offline build + RGA).** +- [ ] **Step 3: Commit (when greenlit):** `git add occ_kernel_coop.s occ_dispatch.cpp; git commit -m "dsws: mix-over-time + conversion-counter instrumentation"` + +### Task 4.2 [SUPERVISED GPU]: Converge-from-wrong-start proof + +**Files:** none (run only). + +- [ ] **Step 1: Prepare ONE gated dispatch** — feed-bound shape (`down`, N=2560), deliberately wrong launch mix (e.g. `6c1a1b`, compute-heavy). **STOP for go.** +- [ ] **Step 2: On greenlight, run** and watch the mix table + TF. Expected (the money shot): the controller **shifts toward more feed** (nAfeed/nBfeed climb, nComp falls) and **TF climbs** as it converges, settling near the static-optimal mix. Compare adaptive-from-wrong-start vs static-wrong-start (should beat it) and vs static-optimal (should approach it). +- [ ] **Step 3:** Record the convergence trace + TF curve in `RESULT_DSWS.md`. This is the "it actually adapts" evidence. + +### Task 4.3 [SUPERVISED GPU]: Tuning sweep + issue-mix verification + +**Files:** none (run only); record in `RESULT_DSWS.md`. + +- [ ] **Step 1:** Sweep `{LOW, HIGH, RINGD, EPOCH_SHIFT}` on `down` and `down_pf`, **one gated dispatch at a time** (no auto-sweep on the display GPU — each cell is a separate greenlit run, or a compositor-safe chunked batch with the per-chunk hang-abort). Oracle (STORE=1) before perf (STORE=0) for any new geometry. +- [ ] **Step 2: `--att` issue-mix** on the winning config. Expected: the compute waves issue **near-pure WMMA** — measurably fewer non-WMMA issues per WMMA than the 31:32 static baseline (the wall we set out to break). +- [ ] **Step 3: Success-metric gate:** the adaptive kernel (a) oracle-correct, (b) beats the static 3-role baseline AND the 165.7 TF winner on `down`/`down_pf`, (c) demonstrably adapts across the two shapes (different settled mixes). Record TF, the settled mixes, and the `--att` deltas. + +### Task 4.4: Bank the outcome + +**Files:** +- Modify: `RESULT_DSWS.md` (final), `MAD305_DSWS_MASTER.md` (§2 numbers + §9 next steps) + +- [ ] **Step 1:** Finalize `RESULT_DSWS.md` (numbers, settled mixes, issue-mix deltas, honest verdict incl. any null result). +- [ ] **Step 2:** Update the master doc; `mneme_write` a session_summary banking the result + the controller design. +- [ ] **Step 3: Commit (when greenlit).** Jira MAD-305 update. + +--- + +## Self-Review + +**1. Spec coverage** (every spec section → a task): +- 3-role economy / role slots → T1.2, T2.1. ✓ +- Data-path fixed, feed floor=1, compute floor≥1 → enforced in T3.2/T3.3 floor guards; no dual-path code (compute always A-ring) per T1.2. ✓ +- Sensing / watermark bands → T2.2 (logic), T2.3 (asm read-only), T2.4 (sane gate). ✓ +- Conversion both directions → T3.2 (shrink), T3.3 (grow). ✓ +- Epoch + gate-CAS single-winner → T2.2 (model), T3.1 (asm). ✓ +- Sum-envelope reservation → T2.2 (`reserve_grow`), T3.3 (asm). ✓ +- Role-agnostic POOLTERM terminal → T3.4. ✓ +- No barrier / lock-free → inherited (no new rendezvous); verified by storm T3.5. ✓ +- Two-gate sequencing → Gate 1 = T1.4, Gate 2 = T3.5. ✓ +- Conversion-storm race stress → T3.5 step 3. ✓ +- Adaptivity proof (converge-from-wrong-start) → T4.2. ✓ +- Success metric (beats baselines + --att issue cut) → T4.3. ✓ +- Supervised GPU discipline / brick safety → every `[SUPERVISED GPU]` task + Global Constraints. ✓ +- Open details (oracle determinism, segcnt source, LDS sizing, V_fat/V_lean/BUDGET, feed check cadence) → resolved in T1.4 (RGA constants), T2.1 (LDS assert), T2.3 (segcnt + cadence), oracle tolerance carried from coop in T1.3. ✓ + +**2. Placeholder scan:** No "TBD/implement later." Assembly tasks are gate-defined (precise structural change + exact gate command), which is the honest unit for hand-asm — not a placeholder. ✓ + +**3. Type consistency:** `DswsCfg` fields, defsym names (`NCOMP/NAFEED/NBFEED/RINGD/RINGD_A/EPOCH_SHIFT/DSWS/STORE`), LDS symbols (`ARING_OFF/PROD_A_OFF/CONS_A_OFF/NCOMP_SLOT/NAFEED_SLOT/NBFEED_SLOT/GATE_OFF/VRESV_OFF/SEGCNT_OFF`), and model functions (`watermark_decision/epoch_of/gate_try_win/reserve_grow`) are used consistently across tasks. ✓ + +No gaps found. diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_PHASEB_CONVERSION.md b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_PHASEB_CONVERSION.md new file mode 100644 index 000000000000..c643c00423f7 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_PHASEB_CONVERSION.md @@ -0,0 +1,400 @@ +# DSWS Phase B — Runtime Role Conversion Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add runtime {compute↔A-feed↔B-feed} role conversion to the GPU-proven v2 claim-based split-K substrate (`occ_kernel_dsws.s`) without orphaning output, jamming a feed, or bricking. + +**Architecture:** Per-epoch snapshot quiesce (Decision 1) + bail-time commit (Decision 2), so the claimer's quiesce counter *is* the snapshot handshake. Conversion control law is ported near-verbatim from the proven `occ_kernel_coop.s` DSWS path (`try_gate`, reservation envelope, sensors), transcribing `dsws_ctrl_model.cpp` exactly. Offline-first: extend + TDD the CPU control model, then assemble/RGA/dry-print gate every kernel change, then three supervised GPU gates. + +**Tech Stack:** Hand-written gfx1201 (RDNA4, wave32) assembly; raw-PM4 dispatch (`occ_dispatch.cpp`); CPU fp8 e4m3 oracle (`fp8_oracle.cpp`); C++17 control model (`dsws_ctrl_model.cpp`); `s_alloc_vgpr` dyn-VGPR armed by PM4 RSRC2 bit 6. + +**Design spec:** `SPEC_DSWS_PHASEB_CONVERSION.md` (this directory). Read it before starting. + +## Global Constraints + +- A GPU brick is a **BUG**, never a tax. A hang = full STOP + report; never auto-fire the next variant. +- **The user greenlights EVERY GPU dispatch individually.** Display GPU → only compositor-safe chunked sub-second dispatches: `ML8_POOL=1 ML8_COOP_CHUNK=8 ML8_COOP_CHUNK_MAXS=0.75 ML8_COOP_STREAM=1`, `timeout 30`. +- **NEVER pass `--gl2c`.** SAFEPROBE + bounds gate + padding stay ON. `ML8_COOP_STREAM=1` always. +- **No `s_barrier`** (mixed dyn-VGPR + `s_barrier` hard-deadlocks — proven). LDS-atomic busy-wait only. +- `occ_kernel_coop.s` is **NEVER modified** — known-good reference. All work is additive in `occ_kernel_dsws.s`. +- **Commit to git only when the user explicitly asks.** (Overrides the skill's per-task commit step: do the `git add`/`commit` step only on user request; otherwise leave the tree dirty and report.) +- Config held at Phase-A values **`G=6, SEGK=64, FM=2, FN=4`** for all Phase-B stages. +- Pre-existing dirty files (`occ_kernel_coop.s`, `occ_dispatch.cpp`, `fp8_oracle.*`, `mt_pagedattn*`) are **not ours** — never stage them; flag before any `git diff`. +- Shell is **fish** — do not use bash `set -- $var` word-splitting; use explicit args or `bash script.sh`. +- Round-table discipline (kmbandy's): implement → adversarial review (Fable + Codex) → kmbandy greenlights each GPU dispatch. + +## Existing LDS layout in `occ_kernel_dsws.s` (do not renumber — append only) + +``` +STI_OFF=0 EPOCH_OFF=4 ROWBLK_NEXT_OFF=8 BFRAG_NEXT_OFF=16 AROW_NEXT_OFF=20 +NCOMP_SLOT=24 NAFEED_SLOT=28 NBFEED_SLOT=32 GATE_OFF=36 (u32[4]→36,40,44,48) +VRESV_OFF=52 SEGCNT_OFF=56 ... then RES/ARES tile regions (ARES_OFF..ARES_BYTES) +``` +The claimer already publishes `NCOMP/NAFEED/NBFEED_SLOT`, `GATE_OFF[4]=0`, `VRESV_OFF=NCOMP*NFV+(NAFEED+NBFEED)*VLEAN`, `SEGCNT_OFF=0`. Existing LDS macros: `lds_put OFF, val`, `lds_get dst, OFF`, `lds_fetch_add dst, OFF, imm`. + +## File Structure + +- **`dsws_ctrl_model.cpp`** (modify) — add `snapshot`/`quiesce_ready`/invariant reference logic + the `N−1` cross-check. CPU source of truth the asm transcribes. +- **`test_dsws_ctrl_model.cpp`** (modify) — add thread-race tests for the new logic. +- **`occ_kernel_dsws.s`** (modify, additive) — new LDS slots; sensing; ported `try_gate`+`reserve` macros; bail-time commit; claimer snapshot/quiesce. Guarded under a `DSWS2_CONV` defsym so the pre-conversion substrate stays assemblable/testable in isolation. +- **`build_dsws.sh`** (modify) — thread `DSWS2_CONV` into `mk2()`; keep the no-conversion bins buildable. +- **`run_mix_gates.sh`** (reference, in scratchpad) — the supervised-gate driver pattern from Phase A (env `DSWS_NCOMP/NAFEED/NBFEED` + `DSWS2_NKSEG` + `timeout 30` + dmesg check, abort-on-anomaly). + +--- + +### Task 1: Extend the CPU control model — snapshot/quiesce interaction (offline TDD) + +**Files:** +- Modify: `dsws_ctrl_model.cpp` +- Test: `test_dsws_ctrl_model.cpp` + +**Interfaces:** +- Consumes: existing `watermark_decision(occ,low,high)`, `epoch_of(seg,shift)`, `gate_try_win(gate,E)`, `reserve_grow(resv,delta,budget)`. +- Produces: `struct WgSnap { uint32_t nC, nA, nB; }`; `WgSnap snapshot_counts(uint32_t nC,uint32_t nA,uint32_t nB)`; `bool quiesce_ready(uint32_t rowblk_next,uint32_t bfrag_next,uint32_t arow_next, const WgSnap& s, uint32_t G,uint32_t FN)`; `bool quiesce_ready_nm1(uint32_t quiesce_cnt, uint32_t N)`. Used by Task 5 as the transcription reference. + +- [ ] **Step 1: Write the failing test** — append to `test_dsws_ctrl_model.cpp` `main()` before the final `printf`: + +```cpp + // ---- snapshot/quiesce (Phase B Decision 1) ---- + { + // snapshot freezes the counts used to size the quiesce sentinels + WgSnap s = snapshot_counts(4, 2, 2); // G=6, FN=4 + // not ready: rowblk short of G + nC terminal bails + assert(!quiesce_ready(6 + 3, 4 + 2, 6 + 2, s, 6, 4)); // rowblk 9 < 6+4 + // ready: every counter reached threshold + snapshot bails + assert( quiesce_ready(6 + 4, 4 + 2, 6 + 2, s, 6, 4)); + // a moved partition (3c3a2b) needs different sentinels; old snapshot is wrong high + WgSnap s2 = snapshot_counts(3, 3, 2); + assert( quiesce_ready(6 + 3, 4 + 2, 6 + 3, s2, 6, 4)); + assert(!quiesce_ready(6 + 4, 4 + 2, 6 + 3, s2, 6, 4) == false); // 6+4>=6+3 ok -> ready; sanity + // N-1 cross-check agrees at the ready point (N=8 -> 7 bails) + assert( quiesce_ready_nm1(7, 8)); + assert(!quiesce_ready_nm1(6, 8)); + } +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd && g++ -std=c++17 -O2 -pthread test_dsws_ctrl_model.cpp -o /tmp/test_dsws_ctrl 2>&1 | head` +Expected: compile error — `snapshot_counts` / `quiesce_ready` / `WgSnap` not declared. + +- [ ] **Step 3: Write minimal implementation** — append to `dsws_ctrl_model.cpp`: + +```cpp +struct WgSnap { uint32_t nC, nA, nB; }; + +static inline WgSnap snapshot_counts(uint32_t nC, uint32_t nA, uint32_t nB) { + return WgSnap{nC, nA, nB}; +} + +// Sentinels = work-threshold + snapshot role-count terminal bails (Phase A arithmetic, +// with compile-time constants replaced by the per-epoch snapshot). +static inline bool quiesce_ready(uint32_t rowblk_next, uint32_t bfrag_next, + uint32_t arow_next, const WgSnap& s, + uint32_t G, uint32_t FN) { + return rowblk_next >= (G + s.nC) + && bfrag_next >= (FN + s.nB) + && arow_next >= (G + s.nA); +} + +// Role-agnostic safety net: fixed N waves, wid0 claimer never bails -> exactly N-1 bails. +static inline bool quiesce_ready_nm1(uint32_t quiesce_cnt, uint32_t N) { + return quiesce_cnt >= (N - 1); +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd && g++ -std=c++17 -O2 -pthread test_dsws_ctrl_model.cpp -o /tmp/test_dsws_ctrl && /tmp/test_dsws_ctrl` +Expected: `dsws_ctrl_model: ALL PASS` + +- [ ] **Step 5: Add the interleaving race test** — append inside the same test block: + +```cpp + { + // Under any interleaving of N-1 bails, quiesce_ready_nm1 must not fire before the last bail. + for (uint32_t trial = 0; trial < 64; ++trial) { + std::atomic cnt{0}; + std::atomic early{false}; + std::vector ts; + const uint32_t N = 8; + for (uint32_t w = 0; w < N - 1; ++w) + ts.emplace_back([&]{ + if (quiesce_ready_nm1(cnt.load(), N)) early.store(true); // read BEFORE our bump + cnt.fetch_add(1, std::memory_order_acq_rel); + }); + for (auto& t : ts) t.join(); + assert(!early.load()); // never ready with a bail still outstanding + assert(quiesce_ready_nm1(cnt.load(), N)); // ready once all N-1 landed + } + } +``` + +- [ ] **Step 6: Run to verify pass** — same command as Step 4. Expected: `ALL PASS`. + +- [ ] **Step 7: Commit** — *only if the user asks* (see Global Constraints). Otherwise report tree dirty and continue. + +```bash +git add dsws_ctrl_model.cpp test_dsws_ctrl_model.cpp +git commit -m "feat(dsws): CPU model for Phase-B snapshot/quiesce + N-1 cross-check" +``` + +--- + +### Task 2: Add Phase-B LDS state to `occ_kernel_dsws.s` (offline assemble/RGA) + +**Files:** +- Modify: `occ_kernel_dsws.s` (append LDS offsets after `SEGCNT_OFF`; add `DSWS2_CONV` defsym default 0) + +**Interfaces:** +- Produces: LDS offsets `SNAP_BASE` (u32[6] = nC/nA/nB × 2 epoch-parity buffers), `QUIESCE_CNT_OFF` (u32). Consumed by Tasks 3–5. + +- [ ] **Step 1: Add the defsym gate + offsets** — after the `SEGCNT_OFF` line, add: + +```asm +.ifndef DSWS2_CONV + .set DSWS2_CONV, 0 // 0 = pre-conversion static substrate (Phase A green); 1 = Phase B +.endif +.set SNAP_BASE, (SEGCNT_OFF + 4) // u32[6]: [parity*3 + {0:nC,1:nA,2:nB}] +.set QUIESCE_CNT_OFF,(SNAP_BASE + 6*4) // u32 role-agnostic bail counter +.set DSWS2_STATE_END,(QUIESCE_CNT_OFF + 4) +``` + +Then move the `RES/ARES` tile region base to start at `DSWS2_STATE_END` (find the current `.set` that begins the resident region right after the controller state and repoint its base to `DSWS2_STATE_END`, so the new slots don't overlap resident A/B). + +- [ ] **Step 2: Initialize the new slots in the claimer** — in `.Lclaimer` init block (near the existing `lds_put SEGCNT_OFF, 0`), add: + +```asm + lds_put QUIESCE_CNT_OFF, 0 + lds_put (SNAP_BASE + 0), NCOMP // parity-0 snapshot = launch mix + lds_put (SNAP_BASE + 4), NAFEED + lds_put (SNAP_BASE + 8), NBFEED + lds_put (SNAP_BASE + 12), NCOMP // parity-1 = launch mix too (init) + lds_put (SNAP_BASE + 16), NAFEED + lds_put (SNAP_BASE + 20), NBFEED +``` + +- [ ] **Step 3: Assemble both no-conversion and conversion variants (offline, no GPU)** + +Run: +```bash +cd && L=/opt/rocm/llvm/bin +for CONV in 0 1; do + $L/clang -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx1201 \ + -Wa,-defsym,DSWS2=1 -Wa,-defsym,DSWS2_CONV=$CONV -Wa,-defsym,FM=2 -Wa,-defsym,FN=4 \ + -Wa,-defsym,G=6 -Wa,-defsym,SEGK=64 -Wa,-defsym,SAFEPROBE=1 -Wa,-defsym,DIAG=1 \ + -Wa,-defsym,NCOMP=4 -Wa,-defsym,NAFEED=2 -Wa,-defsym,NBFEED=2 \ + -c occ_kernel_dsws.s -o /tmp/t2_$CONV.o 2>/tmp/t2_$CONV.err \ + && echo "CONV=$CONV ASSEMBLE_OK" || { echo "CONV=$CONV FAIL"; sed -n '1,15p' /tmp/t2_$CONV.err; } +done +``` +Expected: both `ASSEMBLE_OK`. + +- [ ] **Step 4: Confirm `DSWS2_CONV=0` bin is byte-identical to the Phase-A gated bin** (proves the new state is inert when conversion is off) + +Run: +```bash +cd && /opt/rocm/llvm/bin/llvm-objcopy -O binary --only-section=.text /tmp/t2_0.o /tmp/t2_0.bin +cmp /tmp/t2_0.bin occ_dsws2_4c2a2b_gd.bin && echo "INERT-OK: CONV=0 byte-identical to Phase-A green bin" \ + || echo "REGRESSION: CONV=0 diverged from Phase-A bin — investigate before proceeding" +``` +Expected: `INERT-OK`. (If the new `lds_put` init lines shift the binary, gate them under `.if DSWS2_CONV` so CONV=0 stays byte-identical — the init is only needed when conversion is on.) + +- [ ] **Step 5: RGA 0-spill gate** — Run: +```bash +cd && KSRC=occ_kernel_dsws.s ./rga_check.sh t2_conv \ + DSWS2=1 DSWS2_CONV=1 FM=2 FN=4 G=6 SEGK=64 NCOMP=4 NAFEED=2 NBFEED=2 SAFEPROBE=1 DIAG=1 2>&1 \ + | grep -iE "SGPR_SPILLS|VGPR_SPILLS|livereg" +``` +Expected: `SGPR_SPILLS=0`, `VGPR_SPILLS=0`. + +- [ ] **Step 6: Assert LDS fit** — the assemble already fails if `LDS_TOTAL_DSWS2 > 32768` (line 141 guard). Confirm no assembler error mentioning that guard fired in `/tmp/t2_1.err`. Expected: no LDS-overflow error. + +- [ ] **Step 7: Commit** — *only if user asks* (message: `feat(dsws): Phase-B LDS snapshot + quiesce slots (inert at DSWS2_CONV=0)`). + +--- + +### Task 3: Port consume-point sensing into `occ_kernel_dsws.s` (offline) + +**Files:** +- Modify: `occ_kernel_dsws.s` (add an `occ_sample` macro; wire a DIAG readout) + +**Interfaces:** +- Produces: `occ_sample \dst_a, \dst_b` macro computing `occ_A = prod_a − min(cons_a)`, `occ_B = prod_b − min(cons_b)` at the consume point. Consumed by Task 5. + +- [ ] **Step 1: Add the sensing macro** (gate under `.if DSWS2_CONV`). Mirror the coop `occ_a/occ_b` computation. Use only lean-safe temp registers (v14/v15 and scalars ≤ s65) — this code is reachable pre-grow, so a `>v15` temp is OOR-poison (SPEC §4). Read the existing claim counters (`ROWBLK_NEXT_OFF` drain vs `AROW_NEXT_OFF`, `BFRAG_NEXT_OFF`) at the point compute/feed consume, not at the segment boundary. + +```asm +.if DSWS2_CONV +.macro occ_sample dst_a, dst_b + // occ = producer - consumer, sampled where the value is consumed (SPEC §2). + // Reuse the A-ring / B-frag claim counters already maintained by the substrate. + lds_get \dst_a, AROW_NEXT_OFF // A staged so far (producer side) + lds_get s60, ROWBLK_NEXT_OFF // A consumed so far (compute claims) + s_sub_u32 \dst_a, \dst_a, s60 // occ_A + lds_get \dst_b, BFRAG_NEXT_OFF // B frags staged + // B consume proxy: frags retired this super-tile (compute reads all FN) -> depth vs FN + s_sub_u32 \dst_b, \dst_b, s60 // occ_B proxy (bounded [0, RINGD]) +.endm +.endif +``` + +*Note to implementer:* the exact producer/consumer counter identities must match how `occ_kernel_dsws.s` maintains the A-ring/B-frag rings — confirm against the `.Lbfeed`/`.Lafeed`/`.Lcompute` claim sites (grep `lds_fetch_add`) before finalizing the subtraction operands. The invariant to preserve: at the consume point `occ ∈ [0, RINGD]`. + +- [ ] **Step 2: Add a DIAG readout** — under `.if DIAG`, have `wid 0` write the last-sampled `occ_A`/`occ_B` to two spare `occ[]` output slots (e.g. `occ[26]`, `occ[27]`) so the harness can confirm the sensor oscillates (nonzero, varies) rather than reading a stuck 0. + +- [ ] **Step 3: Assemble + RGA (offline)** — repeat Task 2 Steps 3 & 5 (CONV=1). Expected: `ASSEMBLE_OK`, `SGPR_SPILLS=0 VGPR_SPILLS=0`. + +- [ ] **Step 4: Commit** — *only if user asks* (`feat(dsws): Phase-B consume-point sensing (read-only)`). + +--- + +### Task 4: Port the `try_gate` ticket + reservation envelope into `occ_kernel_dsws.s` (offline) + +**Files:** +- Modify: `occ_kernel_dsws.s` (add `try_gate` + `reserve_try` macros, transcribing coop verbatim) + +**Interfaces:** +- Produces: `try_gate \dir, \swin` (sets `\swin=1` iff this wave won the `(dir,epoch)` ticket) and `reserve_try \delta, \won` (shrink always succeeds; grow validates ≤ BUDGET). Consumed by Task 5. + +- [ ] **Step 1: Transcribe `try_gate` from `occ_kernel_coop.s`** — copy the proven macro (coop lines ~297–320) verbatim into `occ_kernel_dsws.s` under `.if DSWS2_CONV`. It computes `E = segcnt >> EPOCH_SHIFT`, reads `gate[dir]`, and does the single-winner `ds_cmpstore_rtn_b32` with operand order `vsrc0=new=E`, `vsrc1=cmp=g` (KG `9ed04f3c` — getting this backwards makes every racer "win"). Keep its scratch registers (coop uses `s62..s65`, `v5/v6/v7`) — verify these are free in every `occ_kernel_dsws.s` role body before finalizing. + +- [ ] **Step 2: Transcribe the reservation envelope** — add `reserve_try \delta, \won`: `compute→feed` shrink = `lds_fetch_add VRESV_OFF, -delta` (always `\won=1`); `feed→compute` grow = `lds_fetch_add VRESV_OFF, +delta` then compare the returned prior+delta vs `BUDGET`; if over, `lds_fetch_add VRESV_OFF, -delta` and `\won=0`. Transcribes `reserve_grow` in `dsws_ctrl_model.cpp` exactly. + +- [ ] **Step 3: Add a DIAG self-test path** — under `.if DIAG && DSWS2_CONV`, add a compile-time-selectable stub (`DSWS2_TICKET_SELFTEST` defsym) where every non-claimer wave calls `try_gate 0, s50` once and writes the win-count to an `occ[]` slot. Assemble-only; used to sanity-check the ticket wins exactly once per epoch on GPU in Task 6 if desired. + +- [ ] **Step 4: Assemble + RGA (offline)** — Task 2 Steps 3 & 5 (CONV=1). Expected `ASSEMBLE_OK`, 0 spills. Also assemble with `DSWS2_TICKET_SELFTEST=1` → `ASSEMBLE_OK`. + +- [ ] **Step 5: Commit** — *only if user asks* (`feat(dsws): port try_gate ticket + reservation envelope`). + +--- + +### Task 5: Wire bail-time commit + claimer snapshot/quiesce (offline integration — the crux) + +**Files:** +- Modify: `occ_kernel_dsws.s` (`.Lcompute`/`.Lafeed`/`.Lbfeed` terminal-bail paths; `.Lclaimer_wait_done` + broadcast) + +**Interfaces:** +- Consumes: `occ_sample`, `try_gate`, `reserve_try`, `WgSnap` sentinel arithmetic (Task 1), the LDS slots (Task 2). +- Produces: the complete Phase-B conversion path (`DSWS2_CONV=1`). + +- [ ] **Step 1: Add the decision at each role's kseg boundary** — under `.if DSWS2_CONV`, before a non-claimer wave loops to claim the next super-tile: `occ_sample s_a, s_b` → `watermark_decision` (inline: `occHIGH`→dir over-serve) → `try_gate dir, s_win`. Store `s_win` + intended `dir` in private scalars (NOT LDS). All temps v14/v15 / scalar-only (pre-grow OOR guard). + +- [ ] **Step 2: Add the commit at the terminal bail — ordered BEFORE the QUIESCE_CNT bump** — in each role's terminal-bail path (where it currently exits the super-tile), insert, guarded by `s_win`: + +```asm + // --- Phase B bail-time commit (SPEC §3.4); runs only if this wave won a ticket --- + // (a) floor guard: CAS-dec source slot only if > 1 + // (b) reserve_try delta, s_ok (shrink always ok; grow may abort) + // (c) on ok: CAS role slots (dec source / inc dest); flip private role reg; + // s_alloc_vgpr GROW(NFV)/SHRINK(32) with SCC-retry + // (d) on floor-fail or reserve-abort: cancel conversion, remain current role + // ORDERING: all of the above completes BEFORE the QUIESCE_CNT increment below. +``` + +Then the existing terminal path increments the (new) `QUIESCE_CNT`: + +```asm + lds_fetch_add s61, QUIESCE_CNT_OFF, 1 // exactly one bump per non-claimer wave/super-tile +``` + +Implementer: the CAS-dec floor guard and the role-slot dec/inc are `ds_cmpstore_rtn_b32` loops on `NCOMP_SLOT`/`NAFEED_SLOT`/`NBFEED_SLOT`; the ticket already bounds concurrency to ≤2 writers/boundary. Keep `s_alloc_vgpr` GROW/SHRINK exactly as coop does (SCC-retry loop), and keep every pre-grow temp in v14/v15. + +- [ ] **Step 2b: Guard the `s_alloc_vgpr` OOR-poison window (SPEC §4 — #1 brick risk)** — audit every register live across the GROW in Step 2c. Any LDS/atomic temp read *before* GROW completes must be v14/v15. Add an in-code comment block marking the pre-grow window, exactly as `occ_kernel_coop.s` does. **This is the review focus before Task 7.** + +- [ ] **Step 3: Switch the quiesce sentinels to the snapshot** — in `.Lclaimer_wait_done`, replace the compile-time constants: + +```asm + // BEFORE (Phase A): ROWBLK_NEXT >= G+NCOMP ; BFRAG_NEXT >= FN+NBFEED ; AROW_NEXT >= G+NAFEED + // AFTER (Phase B): read this-epoch parity snapshot, size sentinels from it. + lds_get s45, EPOCH_OFF + s_and_b32 s45, s45, 1 // parity + s_lshl_b32 s45, s45, ... // -> byte offset into SNAP_BASE (parity*12) + // load snap.nC/nA/nB, compute G+nC, FN+nB, G+nA, compare against the three claim counters +``` + +Keep the existing three-counter structure — only the RHS changes from constants to `G + snap.nC` etc. Additionally gate the advance on `QUIESCE_CNT_OFF >= (WAVES-1)` and, under `.if DIAG`, assert it agrees with the three snapshot sentinels (write a mismatch flag to an `occ[]` slot). + +- [ ] **Step 4: Snapshot at broadcast + reset QUIESCE_CNT** — in the claimer's next-super-tile broadcast (near `lds_put STI_OFF` / epoch bump), after quiesce passes: write live `NCOMP/NAFEED/NBFEED_SLOT` into the `[E+1 parity]` `SNAP_BASE` slots, then `lds_put QUIESCE_CNT_OFF, 0`, then bump epoch LAST (preserve the `STI_OFF`-before-`EPOCH_OFF` ordering). + +- [ ] **Step 5: Assemble all variants + RGA (offline)** — Run: +```bash +cd && L=/opt/rocm/llvm/bin +for MIX in "4 2 2" "6 1 1" "2 3 3"; do read NC NA NB <<< "$MIX" + $L/clang -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx1201 -Wa,-defsym,DSWS2=1 \ + -Wa,-defsym,DSWS2_CONV=1 -Wa,-defsym,FM=2 -Wa,-defsym,FN=4 -Wa,-defsym,G=6 -Wa,-defsym,SEGK=64 \ + -Wa,-defsym,SAFEPROBE=1 -Wa,-defsym,DIAG=1 -Wa,-defsym,NCOMP=$NC -Wa,-defsym,NAFEED=$NA -Wa,-defsym,NBFEED=$NB \ + -c occ_kernel_dsws.s -o /tmp/t5.o 2>/tmp/t5.err && echo "$MIX ASSEMBLE_OK" || { echo "$MIX FAIL"; sed -n '1,15p' /tmp/t5.err; } +done +``` +*(Run this via `bash script.sh` — fish won't `read <<<` the same way.)* Expected: all three `ASSEMBLE_OK`. +Then RGA (Task 2 Step 5) → `SGPR_SPILLS=0 VGPR_SPILLS=0`. + +- [ ] **Step 6: Dry-print sanity (offline, no GPU)** — Run: +```bash +cd && DSWS2_DRYRUN=1 DSWS_NCOMP=4 DSWS_NAFEED=2 DSWS_NBFEED=2 DSWS2_NKSEG=1 \ + ./occ_dispatch --dsws2 4c2a2b 2>&1 | grep -iE "NCOMP|n_kseg|tier|REFUSE" +``` +Expected: prints `NCOMP=4 NAFEED=2 NBFEED=2`, no `REFUSE`. (Rebuild `occ_dispatch` first only if `occ_dispatch.cpp` changed — it should not for Phase B.) + +- [ ] **Step 7: Re-run the CPU control model** — `g++ -std=c++17 -O2 -pthread test_dsws_ctrl_model.cpp -o /tmp/test_dsws_ctrl && /tmp/test_dsws_ctrl` → `ALL PASS` (guards the transcription reference didn't drift). + +- [ ] **Step 8: Commit** — *only if user asks* (`feat(dsws): Phase-B bail-time commit + claimer snapshot/quiesce`). + +--- + +### Task 6: [SUPERVISED GPU] Static-mix-through-conversion re-baseline gate + +**Goal:** Prove the conversion code path is *inert-safe* — with watermarks set so **no conversion fires**, reproduce the Phase-A green across all 3 mixes × 2 tiers. This isolates "did adding the conversion machinery regress the proven substrate" from "does conversion itself work" (Task 7). + +- [ ] **Step 1: Build the conversion bins** — assemble `occ_dsws2_{4c2a2b,6c1a1b,2c3a3b}_gd.bin` with `DSWS2_CONV=1` (Task 5 Step 5 loop + `llvm-objcopy` to `.bin`). Confirm each 0-spill. + +- [ ] **Step 2: STOP — request greenlight.** Present the offline gates (assemble, RGA, dry-print, CPU model all green) and ask kmbandy to greenlight the first Phase-B GPU dispatch. Do not proceed without explicit go. + +- [ ] **Step 3: Run the re-baseline sweep** — set watermarks unreachable so no conversion fires (e.g. `DSWS_LOW=0` and `DSWS_HIGH` ≥ RINGD, or a `DSWS2_CONV_FREEZE=1` env if wired). Use the Phase-A driver pattern (`scratchpad/run_mix_gates.sh`, `DSWS_NCOMP/NAFEED/NBFEED` matching each mix, `DSWS2_NKSEG` 1 then 8, `timeout 30`, dmesg check, abort-on-anomaly). +Expected per gate: `dsws2 oracle CLEAN`, `ok=1536 bad=0`, `occ[0]=0`, exit 0, dmesg silent. **Under DIAG: the `QUIESCE_CNT==N−1` vs snapshot-sentinel agreement flag must read agree (0 mismatches).** + +- [ ] **Step 4: On any anomaly (bad>0, hang/124, dmesg fault, DIAG mismatch): full STOP + bisect.** Report; do not run Task 7. + +- [ ] **Step 5: Record** — on all-green, write a KG note (mirror the Phase-A milestone entry) and update the plan checkboxes. + +--- + +### Task 7: [SUPERVISED GPU] Dynamic-mix gate (conversions firing) + +**Goal:** Watermarks that *do* fire conversions; oracle stays green as the partition moves. + +- [ ] **Step 1: Round-table the `s_alloc_vgpr` OOR window (Task 5 Step 2b) before any dispatch.** Fable + Codex adversarial review of the pre-grow register discipline in the commit path — this is the #1 brick risk. Fix findings offline; re-gate (assemble/RGA). + +- [ ] **Step 2: STOP — request greenlight** for the first conversion-firing dispatch. + +- [ ] **Step 3: Run TIGHT first, single mix** — `DSWS2_NKSEG=1` (exact tier), `4c2a2b`, watermarks that fire (`DSWS_LOW=1 DSWS_HIGH=RINGD-1 EPOCH_SHIFT=3`), `timeout 30`, dmesg check. +Expected: `oracle CLEAN ok=… bad=0`, `occ[0]=0`, exit 0, dmesg silent, DIAG agreement. The conversion actually fired: confirm via a DIAG conversion-count `occ[]` slot > 0. + +- [ ] **Step 4: On green, widen** — LOOSE tier (`DSWS2_NKSEG=8`), then the other two mixes, one supervised dispatch at a time. Any anomaly → full STOP + bisect. + +- [ ] **Step 5: Record** — KG note + checkboxes on all-green. + +--- + +### Task 8: [SUPERVISED GPU] Storm gate (race-hunt) + +**Goal:** The lock-free race stress that historically caught 136/552 — tight watermarks + `EPOCH_SHIFT=0` (every segment its own epoch = max conversion pressure) + ×10 repeats. + +- [ ] **Step 1: STOP — request greenlight.** + +- [ ] **Step 2: Run the storm** — `EPOCH_SHIFT=0`, `DSWS_LOW`/`DSWS_HIGH` tight, loop each (mix × tier) ×10 via a `bash` driver, `timeout 30` each, dmesg check between. Expected every iteration: `oracle CLEAN bad=0`, exit 0, dmesg silent, DIAG agreement, conversion-count > 0. + +- [ ] **Step 3: On any single-iteration anomaly: full STOP + bisect** (a storm failure is a real race — do not average it away). + +- [ ] **Step 4: Record the Phase-B completion milestone** — KG session_summary: dynamic role conversion GPU-proven through storm, both tiers, all mixes, zero bricks. Note Phase 4 (adaptivity + tuning + `--att` issue-mix on ml8 `down`/`down_pf`) as the next campaign, separate spec. + +--- + +## Self-Review + +**Spec coverage:** Decision 1 (snapshot quiesce) → Tasks 1,2,5; Decision 2 (bail-time commit) → Task 5; sensing §2 → Task 3; ticket+reservation §3 → Task 4; safety/OOR §4 → Task 5 Step 2b + Task 7 Step 1; control-model §5 → Task 1; build sequence §6 (static→dynamic→storm) → Tasks 6,7,8; testing → offline gates in every task + GPU gates 6–8; success metric → Task 8. `N−1` DIAG safety net → Task 1 + Task 5 Step 3 + Task 6 Step 3. All spec sections covered. + +**Placeholder scan:** The asm tasks intentionally show the load-bearing blocks (offsets, snapshot arithmetic, commit ordering, sentinel edit) as concrete code and mark the two spots (occ counter identities in Task 3 Step 1; try_gate scratch-register freeness in Task 4 Step 1) where the implementer must confirm against the live kernel before finalizing — these are verification instructions, not deferred design. No "TBD/add error handling/similar to Task N". + +**Type/name consistency:** `WgSnap`/`snapshot_counts`/`quiesce_ready`/`quiesce_ready_nm1` consistent Task 1↔5. LDS names (`SNAP_BASE`, `QUIESCE_CNT_OFF`) consistent Tasks 2↔5. `try_gate \dir,\swin`, `reserve_try \delta,\won`, `occ_sample` consistent Tasks 3,4↔5. Gate commands use the same defsym set throughout. diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_SUBSTRATE_V2.md b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_SUBSTRATE_V2.md new file mode 100644 index 000000000000..c8f9c05ffda9 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_SUBSTRATE_V2.md @@ -0,0 +1,338 @@ +# DSWS Substrate v2 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Re-found the DSWS GEMM substrate so matrix work is *claimed* by whichever wave holds a role (not *owned* by compile-time wave identity), with split-K folded in — enabling correct, brick-free runtime role conversion. + +**Architecture:** New gfx1201 kernel `occ_kernel_dsws.s`. Work = a pool of `(mblk, tcol, ksi)` super-tiles; a pinned claimer (`wid 0`) broadcasts the current super-tile; live waves of each role drain shared LDS atomic counters (compute claims rowblks, feeds claim operand frags) against resident-in-LDS A/B for that super-tile; per-segment partials combine into C via `global_atomic_add`. Build static-substrate-first (oracle-green), then layer conversion. + +**Tech Stack:** Hand-written gfx1201 (RDNA4, wave32) assembly; raw-PM4 dispatch (`occ_dispatch.cpp`); CPU fp8 e4m3 oracle (`fp8_oracle.cpp`); control law (`dsws_ctrl_model.cpp`, unchanged); dyn-VGPR via `s_alloc_vgpr` (PM4 RSRC2 bit 6). + +**Source of truth:** `SPEC_DSWS_SUBSTRATE_V2.md` (this plan implements it). Read it first. + +## Global Constraints + +- A GPU brick is a **BUG**, never an accepted tax. A hang = full STOP + report; never auto-fire the next variant. +- **The user greenlights EVERY GPU dispatch individually.** Display GPU → only sub-second compositor-safe-chunked dispatches: `ML8_POOL=1 ML8_COOP_CHUNK=8 ML8_COOP_CHUNK_MAXS=0.75 ML8_COOP_STREAM=1`, timeout 30. +- **NEVER pass `--gl2c`** (MES-crash landmine). SAFEPROBE + bounds gate + padding stay ON. +- Commit only when the user asks. +- Barrier-free / lock-free: LDS atomics + busy-wait only. **No `s_barrier`.** +- `occ_kernel_coop.s` (1716B `DSWS=0` coop binary) is **never modified** — it is the known-good reference. +- Single-variable isolation; fix bugs don't dodge them; never declare a wall from an unmeasured assumption. +- Every offline task gate = (1) assembles clean, (2) RGA **0-spill**. GPU oracle gates are batched and **[SUPERVISED]**. + +## TDD adaptation for hand-asm + +This is hand-written assembly, so the red/green cycle is **build-gate-driven**: +- **"Failing test" →** the gate that is red before the step (RGA reports spill / wrong bin size / oracle mismatch / missing symbol). +- **"Green" →** assembles clean + RGA 0-spill (offline) and, at milestones, GPU oracle `bad=0` (supervised). +- CPU-only logic (oracle tolerance math, control law) keeps true pytest-style TDD. + +## File Structure + +- **Create `occ_kernel_dsws.s`** — the v2 kernel. Owns: prologue/arming scaffold (lifted from `occ_kernel_coop.s`), v2 LDS layout, the pinned claimer, resident-A/B feed loops (claim-based), the compute loop (claim rowblk → WMMA over `SEGK` → partial-flush), role-agnostic terminal, and (Phase B) conversion actuation. +- **Modify `fp8_oracle.cpp`** — add a relative-tolerance compare mode (Tier 2); exact bit-match stays the default (Tier 1). +- **Modify `occ_dispatch.cpp`** — add the v2 launch path: super-tile pool sizing, `G`/`SEGK`/`n_kseg` params, v2 LDS size, RSRC2 arm, tiered oracle, reuse compositor-safe chunking + streaming + snapshot readback. +- **Modify `build_dsws.sh`** — add v2 build target(s) with `G/SEGK/FM/FN/NCOMP/NAFEED/NBFEED` defsyms + RGA gate (mirrors the existing `mk()` pattern). +- **Modify `RESULT_DSWS.md`** — append v2 records. +- **Unchanged:** `dsws_ctrl_model.cpp`, `test_dsws_ctrl_model.cpp` (carry over verbatim). + +## Naming / symbols (used across tasks — define once, reuse exactly) + +- `G` = compile-time cooperative M-extent (rowblks per super-tile) = `NCOMP_MAX`. + **First fitting config (LDS-budget-derived, A1):** `G=6, SEGK=64, FM=2, FN=4` → resident + A(`192·SEGK`) + B(`64·SEGK`) = `256·SEGK = 16 KB` at `SEGK=64`, fits the 32 KB group segment. + `SEGK=256` busts it (64 KB); `SEGK` is the LDS/combine-traffic knob swept in Phase C. +- `SEGK` = split-K segment size (K-elements/segment); `n_kseg = KT/SEGK`; `ksi ∈ [0,n_kseg)` = segment index. +- Super-tile id `sti ∈ [0, TOTAL_super)`, `TOTAL_super = (M/(G·16·FM)) · NTL · n_kseg`. Decode: `ksi = sti % n_kseg`; `t = sti / n_kseg`; `tcol = t % NTL`; `mblk = t / NTL`. +- LDS claim/handshake words (v2): `STI_OFF` (broadcast super-tile id), `ROWBLK_NEXT_OFF` (per-super-tile rowblk claim counter), `ROWBLK_DONE_OFF` (per-super-tile completion counter), plus the carried-over `EPOCH_OFF`, role slots `NCOMP_SLOT/NAFEED_SLOT/NBFEED_SLOT`, `GATE_OFF[4]`, `VRESV_OFF`, `SEGCNT_OFF` (from T2.1). Resident regions: `BRES_OFF` (resident B), `ARES_OFF` (resident A). Feed claim counters: `BFRAG_NEXT_OFF`, `AROW_NEXT_OFF` (reset per super-tile by the claimer). +- Kernel build flag: `DSWS2=1` selects the v2 path/file (analogous to existing `DSWS`). + +--- + +## Execution status (2026-06-30, EOD) + +- **A1–A7 + A8 host-launch wiring: DONE, offline-gated** (ASSEMBLE_OK, RGA 0-spill, `mk2` bin 4840B, + harness compiles, dry-print clean). Reviewed via two round-tables (Opus + Codex) which caught **5 + brick/correctness bugs**, all fixed + re-gated (see `RESULT_DSWS.md` "SUBSTRATE v2" section). +- Config landed differs from the early task text: **`SEGK=64` (not 256)** for LDS fit; **15-kernarg + contract** (`n_kseg` derived in-kernel, chunk terminal via `occ[24]`, sentinel retire) — NOT the s15/s16/s17 + scheme; **tiered oracle = tight-vs-loose tolerance** (not bit-exact). These supersede the inline task text + below; `RESULT_DSWS.md` + `occ_kernel_dsws.s` header are authoritative. +- **NEXT = A8 [SUPERVISED GPU], morning 2026-07-01.** First cell: + `ML8_POOL=1 ML8_COOP_CHUNK=8 ML8_COOP_CHUNK_MAXS=0.75 ML8_COOP_STREAM=1 DSWS2_NKSEG=1 timeout 30 ./occ_dispatch --dsws2 4c2a2b` + → expect `ok=32 bad=0` (TIGHT), `occ[0]=0`, fence FIRED, no brick. kmbandy greenlights. Then LOOSE + mixes. +- Phase-B caveat: quiesce sentinels use compile-time role counts (static-only); conversion must use live + counts / epoch-snapshot drained counters. + +--- + +## Phase A — Static split-K + claim substrate (fixed roles, no conversion) + +### Task A1: v2 scaffold + LDS layout + harness pool sizing + +**Files:** +- Create: `occ_kernel_dsws.s` (copy the proven prologue/arming/WMMA-macro/addressing/fp8-LUT/common-retire scaffold from `occ_kernel_coop.s`; gut the work-decomposition body, leave a `s_endpgm` stub in each role). +- Modify: `occ_dispatch.cpp` (add `run_dsws2(...)` entry computing `G/SEGK/n_kseg/TOTAL_super`, allocating C, sizing LDS; wire a `--dsws2` arg; no oracle yet). +- Modify: `build_dsws.sh` (add `mk2()` building `occ_kernel_dsws.s` with `DSWS2=1 FM FN G SEGK NCOMP NAFEED NBFEED`). + +**Interfaces:** +- Produces: the v2 LDS symbol block (all `*_OFF` above) with a `.if LDS_TOTAL_DSWS2 > 65536 .error` assert; `run_dsws2` launch; `mk2` build target. +- Consumes: nothing (scaffold). + +- [ ] **Step 1 (red):** `./build_dsws.sh` has no v2 target → adding `mk2 ...` fails to assemble (file absent). Confirm the failure message. +- [ ] **Step 2:** Create `occ_kernel_dsws.s` = scaffold + the v2 LDS symbol block. Lay out offsets sequentially; assert `LDS_TOTAL_DSWS2 ≤ 32768` (group segment). Each role label (`.Lclaimer`, `.Lbfeed`, `.Lafeed`, `.Lcompute`) is a bare `s_endpgm` stub. +- [ ] **Step 3 (green):** assemble + RGA. Expected: assembles clean, RGA `livereg` within budget, **0 spill**. + +```bash +# assemble (mirror build_dsws.sh mk()): +L=/opt/rocm/llvm/bin; cd +$L/clang -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx1201 \ + -Wa,-defsym,DSWS2=1 -Wa,-defsym,FM=2 -Wa,-defsym,FN=4 -Wa,-defsym,G=6 -Wa,-defsym,SEGK=64 \ + -Wa,-defsym,NCOMP=4 -Wa,-defsym,NAFEED=2 -Wa,-defsym,NBFEED=2 -Wa,-defsym,SAFEPROBE=1 -Wa,-defsym,DIAG=1 \ + -c occ_kernel_dsws.s -o occ_dsws2.o && echo ASSEMBLE_OK +# RGA 0-spill gate: +KSRC=occ_kernel_dsws.s ./rga_check.sh dsws2_a1 DSWS2=1 FM=2 FN=4 G=6 SEGK=64 NCOMP=4 NAFEED=2 NBFEED=2 SAFEPROBE=1 | grep -E "livereg|spill" +# Expected: livereg printed, "spill" count = 0 +``` + +- [ ] **Step 4:** Build the harness; confirm `--dsws2` parses and prints the computed `G/SEGK/n_kseg/TOTAL_super/LDS bytes` without launching. + +```bash +ROCM=/opt/rocm; PM4=../dvgpr_pm4 +systemd-run --user --scope -p MemoryMax=4G clang++ -std=c++17 -O2 -Wall -Wno-unused \ + -I "$PM4/vendor/compat" -I "$PM4/vendor" -I "$PM4" -I "$ROCM/include" \ + occ_dispatch.cpp fp8_oracle.cpp "$PM4/vendor/PM4Packet.cpp" "$PM4/vendor/BasePacket.cpp" \ + "$ROCM/lib/libhsakmt.a" -ldrm_amdgpu -ldrm -lnuma -lpthread -ldl -lrt -o occ_dispatch && echo BUILD_OK +# dry print (no GPU): a --dsws2 path that prints params and returns before dispatch when DSWS2_DRYRUN=1 +DSWS2_DRYRUN=1 ./occ_dispatch --dsws2 # Expected: prints G=4 SEGK=256 n_kseg=.. TOTAL_super=.. LDS=..B +``` + +- [ ] **Step 5:** (no commit — await user.) + +### Task A2: Tiered oracle (CPU TDD) + +**Files:** +- Modify: `fp8_oracle.h` / `fp8_oracle.cpp` (add `OracleCmp oracle_compare(const float* got, const float* ref, long n, float rel, float abs_)` returning `{bool ok; long bad; double max_rel;}` — generalizes the existing inline gate `fabs(got-ref) > rel*fabs(ref)+abs_`). +- Modify: `occ_dispatch.cpp` (define tier thresholds `TIGHT={5e-3,1e-2}`, `LOOSE={3e-2,2e-2}`; the v2 oracle path picks TIGHT when `n_kseg==1`, LOOSE when `n_kseg>1` — actual v2 compare call is wired in A8). +- Test: a `main()`-guarded self-test in `fp8_oracle.cpp` (`#ifdef ORACLE_SELFTEST`). + +**Interfaces:** +- Produces: `OracleCmp oracle_compare(got, ref, n, rel, abs_)`; tier threshold constants. +- Consumes: nothing. + +- [ ] **Step 1 (red):** write the self-test. TIGHT(5e-3) PASSES identical / 0.1%-perturbed, FAILS 1%-perturbed; LOOSE(3e-2) PASSES 1%, FAILS 5%. + +```cpp +// ORACLE_SELFTEST main (rel,abs): +assert( oracle_compare(ident, ident, 256, 5e-3f,1e-2f).ok); // identical +assert( oracle_compare(perturb_0p1pct, ident, 256, 5e-3f,1e-2f).ok); +assert(!oracle_compare(perturb_1pct, ident, 256, 5e-3f,1e-2f).ok); // tight rejects 1% +assert( oracle_compare(perturb_1pct, ident, 256, 3e-2f,2e-2f).ok); // loose accepts 1% +assert(!oracle_compare(perturb_5pct, ident, 256, 3e-2f,2e-2f).ok); // loose rejects 5% +``` +(Use ref values ~O(1) so the abs term doesn't dominate the rel term in the test.) + +- [ ] **Step 2 (red):** compile self-test → fails (function undefined). + +```bash +clang++ -std=c++17 -DORACLE_SELFTEST fp8_oracle.cpp -o /tmp/oracle_selftest 2>&1 | head +``` + +- [ ] **Step 3 (green):** implement `oracle_compare`; rebuild self-test → all asserts pass. + +```bash +clang++ -std=c++17 -DORACLE_SELFTEST fp8_oracle.cpp -o /tmp/oracle_selftest && /tmp/oracle_selftest && echo ORACLE_OK +``` + +- [ ] **Step 4:** add the `TIGHT`/`LOOSE` tier constants + the `n_kseg`-based selector in `occ_dispatch.cpp` (call site wired in A8). Harness rebuilds (command from A1 Step 4). +- [ ] **Step 5:** (no commit — await user.) + +### Task A3: Pinned claimer + super-tile broadcast + +**Files:** +- Modify: `occ_kernel_dsws.s` (`.Lclaimer` = `wid 0` only: loop { `global_atomic_add` next `sti` at the claim offset; decode `(mblk,tcol,ksi)`; reset `ROWBLK_NEXT_OFF=0`, `ROWBLK_DONE_OFF=0`, `BFRAG_NEXT_OFF=0`, `AROW_NEXT_OFF=0`; publish `STI_OFF`; bump `EPOCH_OFF`; also bump `SEGCNT_OFF` (the clock); gate next claim on `ROWBLK_DONE==G` (stub `G` constant until A7) }; terminal publish `sti≥TOTAL_super`). + +**Interfaces:** +- Produces: `STI_OFF` broadcast + epoch bump in the proven `STI`-before-`EPOCH` order; per-super-tile counter resets. +- Consumes: the LDS layout (A1), the claim atomic pattern (lifted from coop `global_atomic_add offset:20`). + +- [ ] **Step 1:** implement `.Lclaimer`. Decode uses the formulas in "Naming/symbols". Publish order: write `STI_OFF` → reset the 4 per-super-tile counters → `s_add EPOCH_OFF` LAST (followers read `STI` before the epoch bump). Non-`wid0` waves skip claimer and fall to their role loop. +- [ ] **Step 2 (green):** assemble + RGA 0-spill (commands from A1 Step 3, retag `dsws2_a3`). +- [ ] **Step 3:** (no commit — await user.) + +### Task A4: Resident-B feed loop (claim frags → stage B) + +**Files:** +- Modify: `occ_kernel_dsws.s` (`.Lbfeed`: follow epoch/`STI`; decode `(tcol,ksi)`; loop { `ds`-atomic-add `BFRAG_NEXT_OFF` to claim frag `f ∈ [0,FN)`; if `f≥FN` break; `global_load_tr_b64` B for `(tcol,ksi,f)`; `ds_store` into resident `BRES_OFF + f*` }; then wait next super-tile. `wid 0` (claimer) ALSO runs this after claiming — it is a B-feed-class wave). + +**Interfaces:** +- Produces: resident B fully staged for the current super-tile at `BRES_OFF`; identity-free frag claiming via `BFRAG_NEXT_OFF`. +- Consumes: `STI_OFF`/`EPOCH_OFF` broadcast (A3); B global addressing (lift from coop B-feed `s_mul..tcol` decode). + +- [ ] **Step 1:** implement the claim-frag → load_tr → ds_store resident staging. B segment size = `FN·16` cols × `SEGK` K → resident bytes at `BRES_OFF` (verify within `LDS_TOTAL_DSWS2`). +- [ ] **Step 2 (green):** assemble + RGA 0-spill (retag `dsws2_a4`). +- [ ] **Step 3:** (no commit — await user.) + +### Task A5: Resident-A feed loop (claim rowblk-A → stage A) + +**Files:** +- Modify: `occ_kernel_dsws.s` (`.Lafeed`: follow epoch/`STI`; decode `(mblk,ksi)`; loop { `ds`-atomic-add `AROW_NEXT_OFF` to claim rowblk-A index `r ∈ [0,G)`; if `r≥G` break; `global_load` A for absolute rowblk `mblk*G+r`, segment `ksi` (rows `(mblk*G+r)*16*FM`, K-range `ksi*SEGK`); `ds_store` into resident `ARES_OFF + r*` }; wait next super-tile). + +**Interfaces:** +- Produces: resident A fully staged for the current super-tile at `ARES_OFF`; identity-free rowblk-A claiming via `AROW_NEXT_OFF`. +- Consumes: A3 broadcast; A global addressing (lift from coop compute A-saddr `rowblk*(16*FM)*K + k*16`, with `rowblk = mblk*G + r`, K offset `ksi*SEGK`). + +- [ ] **Step 1:** implement claim-rowblk-A → global_load → ds_store resident staging. A segment size = `G·16·FM` rows × `SEGK` K → resident bytes at `ARES_OFF` (verify fit). +- [ ] **Step 2 (green):** assemble + RGA 0-spill (retag `dsws2_a5`). +- [ ] **Step 3:** (no commit — await user.) + +### Task A6: Compute loop — claim rowblk → WMMA → partial-flush + +**Files:** +- Modify: `occ_kernel_dsws.s` (`.Lcompute`: follow epoch/`STI`; wait until resident A/B staged (busy-wait on the feed claim counters reaching `FN`/`G`, or a `staged` flag the feeds set); loop { `ds`-atomic-add `ROWBLK_NEXT_OFF` to claim `rowblk r ∈ [0,G)`; if `r≥G` break; `s_alloc_vgpr NFV`; WMMA `FM×FN` over `SEGK/16` k-steps reading resident A[r]/B from LDS, accumulate fp32; `global_atomic_add_f32` the `FM×FN` partials into `C[(mblk*G+r) , tcol]`; `s_alloc_vgpr 32` (shrink); `ds`-atomic-add `ROWBLK_DONE_OFF`++ } ). + +**Interfaces:** +- Produces: partial-summed C contributions; `ROWBLK_DONE` increments (consumed by A7 handshake). +- Consumes: resident A/B (A4/A5); WMMA macros + C addressing (lift from coop compute body); the fp8-LUT front-end if `ML8` (carry the coop variant). + +- [ ] **Step 1:** **verify `global_atomic_add_f32` encodes on gfx1201** (assemble a one-liner). If rejected → fall back to a CAS loop (`ds`/`global_atomic_cmpswap` on the C word) and note it in `RESULT_DSWS.md`. + +```bash +echo 'global_atomic_add_f32 v0, v1, s[0:1]' | $L/clang -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx1201 -c - -o /dev/null 2>&1 && echo ADDF32_OK || echo ADDF32_REJECTED_use_CAS +``` + +- [ ] **Step 2:** implement the compute loop. Staging-ready wait = busy-wait on `BFRAG_NEXT≥FN && AROW_NEXT≥G` (feeds advance these as they stage). Grow/shrink each guarded by SCC-retry (brick-class rule). +- [ ] **Step 3 (green):** assemble + RGA 0-spill (retag `dsws2_a6`). Resident-read + WMMA path must stay ≤ budget under the grown `NFV`. +- [ ] **Step 4:** (no commit — await user.) + +### Task A7: Completion handshake + role-agnostic terminal + +**Files:** +- Modify: `occ_kernel_dsws.s` (claimer's next-claim gate now real: spin until `ROWBLK_DONE_OFF == G` before claiming/broadcasting the next super-tile — frees resident A/B safely. Terminal: every role, at its decision boundary, checks broadcast `sti ≥ TOTAL_super` → retire via the shared `.Lretire` path). + +**Interfaces:** +- Produces: the safe resident-B/A lifetime (claimer advances only when all `G` rowblks done+flushed) + uniform retire. +- Consumes: `ROWBLK_DONE_OFF` (A6), `STI_OFF`/`TOTAL_super`. + +- [ ] **Step 1:** wire the claimer advance-gate (`ROWBLK_DONE==G`) and the role-agnostic `sti≥TOTAL_super` terminal in all four roles. +- [ ] **Step 2 (green):** assemble + RGA 0-spill (retag `dsws2_a7`). Add the 6-bin v2 build set to `build_dsws.sh` (`mk2` for the swept role mixes at `n_kseg=1` and a `n_kseg>1` config); `./build_dsws.sh` → all OK, `fail=0`. +- [ ] **Step 3:** (no commit — await user.) + +### Task A8 [SUPERVISED GPU]: Step-1 BIG GATE — static substrate oracle-green + +**Files:** none (run only). **STOP and request greenlight before EACH dispatch.** + +- [ ] **Step 1:** Prepare ONE gated dispatch — static mix (e.g. `4c2a2b`), `n_kseg=1`, compositor-safe chunked, streamed. **STOP for go.** + +```bash +ML8_POOL=1 ML8_COOP_CHUNK=8 ML8_COOP_CHUNK_MAXS=0.75 ML8_COOP_STREAM=1 \ +DSWS2_NKSEG=1 timeout 30 ./occ_dispatch --dsws2 4c2a2b # Expected: ok=N bad=0 (EXACT), no brick, clean teardown +``` + +- [ ] **Step 2:** On green, repeat at `n_kseg>1` (TOL gate) and at the other mixes (`6c1a1b`, `2c3a3b`) — **one greenlit dispatch at a time**. Expected each: `bad=0`, fence FIRED, clean teardown. +- [ ] **Step 3:** Record Step-1 results (mixes, `n_kseg`, ok/bad, LDS bytes, RGA livereg) in `RESULT_DSWS.md`. **Gate: do not start Phase B until the static substrate is oracle-green at both tiers.** + +--- + +## Phase B — Conversion (layered on the green static substrate) + +### Task B1: Port gate-CAS ticket + reservation + sensors into v2 (no actuation) + +**Files:** +- Modify: `occ_kernel_dsws.s` (lift the proven `try_gate dir, swin` macro, the `mark`/`mark_set`/`mark_inc` snapshot macros, the sensors, and the reservation init from `occ_kernel_coop.s` DSWS path; call `try_gate` at the per-kseg compute boundary as a unit, incrementing a would-win snapshot slot — **no role change**). + +**Interfaces:** +- Produces: `try_gate`, sensors, `vgpr_reserved` init in v2; a would-win counter in the snapshot. +- Consumes: role slots + `GATE_OFF` + `SEGCNT_OFF` + `VRESV_OFF` (A1 layout). + +- [ ] **Step 1:** port the macros + reservation init (`VRESV_OFF = NCOMP*NFV + (NAFEED+NBFEED)*VLEAN`); add the unit `try_gate` call at the compute per-kseg boundary. +- [ ] **Step 2 (green):** assemble + RGA 0-spill (retag `dsws2_b1`). +- [ ] **Step 3:** (no commit — await user.) + +### Task B2: compute→feed shrink actuation + +**Files:** +- Modify: `occ_kernel_dsws.s` (at the compute per-kseg boundary, on `occ_X1`: CAS-dec `nComp`, inc `nFeedX`, `atomic_sub vgpr_reserved` by `NFV−VLEAN`, `s_alloc_vgpr 32`, flip private role register, `s_branch` into the feed loop. Shrink always succeeds). + +**Interfaces:** +- Produces: a wave that leaves compute and starts claiming feed work (no work-item handoff — the payoff of the claim model). +- Consumes: `try_gate` (B1), role slots, reservation, sensors. + +- [ ] **Step 1:** implement the shrink conversion on the `try_gate` win flag (dirs 0/1). Floor-guard `nComp>1` via CAS-dec-if-greater. A converted wave falls through to the terminal check before doing feed work. +- [ ] **Step 2 (green):** assemble + RGA 0-spill (retag `dsws2_b2`). +- [ ] **Step 3:** (no commit — await user.) + +### Task B3: feed→compute grow actuation + envelope abort + +**Files:** +- Modify: `occ_kernel_dsws.s` (at a feed inter-claim point, on `occ_X>HIGH & try_gate(feedX→compute) win & nFeedX>1`: `reserve_grow` = `atomic_add vgpr_reserved` by `NFV−VLEAN`, validate `≤ BUDGET`; on success CAS-dec `nFeedX`/inc `nComp`, flip role register, `s_alloc_vgpr NFV`, `s_branch` into compute; on over-budget `atomic_sub` + abort, stay feed). + +**Interfaces:** +- Produces: a wave that leaves feed and starts claiming rowblks; the sum-envelope abort path. +- Consumes: `try_gate` (B1), reservation model, `BUDGET` (per-SIMD VGPR budget defsym from RGA/ISA). + +- [ ] **Step 1:** implement the grow conversion with reserve→validate→commit-or-undo (transcribe `reserve_grow` from `dsws_ctrl_model.cpp` exactly). Order: reserve first, validate, then slot move + grow; on fail, undo + abort (NO slot change). +- [ ] **Step 2 (green):** assemble + RGA 0-spill (retag `dsws2_b3`); `./build_dsws.sh` v2 set all OK; DSWS2 static-mix still assembles for a degenerate `LOW=0,HIGH=RINGD` (no-conversion) build to keep an offline-comparable control. +- [ ] **Step 3:** (no commit — await user.) + +### Task B4 [SUPERVISED GPU]: Step-2 GATE — dynamic oracle-green + conversion storm + +**Files:** none (run only). **STOP for greenlight before EACH dispatch.** + +- [ ] **Step 1:** ONE gated dispatch — dynamic mix enabled, normal watermarks, `n_kseg=1` EXACT, streamed. **STOP for go.** Expected: `ok=N bad=0`, conversions visible in the snapshot (role slots move), clean teardown, no brick. +- [ ] **Step 2:** On green — conversion-storm: tight `DSWS_LOW`/`DSWS_HIGH` + `EPOCH_SHIFT=0` (max conversion rate) + ×10 repeats, `n_kseg=1` EXACT then `n_kseg>1` TOL. Expected: STILL `bad=0`. Any `bad>0` → STOP, a conversion race exists; bisect offline. +- [ ] **Step 3:** Record Step-2 results in `RESULT_DSWS.md`. **Gate: do not proceed to tuning until dynamic + storm are both clean.** + +--- + +## Phase C — Adaptivity proof + tuning + +### Task C1: Mix-over-time + conversion-counter instrumentation + +**Files:** +- Modify: `occ_kernel_dsws.s` (4 per-direction conversion counters into the snapshot; emit role slots each snapshot). +- Modify: `occ_dispatch.cpp` (decode + print `t, nComp, nAfeed, nBfeed, convs[4]`). + +- [ ] **Step 1:** add the counters + harness table. **Step 2 (green):** assemble + RGA 0-spill; harness builds. **Step 3:** (no commit — await user.) + +### Task C2 [SUPERVISED GPU]: Converge-from-wrong-start proof + +**Files:** none. **STOP for go.** + +- [ ] **Step 1:** feed-bound shape (`down`, N=2560), deliberately wrong launch mix (`6c1a1b`). **STOP for go.** +- [ ] **Step 2:** run; watch mix table + TF. Expected (the money shot): controller shifts toward feed (nAfeed/nBfeed climb, nComp falls), TF climbs, settles near static-optimal. Compare adaptive-from-wrong vs static-wrong vs static-optimal. +- [ ] **Step 3:** record convergence trace + TF curve in `RESULT_DSWS.md`. + +### Task C3 [SUPERVISED GPU]: Tuning sweep + issue-mix + +**Files:** none (record in `RESULT_DSWS.md`). **STOP for go each cell.** + +- [ ] **Step 1:** sweep `{LOW, HIGH, RINGD, EPOCH_SHIFT, G, SEGK}` on `down`/`down_pf`, **one gated dispatch at a time** (oracle STORE=1 before perf STORE=0 for any new geometry). +- [ ] **Step 2:** `--att` issue-mix on the winner. Expected: compute waves issue near-pure WMMA — measurably fewer non-WMMA issues per WMMA than the 31:32 static baseline. +- [ ] **Step 3: Success-metric gate:** adaptive kernel (a) oracle-correct, (b) beats the static 3-role baseline AND the 165.7 TF winner on `down`/`down_pf`, (c) demonstrably adapts across shapes (different settled mixes). Record TF, settled mixes, `--att` deltas. + +### Task C4: Bank the outcome + +**Files:** +- Modify: `RESULT_DSWS.md` (final), `MAD305_DSWS_MASTER.md` (numbers + next steps). + +- [ ] **Step 1:** finalize `RESULT_DSWS.md` (numbers, settled mixes, issue-mix deltas, honest verdict incl. any null result). **Step 2:** `mneme_write` a session_summary banking the result + v2 substrate design. **Step 3:** commit (when greenlit) + Jira MAD-305 update. + +--- + +## Self-Review + +**Spec coverage** (every `SPEC_DSWS_SUBSTRATE_V2.md` section → a task): +- §1 work pool & claim model → A1 (layout), A3 (super-tile claim), A6 (rowblk claim), A7 (coverage via handshake). ✓ +- §2 resident A/B + completion handshake → A4 (B), A5 (A), A7 (handshake). ✓ +- §3 role tags + pinned claimer → A1 (slots), A3 (`wid 0` claimer/clock). ✓ +- §4 partial-C + tiered oracle → A2 (oracle), A6 (`global_atomic_add_f32` flush + encodability check). ✓ +- §5 conversion actuation → B1 (gate/sensors/reservation), B2 (shrink), B3 (grow+abort). ✓ +- §6 file structure + build sequence + gates → A1 (new file + build), A8/B4/C2/C3 (supervised gates). ✓ +- Testing (tiered oracle, RGA, storm, control-law) → A2, every offline step's RGA, B4 storm, carried-over `test_dsws_ctrl_model.cpp`. ✓ +- Risks (LDS budget, atomic-add-f32, combine traffic, G overshoot) → A1 assert, A6 Step 1, C3 measure, B-floor guards. ✓ + +**Placeholder scan:** build/RGA/oracle commands are concrete; `` is the only literal path placeholder (the dvgpr_occ dir). Resident-region byte formulas are explicit; exact instruction sequences for the WMMA/load bodies are lifted from the named coop labels (discovered against RGA per the hand-asm TDD note). + +**Symbol consistency:** `G`, `SEGK`, `ksi`, `n_kseg`, `sti`, `TOTAL_super`, `*_OFF`, `NFV`, `VLEAN`, `try_gate` used identically across tasks; decode formulas defined once in "Naming/symbols" and referenced. diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/RESULT_DSWS.md b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/RESULT_DSWS.md new file mode 100644 index 000000000000..51ffa1ac2301 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/RESULT_DSWS.md @@ -0,0 +1,353 @@ +# DSWS Adaptive Wave-Role Controller — Results + +Substrate: `occ_kernel_coop.s` (dyn-VGPR + split-K + POOLTERM). Plan: `PLAN_DSWS_CONTROLLER.md`. +Spec: `SPEC_DSWS_CONTROLLER.md`. This doc is appended through Phases 1–4. + +## Phase 1 — static 3-role substrate + +### Offline gates (T1.1–T1.3) + +- **T1.1 config + refuse-path** (`occ_dispatch.cpp --dsws`): host compile clean; refusal smoke GREEN + (`DSWS_NCOMP=0` → `REFUSE nComp>=1` rc=4; valid-but-unbuilt → `bin NOT BUILT` rc=4; `LOW>HIGH` → refuse). +- **T1.2 A-feed role + A-ring** (`occ_kernel_coop.s`, behind `.if DSWS`): + - `DSWS=0` **byte-identical** to the proven coop d0 (1716B) — the substrate is untouched. + - `DSWS=1` assembles for all 3 plan cells × {dyn,static}: 4c2a2b, 6c1a1b, 2c3a3b. + - Design: **frag-partitioned B** (B-feed wave owns `{ni : ni%NBFEED==b_id}`, bumps `prod_b[b_id]`, + compute gates on `min(prod_b[*])`); **band-partitioned A** (A-feed wave owns `{bnd : bnd%NAFEED==a_id}`, + band has one producer `prod_a[bnd]` + one consumer `cons_a[bnd]`); lead B-feed (wid 0) is the sole + tile-claimer/broadcaster; A-feed + non-lead B-feed follow the ti/epoch broadcast. Zero inter-feed + rendezvous → barrier-free invariant preserved (KG `4ce31886`). + +### RGA static-analysis constants (gfx1201, 2×4 tile, dyn) — needed for the Phase-3 reservation envelope + +| metric | value | source | +|---|---|---| +| **V_fat** (compute peak-live, grown wave) | **82** VGPR | RGA livereg "Maximum # VGPR used" | +| **NFV** (compute grow alloc target) | **112** VGPR | `((FB+2*FN+15)&~15)`; HW rounds 108→120 | +| **V_lean** (feed/A-feed lean alloc) | **32** VGPR | `s_alloc_vgpr 32` lean footprint | +| VGPR spills / SGPR spills / scratch | **0 / 0 / 0** | RGA stats (all 3 cells) | +| per-grow delta (V_fat_alloc − V_lean) | **80** VGPR | NFV(112) − 32 | +| BUDGET (per-SIMD VGPR pool) | *TBD T1.4/Phase 3* | ISA / occupancy math | + +### T2.2 control-law CPU model (TDD, offline) + +`dsws_ctrl_model.cpp` + `test_dsws_ctrl_model.cpp`: `watermark_decision` / `epoch_of` / +`gate_try_win` (single-winner CAS) / `reserve_grow` (envelope) — **ALL PASS**, stable across repeats +under real multi-thread races. Locks the control semantics the Phase-3 asm must match 1:1. + +### T1.3 dispatch wiring — DONE (2026-06-28) + +`run_mbcoop` extended with two defaulted params (`totalWaves`, `ldsBytesOverride`) — proven coop callers +byte-identical; `--dsws` calls it with `P=NCOMP`, `totalWaves=N`, `ldsBytesOverride=LDS_TOTAL_DSWS`. +`build_dsws.sh` builds all 6 bins (`occ_dsws_{4c2a2b,6c1a1b,2c3a3b}_r2[_dyn]_gd.bin`). **Offline gate GREEN:** +harness links clean; all 6 cells assemble; RGA 0-spill (livereg 82); both refuse-paths intact +(invalid → validation refuse; valid+unbuilt → bin-guard refuse; valid+built → would dispatch = T1.4). + +
(superseded) T1.3 in-progress notes + +Approach: extend `run_mbcoop` with two **defaulted** params so the proven 2-role coop callers stay +byte-identical, then `--dsws` calls it with `P=NCOMP`, `totalWaves=N`, and the DSWS LDS size. + +- **DONE:** `run_mbcoop` signature + `WAVES_LAUNCH = totalWaves>0 ? totalWaves : (1+P)` (occ_dispatch.cpp). + Harness recompiles clean (niced build, 2026-06-27 23:08). +- **REMAINING (3 items, all offline):** + 1. `ldsBytes` override line in `run_mbcoop` (`ldsBytesOverride>0 ? ldsBytesOverride : coop-formula`). + 2. Replace the T1.1 `--dsws` stub (`[T1.1] ... dispatch wiring lands in T1.3`) with the real call: + compute `N`, `ldsDsws = BRING + 4*nComp+16 + 4*(nBfeed-1) + RINGD*nComp*FM*256 + 8*nComp` + (FM=2,FN=4 fixed v1 tile), loop `{down, down_pf}`, small tile-multiple oracle first, + `run_mbcoop(dswsBin, dyn, pool, Mo,No,Ko, 2,4, nComp, ringd, /*fullCheck*/true, /*GENDIV*/true, + 1,0.0, /*totalWaves*/N, /*ldsOverride*/ldsDsws)`, gate on `bad==0`. + 3. `build_dsws.sh` — emits `occ_dsws_cab_r[_dyn]_gd.bin` (DSWS=1 FM=2 FN=4 POOLTERM=1) + for cells {4c2a2b, 6c1a1b, 2c3a3b} × {dyn,static}; + RGA gate. +- **Gate:** `./build_dsws.sh static` (all cells assemble + RGA 0-spill) + harness links. No GPU. +
+ +### Gate 1 dispatch #1 — HUNG (not bricked), 2026-06-29 + +First GPU dispatch of the campaign fired (user-greenlit). `4c2a2b` static, `down_pf`, single-tile, but at the +harness **default pool=64** (I left `ML8_POOL` unset). Result: **clean software deadlock**, NOT a brick — +`timeout 30` caught it, teardown deliberately did not destroy the queue, dmesg silent (zero ring/fence/fault/MES), +both GPUs responsive. Wedge frame (direct atomics, reliable): `maxlive=64 claim=64 live=1` → 63 WGs correctly +retired on `ti>=TOTAL`, the **one WG that won `ti=0` deadlocked** in produce/consume. All fine markers 0 — but +that was instrumentation blindness (see below), not proof of no progress. + +**Root cause of the blindness (2026-06-29):** `DIAG` defaults to 0 and `build_dsws.sh` never passed it → every +`mark` compiled to nothing. NOT the 2026-06-24 "v2 clobber" hypothesis (v2 is set once at kernel:276 and never +rewritten; the v2==0-gated `lds_put` provably works since the 63 WGs retired via LDS broadcast). Additionally the +live DSWS feed bodies (`.Lbfeed_role`/`.Lafeed_role`) had ZERO marks — the feed marks lived in the dead +`.Lfeed_role`. FIX (all offline, verified): `build_dsws.sh` now passes `DIAG=${DIAG:-1}`; added 10 marks to the +DSWS feed bodies (B: feedPhase/ti + slotok[18]/loadtr[19]/dsstore[20]/publish[21]; A: reached[28]/Aload[29]/ +dsstore[30]/publish[31] on fresh occ slots 28-31); harness zeros+prints occ[28..31] + a DIAGINIT line (occ23-27). +Gates: 6 DIAG=1 bins clean, 55 mark atomics in disasm, RGA livereg 82 / 0-spill, DSWS=0 still 1716B byte-identical. + +**Reflight plan (pool=1, awaiting greenlight):** matches the kernel's POOL=1 DIAGNOSTIC TERMINAL design, removes +the 64-WG claim-contention variable, and with DIAG=1 will localize the exact wedge instruction: +`ML8_COOP_STREAM=1 ML8_COOP_CHUNK=8 ML8_COOP_CHUNK_MAXS=0.75 ML8_POOL=1 DSWS_NCOMP=4 DSWS_NAFEED=2 DSWS_NBFEED=2` +`DSWS_RINGD=2 DSWS_DYN=0 DSWS_ONLY=down_pf DSWS_ORACLE_MTL=1 DSWS_ORACLE_NTL=1 timeout 30 ./occ_dispatch --dsws` + +### Gate 1 dispatch #2+#3 — root cause LOCALIZED + CONFIRMED, first DSWS green (2026-06-29) + +**#2 (pool=1, 4c2a2b):** HUNG again, but DIAG-instrumented this time. Wedge frame: +`INIT adm1 tmr1 lds1 rdv1 | feedPh2 compPh4 cons3 tiles0 | feed:tr2 pub3 comp:dsB2 wm3`. Decode: init/rendezvous +FULLY PASSED (kills the old init-deadlock fear for static); the protocol RAN and made real progress (B published 3 +steps, compute did 3 WMMA) then froze at step ~3 of 32. → a MID-K-LOOP producer/consumer stall, not an init wedge. + +**Root cause (LOCALIZED by evidence + code, CONFIRMED by #3):** the A-feed loop is **band-outer / K-inner** +(`occ_kernel_coop.s` `.Lafeed_role`: `.rept NCOMP` wraps the K-loop), so when an A-feed wave owns >1 band it +produces band b's ENTIRE 32-step K-loop before touching band b+NAFEED. The compute waves consuming the not-yet-fed +bands starve → never release their consumer counters → `min_cons` stays low → the shared B-ring (depth RINGD=2) +can't recycle slots → B and all compute jam → WG wedges at ~step RINGD. B-feed does it RIGHT (K-outer / frag-inner). + +**#3 (pool=1, 2c3a3b) — CONFIRMING PROBE: `oracle CLEAN ok=16 bad=0`.** With NAFEED=3 ≥ NCOMP=2 every band has +its own A-feed wave → zero band-sequential starvation → GREEN. Airtight: the only changed variable is bands-per- +A-feed-wave, and it flips green↔hang exactly as predicted. **FIRST oracle-green of the DSWS 3-role kernel — the +cooperative A+B-feed→compute→store protocol is NUMERICALLY CORRECT.** The hang was a pure feed-scheduling bug. + +All three dispatches HUNG-or-completed CLEANLY (no brick; dmesg silent, R9700 responsive each time). + +**THE FIX (applied + verified 2026-06-29):** restructured `.Lafeed_role` to **K-outer / band-inner** (mirror +B-feed): every K-step produces one step for ALL owned bands, so no compute band starves. `astep==k` fell out for +free (use the loop counter directly; recompute per-band saddr from trow each iter — lean feed has SGPR headroom). +Offline gates: 6 cells assemble, RGA livereg 82 / 0-spill, DSWS=0 still 1716B byte-identical. + +### Gate 1 dispatch #4 — FIX VALIDATED: 4c2a2b oracle CLEAN ok=32 bad=0 (2026-06-29) + +The headline cell that deadlocked at step 3 in #2 now runs the full 32-step tile and matches the fp8 oracle +exactly (32/32 frags), clean exit, no hang, no brick. + +### Gate 1 dispatch #5 — extreme case: 6c1a1b oracle CLEAN ok=48 bad=0 (2026-06-29) + +One A-feed wave cycling all 6 bands per K-step (192x64 tile) — the worst case for the old band-outer bug — +runs CLEAN, 48/48 frags. **Both cells that deadlocked (4c2a2b, 6c1a1b) now pass; 2c3a3b passed pre-fix.** +The K-outer A-feed fix is validated across the full band-count range. Pending: a 2c3a3b post-fix regression +re-check (low risk — NAFEED>=NCOMP means each wave owns <=1 band, the simplest K-outer case), then Gate 1 (T1.4) +is fully GREEN and the DSWS static 3-role substrate (Phase 1) is DONE. 5 GPU dispatches total, ZERO bricks. + +### ✅ GATE 1 (T1.4) PASSED — 2026-06-29 + +All three role-mix cells oracle-CLEAN at pool=1 single-tile down_pf, static (DYNVGPR=0): +**2c3a3b ok=16 / 4c2a2b ok=32 / 6c1a1b ok=48, all bad=0.** The DSWS static 3-role substrate (Phase 1) is DONE — +the cooperative fp8 GEMM with separate fat-compute / lean-A-feed / lean-B-feed waves is numerically correct across +the full role-mix range. 6 supervised GPU dispatches total today, ZERO bricks (every one hung-cleanly or completed; +dmesg silent each time). Net of fixes that got here: SAFEPROBE feed ti-clamps + SAFEPROBE=1 build; DIAG=1 + feed-body +marks (instrumentation was DIAG=0/uncompiled, not the old "v2 clobber"); and THE bug — A-feed band-outer/K-inner +loop starved un-fed compute bands → fixed to K-outer/band-inner. + +NEXT: Phase 2 (sensing + role slots + reservation counter), then Phase 3 (conversion/epoch-ticket + dyn-VGPR +DYNVGPR=1), then Phase 4 (adaptivity proof + tuning). NOTE the pool=64 path still hung in dispatch #1 — pool>1 +cross-WG behavior is a separate open item to revisit before any multi-WG perf run (Phase 1 gate is pool=1 by design). + +
(historical) Gate 1 pre-dispatch notes + +First display-GPU dispatch of the whole campaign. As of 2026-06-28: ZERO GPU dispatches run — all offline. + +**HARDENING PASS — DONE 2026-06-29 (all offline, zero GPU):** +1. ✅ **SAFEPROBE `ti`-clamps added to BOTH A-feed and B-feed decodes** (`occ_kernel_coop.s`), mirroring + compute lines 471-472; gated `.if SAFEPROBE` inside `.if DSWS`. The `s_min` clamp is the safety mechanism, + independent of the (possibly-flaky) diagnostic `mark`. On inspection the feeds were ALREADY covered on the + *vector* vaddr (the v8/v9/v10 clamps live in the common prologue, run by all waves) AND already retire on + `ti>=TOTAL` (terminal test) — the new clamp closes the one remaining asymmetry on the *scalar* saddr. +2. ✅ **Rebuilt all 6 bins with `SAFEPROBE=1`** — build_dsws.sh was passing it ZERO times → bins were + SAFEPROBE=0 = NO clamps compiled at all; THIS was the real lever. RGA re-gated: livereg **82**, + SCRATCH/SGPR_SPILLS/VGPR_SPILLS all **0**. DSWS=0 still byte-identical to `.clean_bins` (**1716B**). +3. ⏳ **First contact = single tile** (`DSWS_ORACLE_MTL=1 DSWS_ORACLE_NTL=1` → TOTAL=1) — dispatch-time env, + set in the gated command below. Awaiting user greenlight. + +**Brick-risk assessment (honest, 2026-06-28):** LOW-to-MODERATE. Static (`DYNVGPR=0`) **eliminates the +dyn-VGPR brick class** (#1 historical vector — no `s_alloc_vgpr`). Residual: (a) protocol-hang → teardown +wedge (200 lines of new hand-asm, never executed; `timeout 30` + 0.75s chunk-abort are the net, but +recovery-from-hang is less proven than clean completion); (b) OOB page-fault from a bad `ti` — mitigated +to ~proven-coop level by the hardening pass above. NOT claiming "can't brick." + +**Dispatch discipline:** set `ML8_COOP_CHUNK` (compositor-safe, opt-in; `ML8_COOP_CHUNK_MAXS=0.75`) + +`ML8_COOP_STREAM=1` + `timeout 30`. ONE gated dispatch at a time. Hang/brick = full STOP + report, never +auto-fire the next. Proposed first command (post-hardening, static, single-tile, down_pf): +`ML8_COOP_STREAM=1 ML8_COOP_CHUNK=8 ML8_COOP_CHUNK_MAXS=0.75 DSWS_NCOMP=4 DSWS_NAFEED=2 DSWS_NBFEED=2` +`DSWS_RINGD=2 DSWS_DYN=0 DSWS_ONLY=down_pf DSWS_ORACLE_MTL=1 DSWS_ORACLE_NTL=1 timeout 30 ./occ_dispatch --dsws` + +
+ +--- + +## Phase 2 — Sensing + role slots + reservation (offline, no GPU) — 2026-06-30 + +**Status:** Tasks 2.1, 2.2, 2.3 DONE + gated offline. Task 2.4 (supervised GPU sensor-sanity) is the next STOP-for-go dispatch. + +### T2.1 — controller LDS state (`occ_kernel_coop.s`, behind `.if DSWS`) +Appended after `CONS_A_OFF` so all proven offsets stay byte-identical: +`NCOMP_SLOT / NAFEED_SLOT / NBFEED_SLOT` (role slots), `GATE_OFF[4]` (one epoch-gate per conversion +direction: 0=comp→Afeed 1=comp→Bfeed 2=Afeed→comp 3=Bfeed→comp), `VRESV_OFF` (vgpr_reserved +envelope), `SEGCNT_OFF` (per-WG segments_processed). `LDS_TOTAL_DSWS` rolled forward; existing +`>65536 .error` now covers them. **Gate:** 6 bins assemble; RGA 0-spill, `USED_LDS=32768`, livereg +**V_fat=82**; DSWS=0 byte-identical (1716B). + +### T2.2 — CPU control-law model (already banked, re-verified) +`dsws_ctrl_model.cpp` + `test_dsws_ctrl_model.cpp` → `ALL PASS` (watermark_decision / epoch_of / +gate_try_win single-winner / reserve_grow envelope). This is the 1:1 semantic spec the Phase-3 asm transcribes. + +### T2.3 — leader state-init + read-only sensing + snapshot instrumentation +- **Reservation unit clarified:** the envelope uses the hardware **alloc** footprint, not live-peak. + Fat = `NFV` (=112 this tile, the actual `s_alloc_vgpr` target); lean = `VLEAN`=32 (the feeds' + `s_alloc_vgpr 32`). `vgpr_reserved` init = `NCOMP*NFV + (NAFEED+NBFEED)*VLEAN`. (RESULT table's + "V_fat=82" is live-peak, a different measurement; the grow-delta accounting `NFV−32=80` confirms + the envelope is in alloc units.) +- **Leader init** (extends the barrier-free INITFLAG publish, all lean-32 there): role slots ← launch + mix, gates ← 0, `vgpr_reserved` ← launch envelope, `segcnt` ← 0. Constants written by each wave's + lane-0 to WG-shared LDS (idempotent — identical values). +- **Read-only sensing (NO actuation):** + - New `mark_set off,val` macro = lane-0 **plain global_store** to `occ[off]` (vs `mark`'s atomic-MAX), + so the harness's 200 ms poll sees a sensor *oscillate* instead of latching its max. + - Compute **segment boundary** (once per tile): `occ_b = min(prod_b)−cons` → occ[32]; `occ_a = + prod_a[cid]−a_step` → occ[33] (both consumer-observed ring backlog, range [0,RINGD]/[0,RINGD_A]). + - Per-WG `segcnt` bump by lead compute (cid==0) on each segment (POOL=1: one tile = one WG segment). + - Role slots streamed once per compute wave (read LDS → occ[34/35/36]) — proves the controller-state + init round-trips through cross-wave LDS. +- **`occ_dispatch.cpp`:** DIAGFINE zero-loop extended to 14..36; new fields added to the 200 ms live + stream line (`SENS occ_b.. occ_a.. roles[../../..]`) and the TIMEOUT dump (`DSWS sensors` line). + +**DEVIATION from plan (single-writer sensor, deliberate):** the plan also names a "feed inter-frag" +sensing point. I made the **compute segment boundary the sole writer** of occ_a/occ_b. Having both the +producer (feed) and consumer (compute) write the same occ slot would race and corrupt the very +oscillation T2.4 is meant to validate. Feed-side pressure reads belong in Phase 3, where the feed +actually *actuates* on them (conversion trigger). occ_a/occ_b here are the clean consumer-side signal. + +**Offline gates (all green):** 6 cells assemble; RGA 0-spill, `USED_LDS=32768`, livereg **82**; +DSWS=0 still **1716B byte-identical**; `occ_dispatch` rebuilds clean (rc=0, 0 errors). Everything +UNCOMMITTED (commit-on-ask). **NEXT = T2.4 supervised dispatch (STOP for user go).** + +--- + +## BUG (found at T2.4, fixed offline) — multi-tile A-feed starvation — 2026-06-30 + +**Surfaced by:** the first MULTI-tile DSWS dispatch ever (T2.4 at 32 tiles, `DSWS_ORACLE_MTL=4 NTL=8`). +Gate 1 and the first T2.4 run were all **single-tile** (`MTL=NTL=1`), so the per-tile claim/feed loop +was never exercised. Single-tile = CLEAN; 32-tile = **COOP TIMEOUT** (`claim=2`, compute wedged into tile 1). + +**DIAGFINE evidence (the localization):** feed published B to step 35 (into tile 1 fine); **A-feed +`publish[31]=32`** — stuck at exactly one tile's worth (KT=32); compute `prodwait=32 consRel=33`, wedged +mid tile-1 K-loop. `roles[4/2/2]` streamed correctly (init round-trips); `occ_b/occ_a=0` only because +no compute segment ever *completed*. → compute is **A-starved**. + +**Root cause (confirmed by code, not assumption):** the A-feed published `prod_a[bnd] = k+1` using the +**per-tile** K counter `k` (s26), which **resets to 0 each tile**. The compute consumer's `a_step` +(s54) is **global/cumulative across tiles** (like the B-ring step). Tile 0: both 0..31 → match → CLEAN. +Tile ≥1: compute's `a_step` is 32→63 and waits `prod_a>32`, but the A-feed re-publishes `prod_a=1..32` +→ never exceeds 32 → permanent A-starvation deadlock. The B-feed was correct (global continuous step); +the A-feed was the lone asymmetry. **Pre-existing** in the Phase-1 K-outer A-feed rewrite — NOT caused +by the Phase-2 sensing (compute wedges before reaching the boundary sensor). + +**Fix:** introduce a **global cumulative A-step `s60` (gk)** in the A-feed, used for the ring protocol +(publish count, ring slot index, slot-free gate vs `cons_a`) — mirroring the B-feed and matching +compute's global `s54`. The per-tile `k` (s26) is kept for **A-matrix addressing only** (`k*16` within +the tile's K). `s60` inits to 0 once before `.Lafeed_loop`, increments per K-step, never resets. +(Note: first picked `s49` — caught in review that it's the `lds_get/lds_put` exec-save scratch; moved +to `s60`, 0 refs kernel-wide.) + +**Offline gates (green):** 6 bins assemble; RGA 0-spill, LDS 32768, livereg 82; DSWS=0 byte-identical +(1716B). **Needs a supervised MULTI-tile re-dispatch to verify the deadlock is gone** (this is also the +real T2.4 sensor-oscillation run: 32 segments → occ_a/occ_b vary across the 200ms stream). Uncommitted. + +--- + +### ✅ GATE T2.4 PASSED — sensors report sane + multi-tile clean — 2026-06-30 + +Four supervised dispatches (static DYN=0, pool=1, compositor-safe-chunked, ZERO bricks): +1. single-tile 4c2a2b → `oracle CLEAN ok=32 bad=0` (sensing code correct/safe; too fast to read sensors). +2. **32-tile → COOP TIMEOUT** → surfaced + root-caused the multi-tile A-feed starvation bug (above). +3. 32-tile, A-feed fix → `oracle CLEAN ok=1024 bad=0` (deadlock GONE) but occ read 0 (boundary-drained). +4. 32-tile, sensors relocated to consume points → **`oracle CLEAN ok=1024 bad=0`, `occ_b=1 occ_a=1`, + `roles[4/2/2]`** — sensors read real in-range backlog, NOT pinned. **PASS.** + +**Sensor-placement refinement (banked design note for Phase 3):** occ_a/occ_b are sampled at the per-K +CONSUME points (where the gate `prod>cons` guarantees occ ∈ [1,RINGD]), NOT at the segment boundary +(where the ring has drained → occ≈0, which would make the controller read "always starved"). The +segment boundary remains only the `segcnt` decision-clock tick. The eventual watermark LOW/HIGH must be +read against this consume-point occupancy, and likely wants a per-segment max/avg, not a single sample. + +**Phase 2 status: COMPLETE** (T2.1 LDS state, T2.2 control model, T2.3 init+sensing, T2.4 supervised +gate). Plus an out-of-band WIN: the **multi-tile substrate is now proven** (ok=1024) — Gate 1 had only +ever tested single-tile, so this de-risks all of Phase 3 (real GEMMs are multi-tile). Everything static +(DYN=0), UNCOMMITTED. Open items unchanged: pool>1 cross-WG (separate), dyn-VGPR arm (Phase 3). + +--- + +## Phase 3 — Conversion + epoch/ticket + role-agnostic terminal (offline) + +### T3.1 — epoch + lock-free gate-CAS ticket (no actuation) — 2026-06-30 +`try_gate dir, swin` macro transcribes dsws_ctrl_model.cpp `epoch_of` + `gate_try_win` EXACTLY: +`E = segcnt >> EPOCH_SHIFT`; read `g = gate[dir]`; lose if `g>=E`; else lane0 does an LDS compare-swap +`ds_cmpstore_rtn_b32 v6,v5,v6(=g),v7(=E)` and WINS iff returned-old == g. Single winner per (dir,epoch) +across all racing waves (the LDS atomic serializes). New defsyms `EPOCH_SHIFT=3`, `CTRL_LOW=1`, +`CTRL_HIGH=RINGD-1` (mirror the harness DSWS_* env). New `mark_inc off` macro = lane0 atomic-add counter. +**Unit call site** (compute boundary, ALL compute waves, NO actuation): race `try_gate 0` (compute→Afeed) +each segment, atomic-inc would-win counter `occ[39]`. occ_dispatch decodes `gateWin[39]`. + +**CAS operand-order note:** RDNA4 `ds_cmpstore_rtn_b32` is `(vaddr, vCMP, vDATA)` → `MEM=(MEM==vCMP)?vDATA:MEM` +(GCN's order was flipped). Assumed cmp=g, data=E. **Validated by:** the T3.1 micro-check (would-win must be +≤1 per epoch per dir — a wrong order gives double-winners or none) and the Gate-2 conversion-storm oracle. + +**Offline gates (green):** 6 bins assemble; RGA 0-spill, LDS 32768, livereg 83; DSWS=0 byte-identical +(1716B); occ_dispatch rebuilds rc=0. UNCOMMITTED. **Optional next:** T3.1 Step-3 supervised micro-check +(1 streamed dispatch; confirms `gateWin[39] ≈ #epochs`, NOT NCOMP×#epochs) — validates the gate-CAS before +3.2/3.3 build conversions on it. Else defer to Gate 2 (T3.5). + +### ✅ T3.1 micro-check PASSED (caught + fixed a gate-CAS operand-order bug) — 2026-06-30 +First micro-check run: `gateWin[39]=3` — ANOMALOUS (last chunk reaches only epoch E=1 once, so a correct +single-winner CAS gives exactly 1). Diagnosis: my `ds_cmpstore_rtn_b32` operand order was BACKWARDS. +Verified authoritatively OFFLINE via LLVM `cmpxchg` lowering (`clang -S` of an IR `cmpxchg ptr,%cmp,%new`): +`ds_cmpstore_rtn_b32 vdst,vaddr,vsrc0,vsrc1` is `MEM=(MEM==vsrc1)?vsrc0:MEM` -> **vsrc0=NEW, vsrc1=CMP** +(the GCN order, NOT flipped as I'd assumed). The swapped form compared against E and stored g, so gate +stayed 0 and `old==g` held for every racer -> all ~NCOMP waves "won" each epoch. FIX: swap the two source +operands (`... v7(=E,new), v6(=g,cmp)`). Re-dispatch (same config, kernel-only change): **`gateWin 3 -> 1`**, +oracle CLEAN ok=1024 bad=0, no brick. Single-winner ticket PROVEN. The gate-CAS is correct for 3.2/3.3. + +--- + +## SUBSTRATE v2 — claim-based work decomposition + split-K (2026-06-30) + +**Why v2:** Phase-3 review (3/3 consensus: kmbandy + Claude + Codex) found the proven coop substrate binds +the matrix WORK decomposition to compile-time role counts + wave identity (`rowblk=trow*P+cid`, A-band 1:1 +`cid` pairing, B-frag `owner=ni%NBFEED`). Naive role conversion therefore BRICKS (hang fires before any +wrong-output). Fix = decouple work from identity: make it CLAIMABLE by whoever holds the role. Split-K +folded in (makes B resident-per-segment → replay free → rowblk-count decouples from live nComp). New design +in `SPEC_DSWS_SUBSTRATE_V2.md`; plan in `PLAN_DSWS_SUBSTRATE_V2.md`. New kernel `occ_kernel_dsws.s` (coop +kernel NEVER touched). Config: G=6 SEGK=64 FM=2 FN=4, 8-wave WG, LDS 16640B. + +**Model/workflow:** Sonnet 5 implements; Opus + Codex review; kmbandy greenlights every GPU dispatch. + +**Phase A offline — DONE, oracle/RGA-gated (no GPU yet):** +- A1 scaffold + v2 LDS layout + harness `--dsws2` dry-print + `mk2` build. +- A2 tiered oracle `oracle_compare` (fp8_oracle.cpp) — TIGHT{5e-3,1e-2} for n_kseg=1, LOOSE{3e-2,2e-2} + for n_kseg>1 (the CPU wmma_ref chain is not bit-identical to GPU WMMA, so "exact" = the proven tight + tolerance, not bit-match). Self-test passes. +- A3–A7 datapath (claimer + resident B/A feeds + compute w/ `global_atomic_add_f32` partial-flush + + completion handshake + role-agnostic sentinel terminal). `global_atomic_add_f32` ENCODES on gfx1201. +- Host launch `run_dsws2` + tiered oracle wired. + +**Round-table caught 5 brick/correctness bugs OFFLINE (all fixed + re-gated: ASSEMBLE_OK, RGA +SGPR/VGPR_SPILLS=0, mk2 bin 4840B, harness compiles, dry-print clean):** +1. [Codex] Reset/quiesce straggler race — a compute wave increments ROWBLK_DONE then loops to claim; if + descheduled, the claimer resets ROWBLK_NEXT and the straggler claims row 0 of the next super-tile with + stale state. FIX: claimer advance-gate also drains the CLAIM counters (ROWBLK_NEXT≥G+NCOMP, + BFRAG_NEXT≥FN+NBFEED, AROW_NEXT≥G+NAFEED). [BRICK] +2. [Codex] n_kseg=1 magic-div overflow (ceil(2^32/1) truncates to 0 → t=0 all sti). FIX: shift/mask decode + (`ksi=sti&mask, t=sti>>shift`, shift=ctz(n_kseg)) — handles n_kseg=1 for free. [BRICK] +3. [Sonnet flagged→Opus→Codex corrected] kernargs s16/s17 undeliverable (PM4 only defines + COMPUTE_USER_DATA_0..15; all proven paths use 15) AND s16 doubly-used as the per-chunk terminal. Opus's + first fix was INCOMPLETE (dropped the chunk terminal) — Codex caught it. FINAL: 15 kernargs (s0..s14, + USER_SGPR=15); n_kseg derived in-kernel (KT>>NKSEG_SHIFT); chunk terminal memory-carried in occ[24]; + claimer publishes SENTINEL 0xFFFFFFFF at terminal, followers retire on it. [BRICK] +4. [Opus + Codex] C body never memset=0 (only the canary) → atomic-adds onto garbage. FIX: memset(C,0) + once before the chunk loop (not per-chunk, so split-K accumulates across chunks). [wrong-oracle] +5. [Codex] DSWS2 ignored positional mix arg + no role-floor/32-bit-overflow validation. FIXED. [correctness] + +**Phase-B carry-forward:** the quiesce sentinels use compile-time NCOMP/NAFEED/NBFEED (correct for STATIC +roles only); Phase-B conversion must switch to live role counts / epoch-snapshot drained counters. (In-code +note at `.Lclaimer_wait_done`.) + +**NEXT = A8 [SUPERVISED GPU — kmbandy greenlights], morning 2026-07-01.** First cell (n_kseg=1 TIGHT/exact, +compositor-safe): `ML8_POOL=1 ML8_COOP_CHUNK=8 ML8_COOP_CHUNK_MAXS=0.75 ML8_COOP_STREAM=1 DSWS2_NKSEG=1 +timeout 30 ./occ_dispatch --dsws2 4c2a2b` → expect ok=32 bad=0, occ[0]=0 clean, fence FIRED, no brick +(TOTAL_super=32 → 4 chunks of 8). Then n_kseg=8 LOOSE + mixes 6c1a1b/2c3a3b. Gate: static substrate +oracle-green both tiers before Phase B (conversion). All uncommitted. KG: b8c689cc (A8-ready contract), +48625333 (kernel round-table), 86e33108 (the blocker + 3/3 consensus). diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_CONTROLLER.md b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_CONTROLLER.md new file mode 100644 index 000000000000..de2d332431a1 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_CONTROLLER.md @@ -0,0 +1,217 @@ +# Spec: DSWS Adaptive Wave-Role Controller (MAD-305) + +Date: 2026-06-27. Substrate: `occ_kernel_coop.s` (dyn-VGPR grow/shrink + split-K + +POOLTERM cooperative kernel, raw-PM4 dispatch on gfx1201/RDNA4 wave32). +Companions: [[MAD305_DSWS_MASTER.md]] (campaign master), [[SPEC_WAVESPEC.md]] +(static wave-spec lineage). Prior-art verdict (Murmur `5ec8a958`, KG `2dc2332d`): +the runtime-adaptive rebalancing is **novel**; the lean/fat mechanism is borrowed. + +## Goal + +One fp8 GEMM kernel that **senses its in-kernel bottleneck and rebalances its mix +of wave roles at runtime** to attack the measured wall (the VALU **issue port**: +~31 non-WMMA issues per 32 WMMA = 52% of the 307 TF ceiling). The lever is +wave-specialization — put feed/address instructions on *separate* waves so compute +waves issue near-pure WMMA — with the feed:compute split **found by the kernel per +shape, per moment** instead of hand-tuned and frozen at launch. + +This spec covers the **controller** (the novel "brain"). The static 3-role +substrate it sits on is a prerequisite build phase (Phase 1 below). + +## Scope decisions (settled in brainstorming, 2026-06-27) + +- **3 roles** from the start: `compute` (fat VGPR, holds fp32 accumulators), + `A-feed` (lean), `B-feed` (lean). Rationale: tests the actual novel thesis (the + multi-role economy), and on the `coop` substrate **both** A-feed and B-feed + relieve the issue-port wall — coop currently has compute load its *own* A direct, + so an A-feed role offloads those A-load issues off the compute stream. The + control-law code is role-count-parametric, so 3-role-first avoids throwaway. +- **Data path fixed for v1** (no runtime A-in-LDS-vs-direct switching). That is a + legitimate *future* second control axis (slow/structural cadence, richer sensing) + but it makes compute's hot loop branchy, which fights the issue-port wall. +- **Feed floor = 1; compute floor = 1.** Feeds can be driven down to their last + wave (throw everyone else at the bottleneck) but not to 0 — floor-0 would force a + dual-path (fed-vs-direct) branch into compute's K-loop, adding non-WMMA issues on + the exact waves we're trying to keep pure. Floor-0 is bundled with the future + data-path-switch extension. +- **Build sequencing:** static 3-role substrate oracle-green **first**, then layer + the controller. Do not couple "new A-feed role" and "new control law" in one step. + +## Architecture — the role economy + +A workgroup launches a **fixed N waves**. Each wave is in exactly one role at any +instant. The controller governs only the **partition** +`(n_compute, n_Afeed, n_Bfeed)` with `n_compute + n_Afeed + n_Bfeed = N`. The wave +count never changes; only the partition moves. + +State = three atomic LDS **role-count slots** (`n_compute`, `n_Afeed`, `n_Bfeed`) — +the single source of truth for the current mix. A conversion is one wave atomically +moving itself between slots, then physically re-roling (grow/shrink + switch job +loop), **only at a split-K segment boundary** (the one point a wave is safe to +change: partial flushed, accumulators released). + +Key property: **the controller moves the *target partition*; waves migrate +themselves toward it at their next safe boundary.** This is what keeps it lock-free +and barrier-free — no wave waits on another, no rendezvous. + +Invariants (always hold): +- Floors: `n_compute ≥ 1`, `n_Afeed ≥ 1`, `n_Bfeed ≥ 1`. +- Sum-envelope: `Σ instantaneous VGPR alloc < per-SIMD budget`, enforced at + grow-time (below), never by the controller's arithmetic. + +## Sensing — ring pressure + +Each ring (A-ring, B-ring; depth `RINGD`) is a supply/demand **pressure gauge**. +At its decision boundary a wave reads occupancy from LDS: + + occ_X = prod_count_X − min_cons_X (clamped to [0, RINGD]) + +Watermark bands give hysteresis (the **spatial** damping): +- `occ_X < LOW` → ring draining empty → **starved** for X. +- `occ_X > HIGH` → ring backing up full → **over-served** for X. +- `LOW ≤ occ_X ≤ HIGH` → dead-zone, no action. + +Both conversion directions read the *same* gauge from opposite sides: +- a **compute** wave (consumes from ring X) acts on `occ_X < LOW`; +- a **feed-X** wave (fills ring X) acts on its own `occ_X > HIGH`. + +## Control law — conversion, both directions + +**compute → feed-X** (compute is starved for X): +- Boundary: split-K **segment** boundary (compute holds accumulator state). +- Guard: `occ_X < LOW`, win the epoch ticket for this direction (below), + `n_compute > 1` (floor). +- Actuation: CAS-decrement `n_compute`, increment `n_feedX`, `s_alloc_vgpr` **SHRINK** + to lean (a shrink *always* succeeds), enter the feed loop. + +**feed-X → compute** (feed-X is over-serving): +- Boundary: any inter-frag point (a feed wave holds no accumulator state → far more + frequent safe boundaries), gated to the epoch. +- Guard: `occ_X > HIGH`, win the epoch ticket, `n_feedX > 1` (floor). +- Actuation: CAS-decrement `n_feedX`, increment `n_compute`, `s_alloc_vgpr` **GROW** + to fat — *gated on the sum-envelope* (below). On grow-fail: **abort cleanly** + (undo the slot move + reservation, stay feed this epoch). On success: enter the + compute loop (claim a K-segment, accumulate). + +Designed asymmetry (the good kind): the cheap-to-move role (lean feeds) reacts +**fast**; the expensive-to-move role (fat compute) reacts at **segment** granularity. +Shrink-always-succeeds / grow-can-abort means budget is *freed* promptly but +*consumed* only when proven-available. + +## Epoch + ticket — lock-free single-winner per window + +**Epoch** = a clock derived from work progress (per-workgroup): + + E = (segments_processed >> EPOCH_SHIFT) + +ticks every `2^EPOCH_SHIFT` segments. `EPOCH_SHIFT` is the **decision-cadence knob** +(small = reactive, large = damped). No one "advances" it; it's a function of +throughput. + +**Ticket** = one LDS `gate[d]` per conversion direction `d` (4 directions: +compute→Afeed, compute→Bfeed, Afeed→compute, Bfeed→compute), each holding the last +epoch in which `d` fired. To convert in direction `d` at epoch `E`: + + g = gate[d] + if g >= E: back off # d already fired this epoch + else: won = (atomic_cmpswap(gate[d], g, E) == g) + if won: proceed else: back off + +Exactly one wave's CAS succeeds when many race the same `g < E`. **≤1 conversion per +direction per `2^EPOCH_SHIFT` segments**, per workgroup — the **temporal** damping. + +This one CAS does triple duty: (1) anti-thrash, (2) the **phase-stagger** the +rolling-dyn-VGPR thesis needs — grows are automatically spread across epochs, never +simultaneous — and (3) keeps the sum-envelope satisfiable by construction (you can +never get N waves spiking VGPR at once). + +Combined damping: a feeder can't flip back to compute until **both** its ring exceeds +HIGH (spatial) **and** a fresh epoch ticket is available (temporal). Tuning surface = +`{LOW, HIGH, RINGD, EPOCH_SHIFT}` — small and interpretable. + +Gates + role slots + epoch clock are all **per-workgroup** (in each WG's LDS); every +WG self-balances independently with zero cross-WG coordination. + +## Safety & failure modes + +- **Sum-envelope, concretely:** one LDS counter `vgpr_reserved`. A feed→compute grow: + `r = atomic_add(vgpr_reserved, V_fat − V_lean)`; if `r + Δ > BUDGET` → + `atomic_sub` back and **abort**. compute→feed shrink: `atomic_sub` (always + succeeds). This reservation correctly arbitrates the ≤2 concurrent grows an epoch + permits (the atomic serializes them; the second to validate sees the first's + reservation and backs off). +- **Role floors:** CAS-guarded decrements (compute→feed needs `n_compute > 1`; + feed→compute needs `n_feedX > 1`). +- **No barrier, by construction:** pure LDS atomics + busy-wait flags (inherits + coop's `BUSYWAIT`); conversions add zero rendezvous → the dyn-VGPR / §3.3.3.2 + barrier deadlock is structurally absent. +- **POOLTERM terminal must be role-agnostic** (the one genuinely new requirement): + every role's loop checks the `ti ≥ TOTAL` terminal broadcast at the *same* boundary + it checks for conversion, so no wave is stranded in a role past drain, and a wave + that converts then sees the terminal exits immediately. +- **No livelock:** a full envelope just makes feed→compute grows keep aborting — the + wave stays feeding (productive); the mix holds at a safe, correct partition. No + state spins doing nothing. + +All failure modes degrade to "stay in current role, keep doing useful work" — never +to a brick, never to a stall. + +## Testing & verification + +- **CPU oracle = correctness ground truth, under ANY schedule.** DSWS is a pure + *performance* transform — the adaptive mix must never change the math. Gate: for + any (shape, mix, conversion schedule), stored C matches the fp8 e4m3 oracle. + Split-K flushes fp32 partials via atomic-add (nondeterministic ordering) → gate + with an fp32-reorder **tolerance** or a deterministic-accumulation gate mode + (carry coop's existing choice — resolve in planning). +- **Two-gate sequencing:** + - *Gate 1 — static 3-role green:* fixed `(n_compute, n_Afeed, n_Bfeed)`, no + controller, oracle-clean across shapes + several hand-set mixes. + - *Gate 2 — dynamic green:* same oracle, conversions firing. +- **Conversion-storm race stress:** oracle + many repeats under adversarial tuning + (tight watermarks + tiny `EPOCH_SHIFT` → maximal conversion rate) to expose + cross-wave races the strong oracle catches and `acc00` would miss. +- **Prove it adapts (not just runs):** instrument role-count slots + conversion + counters into the occ-snapshot stream. (a) asymmetric shape → mix moves the right + way and settles; (b) **start from a deliberately wrong mix** (e.g. all-compute on a + feed-bound shape) → controller converges to a better partition **and TF climbs**. +- **Success metric (v1)** on ml8 `down` (M=2048 K=9216 N=2560) / `down_pf` (M=512): + (a) oracle-correct, (b) converges to a mix that beats the static 3-role baseline + AND the 165 TF static winner, (c) demonstrably adapts when shape changes — + confirmed with `--att` showing **cut non-WMMA issues on the compute waves**. +- **Supervised GPU discipline:** display GPU = brick risk. Compositor-safe chunking + + per-chunk hang-abort; oracle (STORE=1) before perf (STORE=0); **one gated dispatch + at a time, no sweeps until proven safe**; stream to disk for brick forensics; user + greenlights each dispatch. Never `--gl2c`; SAFEPROBE + bounds gate stay on. + +## Build phases (high level; detailed plan is the next step) + +1. **Static 3-role substrate** on coop: add the A-feed role (A-ring in LDS, A-feed + waves) — *port* the proven A-LDS-share pattern from `occ_kernel_wggemm2.s` rather + than invent it. Fixed mix. **Gate 1** oracle-green. +2. **Sensing + role slots:** ring-occupancy reads, watermark bands, the LDS + role-count slots + `vgpr_reserved` reservation counter (no conversions yet — + read-only sensing + a STATIC mix still). Verify the sensors report sane occupancy. +3. **Conversion + epoch/ticket:** the CAS gate, both-direction conversion with floor + + envelope guards, role-agnostic POOLTERM terminal. **Gate 2** oracle-green + + conversion-storm stress. +4. **Adaptivity + tuning:** instrument role counts; converge-from-wrong-start proof; + sweep `{LOW, HIGH, RINGD, EPOCH_SHIFT}`; measure on ml8 `down`/`down_pf` with + `--att`. RESULT doc + KG bank. + +## Open details to resolve in planning + +- Oracle accumulation determinism vs tolerance (carry coop's choice). +- Exact source/owner of the per-WG `segments_processed` counter feeding `E`. +- A-feed ring sizing / LDS budget at the chosen tile (must fit A-ring + B-ring + + counters within 64 KB; confirm not occupancy-binding since we're not occupancy-maxxing). +- Fat/lean VGPR footprints `V_fat`, `V_lean` and the per-SIMD `BUDGET` constant + (from RGA on the static substrate). +- Feed wave's exact inter-frag check cadence (every frag vs every K-window). + +## Pointers + +- KG: `2dc2332d` (prior-art verdict), `dac0bb8c`/`0a2cea44` (POOLTERM), `5fedf098` + (rolling-peak origin), `dc9faf0d`/`63583120` (DSWS v1/v2), `8a9ce97f`/`17f209af` + (dyn-VGPR OOR-temp deadlock fix — the barrier-free precedent). +- Master: `MAD305_DSWS_MASTER.md` §5–§6. Jira epic MAD-305. Tasks #323/#324. diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_PHASEB_CONVERSION.md b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_PHASEB_CONVERSION.md new file mode 100644 index 000000000000..fafb6a271789 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_PHASEB_CONVERSION.md @@ -0,0 +1,222 @@ +# DSWS Substrate v2 — Phase B: Runtime Role Conversion (design) + +**Status:** approved design (2026-07-01). Realizes *step 2* of the build sequence in +`SPEC_DSWS_SUBSTRATE_V2.md` §6 ("Add conversion"). Phase A (static claim-based split-K +substrate) is GPU-proven green: all 3 role mixes × 2 split-K tiers oracle-CLEAN, bit-exact, +zero bricks (KG `0c5537e6`, 2026-07-01). This spec covers only the conversion actuation added +on top of that green substrate. + +**Goal (one sentence):** Let a workgroup move waves between {compute / A-feed / B-feed} at +runtime — the DSWS payload — on the claim-based v2 substrate, without orphaning output, jamming +a feed, or bricking, so the partition `(nComp, nAfeed, nBfeed)` self-tunes to the bottleneck. + +## Relationship to prior work + +- The **conversion control law** already exists and is unit-tested in `dsws_ctrl_model.cpp` + (`watermark_decision`, `epoch_of`, `gate_try_win`, `reserve_grow`) and is transcribed + exactly by the `try_gate` macro + reservation logic in `occ_kernel_coop.s`. Phase B **ports** + that proven actuation into `occ_kernel_dsws.s` and binds it to the claim-based role branch — + it does not re-derive it. +- The blocker that killed the *coop* substrate's conversion (work bound to wave identity; + KG `86e33108`) is structurally absent in v2: a converted wave inherits **no work item**, it + just changes its role tag and starts claiming from the dest role's counters. This spec relies + on that property and does not re-litigate it. + +## The two locked design decisions + +### Decision 1 — Quiesce model: per-epoch snapshot (+ `N−1` DIAG assert) + +The claimer's `.Lclaimer_wait_done` quiesce currently waits on **compile-time** +`NCOMP/NAFEED/NBFEED` (`ROWBLK_NEXT ≥ G+NCOMP`, `BFRAG_NEXT ≥ FN+NBFEED`, +`AROW_NEXT ≥ G+NAFEED`). Correct for static roles only. Once conversions move the partition, +those constants are wrong → the claimer advances early (straggler races the counter reset → +brick / stale-resident wrong-oracle) or waits forever (deadlock). + +**Fix:** the claimer snapshots the *live* role counts `(nComp, nAfeed, nBfeed)` at super-tile +broadcast into per-epoch LDS slots; the quiesce sentinels read the snapshot, not the constants: +`ROWBLK_NEXT ≥ G + snap.nComp`, `BFRAG_NEXT ≥ FN + snap.nBfeed`, `AROW_NEXT ≥ G + snap.nAfeed`. + +**Why this model** (vs. the two rejected alternatives): smallest diff from the GPU-proven +Phase-A quiesce (swap three constants for three LDS reads — tiny bisect surface); preserves the +per-operand tripwire (a jammed single role's counter is still caught *at* the quiesce, not only +downstream at the oracle); and maps 1:1 onto `dsws_ctrl_model.cpp` so the snapshot/quiesce +interaction is CPU-unit-testable offline before any GPU dispatch. +- Rejected: **role-agnostic `N−1` counter alone** — more robustly partition-independent, but + changes more proven code and blinds the per-operand tripwire. +- Rejected: **live per-role drained counters w/ mid-super-tile conversion** — reopens the exact + cross-wave ordering hazards v2 was built to avoid; reactivity payoff is moot given split-K's + already-short super-tiles. Deferred to Phase 4 only if measured need appears. + +**Safety net (the rejected alternative, kept as an assert):** the `N−1` counter is compiled in +as `QUIESCE_CNT` and, under `DIAG`, cross-checked against `Σ snapshot sentinels`. Because wave +count is fixed and `wid 0` (claimer) never converts, exactly `N−1` non-claimer waves are alive +each super-tile and each does exactly one terminal bail — so `QUIESCE_CNT == N−1` must coincide +with the three snapshot sentinels being satisfied. A disagreement is an ordering bug, caught +immediately (offline model + supervised runs). + +### Decision 2 — Actuation mechanism: bail-time commit (Approach A) + +A converting wave **decides** during the super-tile but **commits** at its terminal bail, +sequenced immediately *before* it increments `QUIESCE_CNT`. + +**Why this composes with Decision 1 for free:** the claimer's quiesce already waits for all +`N−1` bails. If the role-slot CAS (the commit) is ordered *before* the bail-count bump, then +"quiesce satisfied" *implies* "all conversions for this boundary have landed" — the quiesce +counter **is** the publish/snapshot handshake. The claimer therefore snapshots the E+1 +partition only after every commit is visible. No extra barrier, no second ordering, no +intent-buffer. (Rejected: claimer-mediated commit — splits the commit across two waves and adds +a claimer→wave-resize ordering, since `s_alloc_vgpr` must run on the converting wave anyway. +Rejected: immediate mid-tile commit + pending-counter — reintroduces mid-super-tile population +mutation = Option-3 hazard.) + +## Global constraints (inherited verbatim from `SPEC_DSWS_SUBSTRATE_V2.md`) + +- A GPU brick is a **BUG**, never a tax. A hang = full STOP + report; never auto-fire the next + variant. **The user greenlights EVERY GPU dispatch individually.** +- Display GPU → only compositor-safe chunked sub-second dispatches + (`ML8_POOL=1 ML8_COOP_CHUNK=8 ML8_COOP_CHUNK_MAXS=0.75 ML8_COOP_STREAM=1`, `timeout 30`). +- **NEVER `--gl2c`.** SAFEPROBE + bounds gate + padding stay ON. `ML8_COOP_STREAM=1` always. +- **No `s_barrier`** (mixed dyn-VGPR + `s_barrier` hard-deadlocks — proven). Pure LDS-atomic + busy-wait coordination only. +- `occ_kernel_coop.s` is **never modified** — known-good reference. All work is additive in + `occ_kernel_dsws.s`. +- Commit to git only when the user asks. Single-variable isolation; fix bugs, don't dodge them. + +## Section 1 — New LDS state + +**Symbols (inherited from `SPEC_DSWS_SUBSTRATE_V2.md` / the coop layer):** `N` = total waves/WG +(fixed at launch); `G=6` cooperative M-extent (= `nComp` ceiling); `FN=4`/`FM=2` N/M frag +counts; `NFV` = fat-compute VGPR target (grow size); `VLEAN=32` = lean feed VGPR; `BUDGET` = +per-SIMD VGPR budget the sum-envelope must not exceed; `RINGD` = ring depth; `SEGK=64`, +`n_kseg = KT/(SEGK/16)`. + +Ported from the coop actuation layer (same relative layout), plus two Phase-B additions: +- `GATE_OFF[4]` — `gate[dir]` = last epoch direction `dir` fired (the `try_gate` CAS target). +- `VRESV_OFF` — `vgpr_reserved` sum-envelope counter. Init = `nComp·NFV + (nAfeed+nBfeed)·VLEAN`. +- `SEGCNT_OFF` — per-WG decision clock source (`epoch = segcnt >> EPOCH_SHIFT`). +- Thresholds `CTRL_LOW`, `CTRL_HIGH = RINGD−1`, `EPOCH_SHIFT` (mirror `occ_dispatch` env). +- **NEW — `SNAP_NC/NA/NB`**, double-buffered by epoch parity (`[E&1]`) so the snapshot being + read for `quiesce(E)` is never clobbered by the claimer staging `E+1`. +- **NEW — `QUIESCE_CNT`** — role-agnostic `N−1` bail counter (Decision 1 safety net + advance + gate). + +The role-count slots `NCOMP_SLOT/NAFEED_SLOT/NBFEED_SLOT` and `VRESV_OFF` already exist in +`occ_kernel_dsws.s` (published by the claimer today); Phase B makes them **mutable** (CAS'd on +conversion) instead of write-once. LDS budget delta is a handful of u32 slots — re-assert the +32 KB group-segment fit (Phase A used 16640 B of 16896 B alloc; ample margin). + +## Section 2 — Sensing (faithful port) + +`occ_X` = ring producer minus consumer at the **consume point**: +`occ_A = prod_a − min(cons_a)`, `occ_B` analogously from the B-frag claim counters. Sampled +where the value is consumed, **not** at the segment boundary — the coop lesson (KG `0a3135b2`): +at the boundary the ring is drained so `occ ≈ 0` and the controller would read "always +starved." Fed to `watermark_decision(occ, CTRL_LOW, CTRL_HIGH)`: +`occ_X < LOW` → compute starved for X → shrink compute → feed-X; `occ_X > HIGH` → feed-X +over-serving → grow feed-X → compute. Read-only; no actuation here. + +## Section 3 — Conversion lifecycle (Approach A, per non-claimer wave) + +At each **kseg boundary** (= super-tile boundary; roles are frozen *within* a super-tile): + +1. **Sense** (§2) → `watermark_decision` → candidate direction `dir` (or none). +2. **Win ticket** — `try_gate(dir)`: `E = segcnt >> EPOCH_SHIFT`; win iff `gate[dir] < E` via + the single-winner LDS CAS (`ds_cmpstore_rtn_b32`, operand order per KG `9ed04f3c`: + `vsrc0=new=E`, `vsrc1=cmp=g`). ≤1 winner per `(dir, epoch)`. Non-winners continue unchanged. +3. **Work to terminal bail** in the current role (unchanged claim loop). +4. **Commit — ordered strictly before the `QUIESCE_CNT` bump:** + a. **Floor guard** — CAS-dec the source role slot only if `> 1` (compute floor ≥ 1, feed + floor ≥ 1). Fail → abort conversion, bail as current role. + b. **Reservation envelope** — `compute→feed` shrink: `atomic_sub(vgpr_reserved, NFV−VLEAN)` + (always succeeds). `feed→compute` grow: `atomic_add(vgpr_reserved, NFV−VLEAN)` then + validate `≤ BUDGET`; if over, `atomic_sub` + abort (stay in role this epoch). + c. **CAS role slots** — dec source, inc dest (bounded to ≤2 concurrent writers/boundary by + the ticket; plain atomic-LDS). + d. **Flip private role register** + `s_alloc_vgpr` GROW(`NFV`) / SHRINK(32), each SCC-retry + guarded. +5. **Bump `QUIESCE_CNT`** (every non-claimer wave does this exactly once/super-tile, converted + or not), then enter the dest role's claim loop (or re-enter own). + +**Claimer (pinned `wid 0`, never converts):** on `QUIESCE_CNT == N−1` → all commits landed → +reset per-super-tile counters, **snapshot** live `(nComp,nAfeed,nBfeed)` into the `[E+1 & 1]` +slots, clear `QUIESCE_CNT`, bump epoch (proven `TI_OFF`-before-`EPOCH_OFF` ordering). Terminal: +`sti ≥ TOTAL_super` remains the role-agnostic retire signal every role checks at its boundary. + +## Section 4 — Safety / anti-brick invariants + +- **Ordering contract (the crux):** commit (4c CAS) precedes the `QUIESCE_CNT` bump (step 5), + which precedes the claimer's E+1 snapshot. Therefore the claimer never snapshots a stale + partition. Self-enforced by the bail sequence; no barrier. +- **Floors** `nComp, nAfeed, nBfeed ≥ 1`; **ceiling** `nComp ≤ G` (asserted in the control + model — surplus compute waves would find `rowblk_next` exhausted and idle, but floors+ceiling + keep `nComp ≤ G` by construction). +- **`s_alloc_vgpr` OOR-poison guard (highest brick-risk item):** on RDNA4, any LDS/atomic temp + register reachable *before* a grow must be v14/v15 under dyn — a `>v15` source pre-grow is + poison (coop learned this; `occ_kernel_coop.s` gates every pre-grow-reachable temp to + v14/v15). Every new pre-grow-reachable temp in the conversion path (sense, ticket, floor, + envelope) inherits this constraint. Verified by RGA + the sense/ticket temps living in the + lean-safe register window. **This is the single most likely place to brick — reviewed + explicitly by the round table before the first dynamic-mix GPU dispatch.** +- **No `s_barrier`.** All coordination is LDS-atomic busy-wait. +- **Clock never converts:** `SEGCNT` is bumped by the pinned claimer (`wid 0`), which is + non-convertible — kills the clock-stall failure mode. + +## Section 5 — Control model changes (offline, TDD, no GPU) + +Extend `dsws_ctrl_model.cpp` + `test_dsws_ctrl_model.cpp`: +- Model the **snapshot/quiesce interaction**: a `snapshot(E)` reads role counts; conversions + mutate counts at the boundary; assert `quiesce(E)` uses `snap(E)` and that + `Σ snap(E) sentinels ⟺ QUIESCE_CNT == N−1` under arbitrary interleavings (thread-race test, + as the existing `gate_try_win` test does). +- Keep `watermark_decision / epoch_of / gate_try_win / reserve_grow` tests green (unchanged + semantics). All `ALL PASS` before any assemble. + +## Section 6 — Build sequence & gates (isolation-preserving) + +Each stage a supervised GPU gate; you greenlight each; brick = full STOP + bisect. Offline +before each: `dsws_ctrl_model` tests green, RGA 0-spill, dry-print sane. + +Config held at Phase-A values **`G=6, SEGK=64`** for stage 1 so the conversion code is the only +variable vs the green static gate. + +1. **Static-mix through the conversion path** — conversion code wired but watermarks set so + **none fire** (`CTRL_LOW=0` / unreachable). Must reproduce the Phase-A green (all 3 mixes × + both tiers, `ok=… bad=0`). Proves the ported actuation + snapshot machinery is inert-safe / + non-regressing. **[SUPERVISED GPU — the re-baseline gate.]** +2. **Dynamic-mix** — watermarks that *do* fire conversions; oracle stays green as roles move. + Start `n_kseg=1` TIGHT (exact) at one mix, then LOOSE, then the other mixes. **[SUPERVISED.]** +3. **Storm** — tight watermarks + `EPOCH_SHIFT = 0` + ×10 repeats: the lock-free race-hunt + (the strong-oracle-plus-repeats discipline that caught 136/552). **[SUPERVISED.]** + +Phase 4 (separate spec): adaptivity proof (converge-from-wrong-start), +`{LOW,HIGH,RINGD,EPOCH_SHIFT,G,SEGK}` sweep, `--att` issue-mix on ml8 `down`/`down_pf`. + +## Testing + +- **CPU oracle gate** (`fp8_oracle.cpp`): Tier-1 tight (`5e-3` rel / `1e-2` abs) at `n_kseg=1`; + Tier-2 loose (`3e-2` / `2e-2`) at `n_kseg>1`. Before every perf run and every kernel change. +- **RGA static gate:** 0-spill, live-VGPR within budget, every assemble. +- **Control-law unit tests:** extended per §5, `ALL PASS`. +- **Storm stress:** §6 stage 3. +- **DIAG cross-check:** `QUIESCE_CNT == N−1` ⟺ `Σ snapshot sentinels` (Decision-1 safety net). + +## Success metric + +Oracle-green through the **storm** at dynamic mix on both tiers, all mixes, **zero bricks** — +the partition provably moves at runtime with correct output. The DSWS *thesis* payoff (beats +static baseline + `--att` shows cut non-WMMA issues on compute waves, on ml8 `down`/`down_pf`) +is Phase 4, not a Phase-B gate. + +## Risks & open items + +- **`s_alloc_vgpr` OOR-poison in the conversion path** — highest brick risk (§4). Round-table + the register assignment before the first dynamic-mix dispatch. +- **Concurrent role-slot CAS** — bounded to ≤2 writers/boundary by the ticket; verify the CAS + is genuinely atomic-LDS and the floor-guard dec/inc pair can't transiently violate a floor. +- **Snapshot double-buffer parity** — confirm `[E&1]` indexing can't alias when the pool is + ≤2 super-tiles deep (degenerate tiny-shape oracle case). +- **`vgpr_reserved` under grow-abort** — the atomic_add-then-sub-on-over-budget must not leave a + transient over-count visible to a *second* concurrent grower; both are ticket-serialized per + epoch, but verify across directions. +- **Round-table structure** (kmbandy's): implement (Sonnet) → adversarial review (Fable + Codex) + → kmbandy greenlights each GPU dispatch. It caught 5 offline bricks in Phase A; keep it. diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_SUBSTRATE_V2.md b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_SUBSTRATE_V2.md new file mode 100644 index 000000000000..55bab7acef7a --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/SPEC_DSWS_SUBSTRATE_V2.md @@ -0,0 +1,222 @@ +# DSWS Substrate v2 — Claim-Based Work Decomposition + Split-K (design) + +**Status:** approved design (2026-06-30). Supersedes the Phase-3 *actuation* plan in +`PLAN_DSWS_CONTROLLER.md` (Tasks 3.2–3.4), which assumed role conversion could be bolted +onto the static-partition coop substrate. It cannot — see "The blocker" below. Phases 1–2 +(static 3-role substrate, sensing, role-count slots, gate-CAS ticket, reservation model) +remain valid and are reused. + +**Goal (one sentence):** Re-found the DSWS substrate so matrix work is *claimed by whatever +wave currently holds a role* rather than *owned by a wave's compile-time identity*, and fold +in split-K — so the controller can move waves between {compute / A-feed / B-feed} at runtime +without orphaning output, jamming a feed, or bricking. + +**Architecture (2-3 sentences):** A workgroup launches a fixed N waves. Work is a pool of +`(mblk, tcol, kseg)` super-tiles; a pinned claimer broadcasts the current super-tile, and the +live waves of each role drain shared atomic counters (compute claims rowblks, feeds claim +which operand fragment to stage) against resident-in-LDS A/B for that super-tile. Because work +is claimed, not owned, `nComp/nAfeed/nBfeed` can change at any per-kseg boundary with no +work-item handoff. + +**Tech stack:** Hand-written gfx1201 (RDNA4, wave32) assembly (`occ_kernel_dsws.s`, NEW file), +raw-PM4 dispatch (`occ_dispatch.cpp`), CPU fp8 e4m3 oracle (`fp8_oracle.cpp`), control law +(`dsws_ctrl_model.cpp`, unchanged). dyn-VGPR via `s_alloc_vgpr` (armed by PM4 RSRC2 bit 6). + +## Global constraints (verbatim, every task inherits these) + +- A GPU brick is a **BUG**, never an accepted tax. A hang = full STOP + report, never + auto-fire the next variant. +- **The user greenlights EVERY GPU dispatch individually.** Display GPU (R9700 drives the + monitors) → only sub-second, compositor-safe-chunked dispatches + (`ML8_POOL=1 ML8_COOP_CHUNK=8 ML8_COOP_CHUNK_MAXS=0.75 ML8_COOP_STREAM=1`, timeout 30). +- **NEVER pass `--gl2c`** (MES-crash landmine). SAFEPROBE + bounds gate + padding stay ON. +- Every run streams to disk (`ML8_COOP_STREAM=1`). +- Commit only when the user asks. +- Single-variable isolation; fix bugs, don't dodge them; never declare a wall from an + unmeasured assumption; don't blame the GPU/model prematurely. +- Barrier-free / lock-free: pure LDS atomics + busy-wait flags. **No `s_barrier`** (mixed + dyn-VGPR allocations + `s_barrier` hard-deadlock the GPU — proven). +- The proven `occ_kernel_coop.s` (1716B `DSWS=0` coop binary) is **never modified** — it is + the known-good reference. v2 is an additive new file. + +## The blocker (why v2 exists) + +3/3 consensus (kmbandy + Claude + Codex, session 019f19da, 2026-06-30; KG `86e33108`): the +proven coop substrate binds the matrix **work decomposition** to compile-time role counts and +physical wave identity (`wid`). Naive role conversion therefore **bricks** (the hang fires +before any wrong-output is even observable): + +1. Compute M-rows: `rowblk = trow*P + cid`, `P = NCOMP` compile-time, `cid` fixed from `wid`. + A departed `cid`'s rows have **no writer** → orphaned output. +2. A-feed: `NCOMP` bands paired 1:1 to compute `cid` via `prod_a[cid]/cons_a[cid]`. A departed + consumer → `prod_a` blocks at `RINGD_A` → **WG hangs**. B-feed's `min_cons` also scans the + `P` compute counters, so a departed `cid` wedges B production too. +3. B-feed frags: `owner = ni % NBFEED`; a converted-in wave with `b_id ≥ NBFEED` computes a + `prod_b` address that **overflows into A-ring/LDS storage** → memory corruption. + +Root cause: work bound to wave **identity** instead of being **claimable by whoever holds the +role**. v2 removes that binding. + +## Architecture — the role economy (unchanged from SPEC_DSWS_CONTROLLER.md) + +Fixed N waves/WG; each wave is in exactly one role at any instant. The controller governs only +the partition `(nComp, nAfeed, nBfeed)` with `nComp + nAfeed + nBfeed = N`. Wave count never +changes; only the partition moves. State = three atomic LDS role-count slots (already built, +T2.1). Floors: each role `≥ 1`. Sum-envelope: `Σ instantaneous VGPR alloc < per-SIMD budget`, +enforced at grow-time via the `vgpr_reserved` counter (already built, T2.2). + +## Section 1 — Work pool & claim model + +**Super-tile** = `(mblk, tcol, ksi)`. K-loop terms used throughout: `SEGK` = segment size +(K-elements per split-K segment, a compile-time knob); `n_kseg = KT / SEGK` = number of +segments; `ksi ∈ [0, n_kseg)` = segment index. +- `mblk` indexes a group of `G` consecutive rowblks — `G` = compile-time cooperative M-extent + (set to the launch's max compute count, `NCOMP_MAX`). Bounds resident A. +- `tcol` = the shared N-column tile (`FN` frags). `ksi` = the split-K segment index. +- Pool size = `(M / (G·16·FM)) × NTL × n_kseg` super-tiles. + +**Two claim levels** (reuse the proven "claimer claims + broadcasts, followers wait on epoch" +machinery): +- **Super-tile claim:** a pinned claimer wave (Section 3) does the existing `global_atomic_add` + to grab the next super-tile index `sti`, decodes `(mblk, tcol, kseg)`, publishes it, then + bumps epoch (the proven `TI_OFF`-before-`EPOCH_OFF` ordering). All WG waves work the same + current super-tile. +- **Rowblk claim (within a super-tile):** a per-super-tile LDS counter `rowblk_next`. Live + compute waves `ds`-atomic-add to grab the next `rowblk ∈ [0, G)`. Exhausted (`≥ G`) → this + super-tile's compute is done. + +**Coverage proof:** the super-tile pool covers every `(mblk, tcol, kseg)` once; `rowblk_next` +covers every rowblk in the group once. So every `(rowblk, tcol, kseg)` is computed exactly +once and `C[rowblk,tcol] = Σ_kseg partial`. Holds for **any** live `nComp ≤ G` — fewer compute +waves just drain `rowblk_next` slower. (If the controller ever set `nComp > G`, the extra waves +find the counter exhausted and idle/convert; floors+ceilings keep `nComp ≤ G`.) + +## Section 2 — Resident A/B lifetime & completion handshake + +Split-K keeps each segment's operands small enough to stage **resident** in LDS: +- **Resident B** for `(tcol, ksi)` = `FN·16` cols × `SEGK` K. Loaded by B-feed waves (each + claims which frag to stage from a frag counter — identity-free). +- **Resident A** for `(mblk, ksi)` = `G` rowblks × `16·FM` rows × `SEGK` K. Loaded by A-feed + waves (each claims which rowblk's A to stage from an A-load counter — identity-free). Compute + reads its claimed rowblk's A from this resident region (replayable — any compute wave + re-reads freely; this is what decouples rowblk-count from `nComp`). + +**Completion handshake** (barrier-free; the safety core): +- **`rowblk_done`** (per-super-tile LDS counter). A compute wave, after flushing its rowblk's + partial, `ds`-atomic-increments `rowblk_done`. +- **Claimer gate:** the claimer may claim/broadcast the *next* super-tile only once + `rowblk_done == G` for the current one (all rowblks computed *and* flushed). This frees the + resident A/B safely — a counter compare, not `s_barrier`. +- A compute wave that finishes its claimed rowblks before the super-tile is globally complete + spins on the completion gate (or attempts a conversion) rather than racing ahead — the + busy-wait discipline the kernel already uses. + +**Terminal:** super-tile claim returning `sti ≥ TOTAL_super` is the role-agnostic retire +signal; every role checks it at its decision boundary (replaces the per-tile POOLTERM, now +sub-tile aware). A just-converted wave re-checks immediately. + +## Section 3 — Role tags & the pinned claimer (simplification over a published role map) + +The claim-counter model makes a *published per-wave role/rank map* unnecessary — every role +claims work from shared counters, so no role needs a stable logical rank; count affects only +speed, never coverage. What remains: +- **Role-count slots** (already built, T2.1) — CAS'd on conversion; drive sensing + floor + guards. +- **Per-wave private current-role register** — the wave branches to its role loop; on + conversion it rewrites this register and jumps. Cross-wave visibility is carried by the + atomic work-counters themselves, not a broadcast map. +- **Pinned claimer + clock = physical `wid 0`** (never converts). Permanently owns super-tile + claiming, the `ti`/epoch broadcast, and the `SEGCNT` controller clock. This kills the + clock-stall failure mode — the clock owner can never convert away. + +This is a deliberate, documented deviation from the consensus "generationed role map" step: the +claim model dissolved the need, removing a class of cross-wave ordering hazards. + +## Section 4 — Partial-C reduction & tiered oracle + +Split-K's headroom requires a wave to do **one kseg then release** (brief VGPR peak), so +different waves compute different ksegs of the same `(rowblk,tcol)` → their partials combine +**across waves** → the low bits of `C` are no longer bit-deterministic (fp combine order). The +combine uses `global_atomic_add` of fp32 partials into `C`. + +**Tiered oracle** — tight tolerance where the risk is, looser only where split-K fp +reassociation forces it. (NB: the CPU `wmma_ref` chain is not bit-identical to the GPU fp8→fp32 +WMMA even today, so the established gate is already a *tight tolerance*, `5e-3` rel + `1e-2` abs, +not bit-exact — that is the discipline that caught 136/552.) +- **Tier 1 — tight.** Run the correctness gate at `n_kseg = 1`: one partial per `(rowblk,tcol)`, + no cross-wave combine. Reuse the **existing tight tolerance** (`5e-3` rel / `1e-2` abs). This + exercises all the dangerous new logic (claim, coverage, resident A/B, handshake, conversion — + identical code regardless of kseg count). +- **Tier 2 — loose.** Run `n_kseg > 1` against the reference with a looser tolerance + (`~3e-2` rel / `2e-2` abs). A structural combine bug (missed/double kseg) is a *large* error → + caught; only the benign split-K reassociation slips under the looser bound. + +Build-time check: confirm `global_atomic_add_f32` is encodable on gfx1201; if not, the combine +uses a CAS loop or a scratch-slot + final-reduction path (the scratch path also recovers exact +determinism at `n_kseg > 1` if ever needed). + +## Section 5 — Conversion actuation + +Reuses the already-built gate-CAS ticket (T3.1), reservation model (T2.2), and sensors (T2.3). +At a per-kseg boundary (frequent, sub-tile) a wave eligible to convert: +1. **Sense** ring/counter occupancy → watermark decision (`watermark_decision`). +2. **Win the epoch ticket** `try_gate(dir)` — single winner per `(dir, epoch)`. +3. **Floor guard** — CAS-dec the source role-count only if `> 1`. +4. **Reservation envelope** — `compute→feed` shrink: `atomic_sub vgpr_reserved` (always + succeeds). `feed→compute` grow: `atomic_add` then validate `≤ BUDGET`, else `atomic_sub` + + abort (stay in current role this epoch). +5. **Actuate** — CAS role slots (dec source, inc dest), flip own private role register, + `s_alloc_vgpr` GROW(`NFV`)/SHRINK(32) (each guarded by SCC-retry), jump to the dest role's + loop. + +**Payoff:** a converted wave inherits **no work item** — it changes its tag and starts claiming +from the dest role's counters like any other wave of that role. No orphaned rows, no rank +handoff, no jammed feed. Every blocker failure mode is structurally absent. Barrier-free +throughout. + +## Section 6 — File structure, build sequencing & gates + +**New file `occ_kernel_dsws.s`.** Diverges enough (resident A/B, split-K partial-combine, +claim-counters) that it is a clean new kernel; `occ_kernel_coop.s` stays pristine as the +known-good reference. Trade-off: no literal "byte-identical to 1716B" guard inside the new +file, bought with the proven kernel never being touched (smallest blast radius). + +**Build sequence (isolation within the folded scope — a brick/oracle break stays bisectable):** +1. **Static split-K + claim-counter substrate, fixed roles, no conversion.** Oracle-green at + `n_kseg = 1` (exact) *and* `n_kseg > 1` (tolerance). Proves claim-coverage + resident A/B + + partial-combine + completion handshake with static roles. RGA 0-spill. **[SUPERVISED GPU — + the big gate.]** +2. **Add conversion** (Section 5). Oracle-green static-mix, then dynamic-mix (conversions + firing), then the storm (tight watermarks, `EPOCH_SHIFT = 0`, ×10 repeats). **[SUPERVISED + GPU.]** +3. **Adaptivity proof + tuning** (Phase 4 carries over: converge-from-wrong-start, `{LOW, HIGH, + RINGD, EPOCH_SHIFT, G, SEGK}` sweep, `--att` issue-mix). **[SUPERVISED.]** + +Every GPU dispatch: compositor-safe chunked, one at a time, user greenlights each, brick = full +STOP + bisect. + +## Testing + +- **CPU oracle gate** (`fp8_oracle.cpp`): Tier-1 exact bit-match at `n_kseg = 1`; Tier-2 tight + tolerance at `n_kseg > 1`. Gate before every perf run and on every kernel change. +- **RGA static gate:** 0-spill, live-VGPR within budget, every assemble. +- **Control-law unit tests** (`test_dsws_ctrl_model.cpp`): unchanged, still `ALL PASS`. +- **Storm stress:** tight watermarks + `EPOCH_SHIFT = 0` + ×10 repeats — proves the lock-free + protocol has no conversion race (the strong-oracle-plus-repeats discipline that caught + 136/552 before). + +## Risks & open items + +- **LDS budget for resident A+B.** Resident A (`G·16·FM` rows × `SEGK`) + resident B (`FN·16` + cols × `SEGK`) + role/claim state + ring-free counters must fit the 32 KB group segment. + `G`, `SEGK`, `FM`, `FN` are the knobs; pick a first config that fits with margin and verify + in step 1. +- **`global_atomic_add_f32` on gfx1201** — verify encodable; fallback CAS loop / scratch path. +- **Partial-combine traffic** — split-K adds `n_kseg` atomic-adds per `(rowblk,tcol)`. The bet + (per the DSWS thesis) is that the issue-port offload + fungibility gain exceeds this overhead; + measured, not assumed, in Phase 4. +- **`G` vs `nComp` overshoot** — floors/ceilings must keep `nComp ≤ G`; assert in the + controller. +- **Claimer single point** — `wid 0` pinned as claimer means it is always a B-feed-class wave; + confirm the role economy still balances with one permanently-non-compute wave (it is one wave + of N; negligible, but noted). diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_dsws.sh b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_dsws.sh new file mode 100755 index 000000000000..57d9ad57815c --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_dsws.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# build_dsws.sh — MAD-305 DSWS static 3-role substrate bins (occ_kernel_coop.s, DSWS=1). +# FM=2 FN=4 POOLTERM=1 fixed (the v1 coop tile; role counts are the swept defsyms). +# Bin name MUST match occ_dispatch.cpp --dsws: occ_dsws_cab_r[_dyn]_gd.bin +# OFFLINE/CPU only — assemble + RGA, no GPU. Usage: ./build_dsws.sh [static] [rga] +set -e +cd "$(dirname "$0")" +L=/opt/rocm/llvm/bin +fail=0 +mk() { # $1=NCOMP $2=NAFEED $3=NBFEED $4=RINGD $5=DYN + local dtag=""; [ "$5" = "1" ] && dtag="_dyn" + local tag="occ_dsws_${1}c${2}a${3}b_r${4}${dtag}_gd" + nice -19 ionice -c3 "$L/clang" -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx1201 \ + -Wa,-defsym,DSWS=1 -Wa,-defsym,FM=2 -Wa,-defsym,FN=4 -Wa,-defsym,POOLTERM=1 -Wa,-defsym,SAFEPROBE=1 -Wa,-defsym,DIAG=${DIAG:-1} \ + -Wa,-defsym,NCOMP=$1 -Wa,-defsym,NAFEED=$2 -Wa,-defsym,NBFEED=$3 \ + -Wa,-defsym,RINGD=$4 -Wa,-defsym,DYNVGPR=$5 \ + -c occ_kernel_coop.s -o "$tag.o" 2>/tmp/dsws_build.err \ + && { "$L/llvm-objcopy" -O binary --only-section=.text "$tag.o" "$tag.bin"; echo " OK $tag.bin ($(wc -c < "$tag.bin")B)"; } \ + || { echo " FAIL $tag"; sed -n '1,15p' /tmp/dsws_build.err; fail=1; } +} + +mk2() { # $1=NCOMP $2=NAFEED $3=NBFEED (DSWS2 v2 substrate, occ_kernel_dsws.s; G=6 SEGK=64 FM=2 FN=4) + local tag="occ_dsws2_${1}c${2}a${3}b_gd" + nice -19 ionice -c3 "$L/clang" -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx1201 \ + -Wa,-defsym,DSWS2=1 -Wa,-defsym,FM=2 -Wa,-defsym,FN=4 -Wa,-defsym,G=6 -Wa,-defsym,SEGK=64 \ + -Wa,-defsym,SAFEPROBE=1 -Wa,-defsym,DIAG=${DIAG:-1} \ + -Wa,-defsym,NCOMP=$1 -Wa,-defsym,NAFEED=$2 -Wa,-defsym,NBFEED=$3 \ + -c occ_kernel_dsws.s -o "$tag.o" 2>/tmp/dsws2_build.err \ + && { "$L/llvm-objcopy" -O binary --only-section=.text "$tag.o" "$tag.bin"; echo " OK $tag.bin ($(wc -c < "$tag.bin")B)"; } \ + || { echo " FAIL $tag"; sed -n '1,15p' /tmp/dsws2_build.err; fail=1; } +} + +echo "== DSWS static 3-role bins (FM=2 FN=4 POOLTERM=1) ==" +# NCOMP NAFEED NBFEED RINGD DYN +mk 4 2 2 2 1 ; mk 4 2 2 2 0 # 4c2a2b +mk 6 1 1 2 1 ; mk 6 1 1 2 0 # 6c1a1b +mk 2 3 3 2 1 ; mk 2 3 3 2 0 # 2c3a3b +echo "dsws build done. fail=$fail" + +echo "== DSWS2 v2 substrate scaffold bin (occ_kernel_dsws.s; G=6 SEGK=64) ==" +mk2 4 2 2 # 4c2a2b +echo "dsws2 build done. fail=$fail" + +# RGA gate (offline static analysis; 0-spill is the bar). Runs by default unless 'norga' passed. +if [ "${1:-}" != "norga" ] && [ "${2:-}" != "norga" ]; then + echo "== RGA gate (4c2a2b dyn — compute peak-live is tile-fixed, representative) ==" + KSRC=occ_kernel_coop.s ./rga_check.sh dsws_build_4c2a2b \ + DSWS=1 FM=2 FN=4 NCOMP=4 NAFEED=2 NBFEED=2 RINGD=2 POOLTERM=1 SAFEPROBE=1 DYNVGPR=1 2>&1 \ + | grep -E "gfx1201,|livereg" || true +fi +exit $fail diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/dsws_ctrl_model.cpp b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/dsws_ctrl_model.cpp new file mode 100644 index 000000000000..4766045ec531 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/dsws_ctrl_model.cpp @@ -0,0 +1,54 @@ +// DSWS control-law reference model (SPEC_DSWS_CONTROLLER.md). Pure host functions that define the +// EXACT semantics the Phase-3 gfx1201 asm transcribes. Kept header-light (included directly by the +// test and, later, mirrored by hand-asm) so there is one source of truth for the protocol. +// +// watermark_decision -> the boundary band check (sensing -> action sign) +// epoch_of -> the per-WG decision clock E = segments_processed >> EPOCH_SHIFT +// gate_try_win -> the lock-free single-winner ticket (asm: ds_cmpst_b32 on gate[dir]) +// reserve_grow -> the sum-envelope reservation (asm: atomic_add/sub on vgpr_reserved) +#pragma once +#include +#include + +// Sensing -> action sign for ring X. Bands are STRICT (edges are dead-zone) so LOW==HIGH degenerates +// to "act only when strictly past the edge", and LOW +1 (ring draining empty: the consumer is STARVED for X -> wants more feed-X) +// occ > high -> -1 (ring backing up full: feed-X is OVER-SERVING -> a feed-X wave can leave) +// else -> 0 (dead-zone: hold) +static inline int watermark_decision(uint32_t occ, uint32_t low, uint32_t high) { + if (occ < low) return +1; + if (occ > high) return -1; + return 0; +} + +// Per-workgroup decision clock. Ticks every 2^shift segments of progress; no one "advances" it, it is +// purely a function of throughput. shift is the cadence knob (small=reactive, large=damped). +static inline uint32_t epoch_of(uint32_t segments_processed, uint32_t shift) { + return segments_processed >> shift; +} + +// Lock-free single-winner ticket for one conversion direction at epoch E. gate holds the last epoch in +// which this direction fired. Among many waves racing the same (g < E), exactly one CAS succeeds; the +// rest observe g advanced and back off. Guarantees <=1 conversion per direction per epoch, per WG. +// asm: g = ds_read gate[dir]; if g>=E -> lose; else old = ds_cmpst_b32(gate[dir], g, E); win iff old==g +static inline bool gate_try_win(std::atomic& gate, uint32_t E) { + uint32_t g = gate.load(std::memory_order_relaxed); + if (g >= E) return false; // direction already fired this (or a later) epoch + // compare_exchange_strong updates `g` to the observed value on failure -> mirrors ds_cmpst's + // returned-old semantics; a single retry loop is unnecessary because a failed CAS means someone + // else won this epoch (g becomes >=E), so we simply lose. + return gate.compare_exchange_strong(g, E, std::memory_order_acq_rel, std::memory_order_relaxed); +} + +// Sum-envelope reservation for a feed->compute GROW. Reserve first (atomic_add), then validate against +// budget; if the reservation would blow the envelope, undo it (atomic_sub) and reject. The atomic +// serializes concurrent grows: the second to validate sees the first's reservation and backs off. +// asm: r = atomic_add(vgpr_reserved, delta); if (r+delta) > budget -> atomic_sub(delta), abort +static inline bool reserve_grow(std::atomic& resv, uint32_t delta, uint32_t budget) { + uint32_t prev = resv.fetch_add(delta, std::memory_order_acq_rel); + if (prev + delta > budget) { + resv.fetch_sub(delta, std::memory_order_acq_rel); // over-budget: cleanly undo, stay current role + return false; + } + return true; +} diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s new file mode 100644 index 000000000000..4bf185b06788 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s @@ -0,0 +1,726 @@ +// occ_kernel_dsws.s (gfx1201, wave32) -- MAD-305 DSWS v2 SUBSTRATE SCAFFOLD (PLAN_DSWS_SUBSTRATE_V2.md, Task A1). +// +// v2 re-founds the DSWS GEMM substrate so matrix work is *claimed* (not owned by compile-time wave id), with +// split-K folded in. Work = a pool of (mblk, tcol, ksi) super-tiles; a pinned claimer (wid 0) broadcasts the +// current super-tile; live waves of each role drain shared LDS atomic counters against resident-in-LDS A/B. +// +// Task A1 is SCAFFOLD-ONLY: lift the proven prologue/arming setup from occ_kernel_coop.s (NEVER modified), +// assert the v2 LDS layout fits the 32 KB group segment, and leave each role as a distinct s_endpgm stub. +// The real claimer / feed / compute loops land in A3..A7. The only gates here: (1) assembles clean, +// (2) RGA 0-spill, (3) harness --dsws2 dry-prints the computed params. +// +// ============================================================================================ +// v2 KERNARG CONTRACT (USER_SGPR=15, s0..s14; hardware-preloaded user SGPRs). The host A8 launch +// MUST set COMPUTE_PGM_RSRC2.USER_SGPR=15 and load COMPUTE_USER_DATA_0..14 to match EXACTLY: +// s0:s1 = occ buffer base (>=0x1000B; host zero-inits; see CLAIM-COUNTER / completion offsets below) +// s2:s3 = A base (fp8 e4m3, row-major, 1 byte/elem) +// s4:s5 = Bshuf base (the shuffled-B layout the coop B-feed consumes; same global_load_tr_b64 idiom) +// s6:s7 = C base (fp32; HOST MUST MEMSET C=0 before dispatch -- compute uses global_atomic_add_f32) +// s8 = KT (total K16-steps for the whole matrix = K/16) +// s9 = K(bytes/A-row) (= K, fp8 1 byte/elem) +// s10 = NT*256 (B-saddr advance per K16-step) +// s11 = TOTAL (coop-style total *tiles* = MTL*NTL; carried for addressing compat, NOT the pool size) +// s12 = magic(ceil(2^32/NTL)) (unsigned-div magic for /NTL ; tcol/mblk decode) +// s13 = NTL (number of N tile-columns) +// s14 = FN*256 (B-saddr stride per N-frag) +// (TGID_X now lands in s15 -- UNUSED; this kernel is pool-claim, not workgroup-id based.) +// NOTE: G and SEGK are COMPILE-TIME defsyms (baked into instruction immediates); they are NOT kernargs. +// FIX 1 (round-table Opus+Codex pass): v1 of this contract passed n_kseg/TOTAL_super/magic_kseg as +// s15/s16/s17, but the PM4 host only preloads COMPUTE_USER_DATA_0..15 (USER_SGPR<=16; every proven +// launch path in this tree uses 15) -- s16/s17 could NEVER actually arrive in hardware SGPRs, AND +// s16 was independently being reused per-chunk on the host as the compositor-safe chunk terminal +// (a second, unrelated collision on the same slot). This file now drops s15/s16/s17 entirely: +// n_kseg is DERIVED in-kernel from KT (s8) and the compile-time KSEG_STEPS=SEGK/16: +// n_kseg = KT >> NKSEG_SHIFT, where NKSEG_SHIFT=log2(KSEG_STEPS) is a compile-time `.set` +// (small .if ladder over KSEG_STEPS in {1,2,4,8,16}; SEGK is always a power-of-two +// multiple of 16, so KSEG_STEPS is always a power of two in that set). +// shift/mask (the sti -> (t,ksi) split) are derived ONCE in the prologue from n_kseg: +// shift = s_ff1_i32_b32(n_kseg) (bit index of n_kseg's single set bit; n_kseg=1 -> 0) +// mask = n_kseg - 1 +// DECODE_STI then does ksi = sti & mask ; t = sti >> shift -- this handles n_kseg=1 for +// free (shift=0, mask=0 -> ksi=0, t=sti), so the old magic-div n_kseg==1 special-case is +// GONE (it's no longer needed, not just hidden). +// the chunk terminal (old TOTAL_super/"chunkHi") is now MEMORY-CARRIED instead of a kernarg: the +// host writes the current chunk's terminal sti bound to occ[24] (occW[6]) once per chunk; +// the claimer reads occ[24] ONCE per dispatch (stable for the whole chunk) instead of +// receiving it as a broadcast kernarg. On sti >= occ[24] the claimer publishes a SENTINEL +// (0xFFFFFFFF) into STI_OFF instead of the raw over-claimed sti; followers (b-feed/a-feed/ +// compute) retire when STI_OFF == 0xFFFFFFFF instead of comparing against the (now +// nonexistent) TOTAL_super kernarg. +// See "CLAIM-COUNTER & completion occ offsets" by .Lclaimer for the full occ-buffer layout +// (occ[24]/occW[6] = chunk terminal bound, added by FIX 1). +// +// SCALAR REGS (derived in the prologue, before any clobber; none collide with DECODE_STI's own clobber +// list s18/s36, lds_*'s s49, the claimer's s16/s17/s35/s44, or any role body's transients, all <= s65): +// s66 = n_kseg (derived; dead after shift/mask below are computed -- kept only for that derivation) +// s67 = mask (n_kseg - 1) -- LIVE for the whole kernel; read by every DECODE_STI call, every role. +// s68 = shift (log2 n_kseg) -- LIVE for the whole kernel; read by every DECODE_STI call, every role. +// s69 = chunkHi (claimer-only; loaded once per dispatch from occ[24] right before .Lclaim_loop). +// +// Everything new is gated behind the fresh `DSWS2` build symbol (analogous to coop's `DSWS`). + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1201" + +// ---- tile defsyms (lifted from occ_kernel_coop.s) ---- +.ifndef FM + .set FM, 2 // per-compute-wave M-frags (M-band = FM*16 rows) +.endif +.ifndef FN + .set FN, 4 // shared N-frags (the reuse operand) +.endif +.ifndef RGADESC + .set RGADESC, 0 // 1 = emit analysis-only AMDHSA descriptor for RGA livereg +.endif +.ifndef DIAG + .set DIAG, 0 // 1 = phase-marker instrumentation (unused in the A1 scaffold) +.endif +.ifndef SAFEPROBE + .set SAFEPROBE, 0 // 1 = clamp per-lane vector address regs into a provable in-buffer bound +.endif +.ifndef DYNVGPR + .set DYNVGPR, 1 // 1 = compute waves s_alloc_vgpr-grow per rowblk; feeds/claimer stay lean 32 +.endif +.ifndef SLEEPN + .set SLEEPN, 2 // s_sleep arg in the busy-waits (yield issue cycles to partner waves) +.endif + +// ============================================================================================ +// DSWS v2 LDS layout (bytes from group-segment base; words u32 unless noted). Mirrors the placement +// of the coop file's LDS `.set` block. Defined unconditionally (uses only G/SEGK/FM/FN, always set). +// ============================================================================================ +.ifndef DSWS2 + .set DSWS2, 0 +.endif +.ifndef G + .set G, 6 // cooperative M-extent (rowblks per super-tile) = NCOMP_MAX +.endif +.ifndef SEGK + .set SEGK, 64 // split-K segment size in K-elements (multiple of 16) +.endif +// ---- v2 control/claim words ---- +.set STI_OFF, 0 // broadcast super-tile id +.set EPOCH_OFF, 4 +.set ROWBLK_NEXT_OFF, 8 // per-super-tile rowblk claim counter +.set ROWBLK_DONE_OFF, 12 // per-super-tile completion counter +.set BFRAG_NEXT_OFF, 16 // B-frag claim counter +.set AROW_NEXT_OFF, 20 // A-rowblk claim counter +.set NCOMP_SLOT, 24 +.set NAFEED_SLOT, 28 +.set NBFEED_SLOT, 32 +.set GATE_OFF, 36 // u32[4] -> 36,40,44,48 (conversion gates) +.set VRESV_OFF, 52 // vgpr_reserved +.set SEGCNT_OFF, 56 // controller clock +// ---- A3..A7 additions (still inside the 0..256 control region; A1 offsets 0..56 unchanged) ---- +.set BFRAG_DONE_OFF, 60 // B-frag STORE-completion counter (compute gates on this, NOT the claim ctr) +.set AROW_DONE_OFF, 64 // A-rowblk STORE-completion counter (compute gates on this) +.set INITFLAG_OFF, 68 // barrier-free LDS-init publish flag (claimer writes 0xACED LAST) +.set KSEG_STEPS, (SEGK/16) // K16-steps per split-K segment = SEGK K-elements / 16 +// FIX 1(b): NKSEG_SHIFT = log2(KSEG_STEPS), so the prologue can derive n_kseg = KT >> NKSEG_SHIFT instead +// of receiving it as a (now-dropped) kernarg. SEGK is always a power-of-two multiple of 16 in every +// config this file is built with, so KSEG_STEPS is always a power of two in {1,2,4,8,16}; a static +// ladder over that small set is simpler/safer than a general-purpose compile-time log2. +.if KSEG_STEPS == 1 + .set NKSEG_SHIFT, 0 +.elseif KSEG_STEPS == 2 + .set NKSEG_SHIFT, 1 +.elseif KSEG_STEPS == 4 + .set NKSEG_SHIFT, 2 +.elseif KSEG_STEPS == 8 + .set NKSEG_SHIFT, 3 +.elseif KSEG_STEPS == 16 + .set NKSEG_SHIFT, 4 +.else + .error "KSEG_STEPS (SEGK/16) must be a power of two in {1,2,4,8,16}" +.endif +// resident regions aligned to 256B +.set BRES_OFF, 256 // resident B for current super-tile +.set BRES_BYTES, (FN*16*SEGK) // = 4*16*64 = 4096 at the default config +.set ARES_OFF, (BRES_OFF + BRES_BYTES) // resident A for current super-tile +.set ARES_BYTES, (G*16*FM*SEGK) // = 6*16*2*64 = 12288 at the default config +.set LDS_TOTAL_DSWS2, (ARES_OFF + ARES_BYTES) +.if LDS_TOTAL_DSWS2 > 32768 + .error "DSWS2 LDS layout exceeds 32768B group segment" +.endif + +.if DSWS2 + // ---- role counts (lifted from coop's `.ifndef NCOMP` etc., gated under DSWS2) ---- + .ifndef NCOMP + .set NCOMP, 4 // compute waves (fat, dyn-grow). Compute floor >= 1. + .endif + .ifndef NAFEED + .set NAFEED, 2 // A-feed waves (lean). Feed floor >= 1. + .endif + .ifndef NBFEED + .set NBFEED, 2 // B-feed waves (lean). Feed floor >= 1. + .endif + .set WAVES, (NCOMP + NAFEED + NBFEED) // total waves launched per WG (harness dims must match) +.endif + +// ============================================================================================ +// VGPR layout (lifted from occ_kernel_coop.s) -- compute frags live ABOVE the lean-32 block and are +// only touched AFTER s_alloc_vgpr NFV. Feeds/claimer stay in the lean block (v0..v31). +// ============================================================================================ +.set ACC, 32 // accumulators: FM*FN frags x 8 f32 (v32..) +.set FA, (ACC + 8*FM*FN) // compute A frags (from resident LDS): FM x 2 +.set FB, (FA + 2*FM) // compute B frags (from resident LDS): FN x 2 +.set NFV, ((FB + 2*FN + 15) & ~15) // grown footprint, rounded to a 16-VGPR dyn block (=112 @ 2x4) +.set VLEAN, 32 // lean footprint (feeds, claimer, compute pre/post rowblk) +.set BSTG, 16 // staging regs (lean block, < 32): B-feed FN-frag / A-feed FM-frag + +// ---- dyn-VGPR PRE-GROW temp-reg ceiling (coop death-cert: a >v15 src pre-grow is poison under dyn). +// Gate every PRE-grow-reachable LDS/atomic temp to v11/v14 (INTERIOR to the launched 16-VGPR block). ---- +.if DYNVGPR + .set RG_A, 11 // lds_get / fetch_add address + .set RG_D, 14 // lds_get / fetch_add data+return + .set RP_A, 11 // lds_put address + .set RP_D, 14 // lds_put data +.else + .set RG_A, 27 + .set RG_D, 28 + .set RP_A, 28 + .set RP_D, 29 +.endif + +// ============================================================================================ +// LDS helper macros (s49 = exec save; v2 = lane = tid&31, set in prologue). +// ============================================================================================ +.macro lds_get sdst, off // wave-uniform read LDS[off] -> scalar sdst + v_mov_b32 v[RG_A], \off + ds_load_b32 v[RG_D], v[RG_A] + s_wait_dscnt 0x0 + v_readfirstlane_b32 \sdst, v[RG_D] +.endm +.macro lds_get_r sdst, saddr // wave-uniform read LDS[saddr] (RUNTIME addr in a sreg) -> sdst + v_mov_b32 v[RG_A], \saddr + ds_load_b32 v[RG_D], v[RG_A] + s_wait_dscnt 0x0 + v_readfirstlane_b32 \sdst, v[RG_D] +.endm +.macro lds_put off, ssrc // lane-0-of-wave writes scalar ssrc -> LDS[off] + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lput_skip\@ + v_mov_b32 v[RP_A], \off + v_mov_b32 v[RP_D], \ssrc + ds_store_b32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lput_skip\@: + s_mov_b32 exec_lo, s49 +.endm +.macro lds_fetch_add sdst, off, val // sdst <- old LDS[off]; LDS[off]+=val (lane-0 atomic, broadcast) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lfa_skip\@ + v_mov_b32 v[RP_A], \off + v_mov_b32 v[RP_D], \val + ds_add_rtn_u32 v[RP_D], v[RP_A], v[RP_D] // v[RP_D] <- old; LDS[off] += val + s_wait_dscnt 0x0 +.Lfa_skip\@: + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 \sdst, v[RP_D] // broadcast lane-0's old value +.endm +.macro lds_inc off // lane-0-of-wave LDS[off] += 1 (no return) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Linc_skip\@ + v_mov_b32 v[RP_A], \off + v_mov_b32 v[RP_D], 1 + ds_add_u32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Linc_skip\@: + s_mov_b32 exec_lo, s49 +.endm + +// ============================================================================================ +// Super-tile decode + resident A/B staging macros (A3..A6). Decode (Naming/symbols): +// ksi = sti & mask ; t = sti >> shift ; mblk = t / NTL ; tcol = t % NTL. +// FIX 1(d): n_kseg is ALWAYS a power of two (it's KT >> NKSEG_SHIFT, both compile-time-shift-derived), +// so the sti->(t,ksi) split is an exact shift/mask -- no magic-div, no n_kseg==1 special-case (shift=0, +// mask=0 falls out of the general path for free: ksi=0, t=sti). /NTL still goes via magic (s12), +// unsigned-division mul_hi (coop GENDIV idiom), since NTL is not generally a power of two. +// ============================================================================================ +.macro DECODE_STI // in: s17=sti, s67=mask, s68=shift ; out: s19=mblk s30=tcol s31=ksi ; clob: s18,s36 + s_and_b32 s31, s17, s67 // ksi = sti & mask + s_lshr_b32 s18, s17, s68 // t = sti >> shift + s_mul_hi_u32 s19, s18, s12 // mblk = t / NTL + s_mul_i32 s36, s19, s13 // mblk * NTL + s_sub_u32 s30, s18, s36 // tcol = t - mblk*NTL +.endm + +// RESIDENT B FRAG LAYOUT: B frag (kstep ks, frag f) at BRES_OFF + (ks*FN + f)*256 +// (each frag = the SAME 256B block coop stores per B-ring slot; lane*8 vaddr base = v9). +// Built here as: dst vbase = v9 + BRES_OFF + f*256 , ds_store offset:(ks*FN*256). +// B global addr (lift coop B-feed): Bshuf + tcol*(FN*256=s14) + (seg k0)* [ksi*KSEG_STEPS*(NT*256=s10)] +// + f*256 (frag, folded into saddr) + ks*(NT*256=s10) (k-step, folded into saddr). +.macro BSTAGE // in: s30=tcol s31=ksi ; clob: s20,s21,s23,s25,s26,s27,v13,v[BSTG..] + s_mul_i32 s20, s30, s14 // tcol * FN*256 + s_mul_i32 s21, s31, KSEG_STEPS // ksi * KSEG_STEPS + s_mul_i32 s21, s21, s10 // * NT*256 -> segment k-start byte offset + s_add_u32 s20, s20, s21 + s_add_u32 s20, s4, s20 + s_addc_u32 s21, s5, 0 // s[20:21] = B base (tcol,ksi, seg k-step 0) +.Lbcl\@: + lds_fetch_add s23, BFRAG_NEXT_OFF, 1 // claim frag f + s_cmp_ge_u32 s23, FN + s_cbranch_scc1 .Lbsd\@ // f>=FN -> all frags claimed + s_lshl_b32 s25, s23, 8 // f*256 + s_add_u32 s26, s20, s25 + s_addc_u32 s27, s21, 0 // s[26:27] = frag f base (seg k0) + v_add_nc_u32 v13, v9, BRES_OFF + v_add_nc_u32 v13, v13, s25 // resident B dst vbase for frag f + .set ks, 0 + .rept KSEG_STEPS + global_load_tr_b64 v[BSTG+ks*2:BSTG+ks*2+1], v9, s[26:27] + s_add_u32 s26, s26, s10 // next k-step (last iter over-advances; unused) + s_addc_u32 s27, s27, 0 + .set ks, ks+1 + .endr + s_wait_loadcnt 0x0 + .set ks, 0 + .rept KSEG_STEPS + ds_store_b64 v13, v[BSTG+ks*2:BSTG+ks*2+1] offset:(ks*FN*256) + .set ks, ks+1 + .endr + s_wait_dscnt 0x0 + lds_inc BFRAG_DONE_OFF // frag f STORED -> publish completion (compute gates on this) + s_branch .Lbcl\@ +.Lbsd\@: +.endm + +// RESIDENT A FRAG LAYOUT: A frag (kstep ks, rowblk r, mi) at ARES_OFF + ((ks*G + r)*FM + mi)*256 +// Built as: dst vbase = v9 + ARES_OFF + r*(FM*256) , ds_store offset:((ks*G*FM + mi)*256). +// A global addr (lift coop compute/A-feed): A + rowblk_abs*(16*FM)*K + mi*16*K + koff, rowblk_abs=mblk*G+r, +// koff = ksi*SEGK (segment K byte offset, fp8 1B/elem), k-step within segment via global offset:ks*16. +.macro ASTAGE // in: s19=mblk s31=ksi ; clob: s22,s23,s25,s32,s36,s40,s41,s44,s45,v13,v[BSTG..] + s_lshl_b32 s32, s9, 4 // rowstride16 = 16*K +.Lacl\@: + lds_fetch_add s23, AROW_NEXT_OFF, 1 // claim rowblk r + s_cmp_ge_u32 s23, G + s_cbranch_scc1 .Lasd\@ + s_mul_i32 s36, s19, G + s_add_u32 s36, s36, s23 // rowblk_abs = mblk*G + r + s_mul_i32 s22, s36, (16*FM) + s_mul_i32 s22, s22, s9 // rowblk_abs*(16*FM)*K + s_mul_i32 s25, s31, SEGK // ksi*SEGK (segment K byte offset) + s_add_u32 s22, s22, s25 + s_add_u32 s40, s2, s22 + s_addc_u32 s41, s3, 0 // s[40:41] = A base (rowblk_abs, mi0, seg k0) + s_mul_i32 s25, s23, (FM*256) // r*FM*256 + v_add_nc_u32 v13, v9, ARES_OFF + v_add_nc_u32 v13, v13, s25 // resident A dst vbase for rowblk r + .set mi, 0 + .rept FM + .if mi == 0 + s_mov_b32 s44, s40 + s_mov_b32 s45, s41 + .else + s_add_u32 s44, s44, s32 // += 16*K (next M-frag) + s_addc_u32 s45, s45, 0 + .endif + .set ks, 0 + .rept KSEG_STEPS + global_load_b64 v[BSTG:BSTG+1], v8, s[44:45] offset:(ks*16) + s_wait_loadcnt 0x0 + ds_store_b64 v13, v[BSTG:BSTG+1] offset:((ks*G*FM + mi)*256) + s_wait_dscnt 0x0 + .set ks, ks+1 + .endr + .set mi, mi+1 + .endr + lds_inc AROW_DONE_OFF // rowblk r fully STAGED -> publish completion + s_branch .Lacl\@ +.Lasd\@: +.endm + +// ============================================================================================ +// KERNEL +// ============================================================================================ + .text + .globl occ_kernel + .p2align 8 + .type occ_kernel,@function +occ_kernel: + // ---- FIX 1(b,c): derive n_kseg from KT (s8) + the compile-time NKSEG_SHIFT, then the shift/mask + // decode pair, into the reserved high SGPRs s66/s67/s68 BEFORE any clobber (SAFEPROBE below reuses + // s16 purely as scratch; the role bodies keep all transients <= s65). No v2 kernargs are read here + // anymore -- s15/s16/s17 are NOT hardware-preloaded under USER_SGPR=15 (see KERNARG CONTRACT above). ---- + s_lshr_b32 s66, s8, NKSEG_SHIFT // n_kseg = KT >> NKSEG_SHIFT (KT=s8) + s_ff1_i32_b32 s68, s66 // shift = log2(n_kseg) (bit index of the single set bit; n_kseg=1 -> 0) + s_sub_u32 s67, s66, 1 // mask = n_kseg - 1 + // ---- identity (lifted from coop prologue; v0=tid hardware-preloaded) ---- + v_lshrrev_b32 v1, 5, v0 // wid = tid >> 5 + v_and_b32 v2, 31, v0 // lane = tid & 31 + v_and_b32 v6, 15, v0 // lane & 15 (A vaddr) + v_mov_b32 v4, 0 + // ---- per-lane address constants (mbgemm-identical; dyn-VGPR arming compatible) ---- + v_mul_lo_u32 v8, v6, s9 // (lane&15)*K + v_bfe_u32 v7, v0, 4, 1 + v_lshlrev_b32 v7, 3, v7 + v_add_nc_u32 v8, v8, v7 // v8 = A vaddr = (lane&15)*K + colhi*8 + v_lshlrev_b32 v9, 3, v2 // v9 = B/ds vaddr = lane*8 + v_lshlrev_b32 v10, 5, v2 // v10 = C store vaddr = lane*32 +.if SAFEPROBE + // brick-PROOF: clamp the per-lane VECTOR address regs to a loose upper bound (>= true max) so even a + // grow-corrupted vaddr cannot push a global access past the data+guard (pairs with the future ti clamp). + s_lshl_b32 s16, s9, 4 // 16*K (>= v8 max = (lane&15)*K + colhi*8 = 15*K+8) + v_min_u32 v8, s16, v8 // clamp A vaddr + v_min_u32 v9, 0x100, v9 // clamp B/ds vaddr (256 >= lane*8 max 248) + v_min_u32 v10, 0x400, v10 // clamp C vaddr (1024 >= lane*32 max 992) +.endif + +.if DSWS2 + // ===== DSWS v2 role branch (wid uniform per wave; scalar-only -> exec stays full for every role). + // wid == 0 -> claimer (pinned super-tile broadcaster; A3) + // wid [0,NBFEED) -> B-feed (A4) + // wid [NBFEED,NBFEED+NAFEED) -> A-feed (A5) + // wid [NBFEED+NAFEED, WAVES) -> compute (A6) ===== + // A1: every role label is just a distinct s_endpgm stub (unique s50 tag keeps them at distinct addresses). + v_readfirstlane_b32 s24, v1 // wid (uniform per wave) + s_cmp_eq_u32 s24, 0 + s_cbranch_scc1 .Lclaimer + s_cmp_lt_u32 s24, NBFEED + s_cbranch_scc1 .Lbfeed + s_cmp_lt_u32 s24, (NBFEED+NAFEED) + s_cbranch_scc1 .Lafeed + s_branch .Lcompute + +// ============================================================================================ +// A3 -- .Lclaimer : pinned wid-0. Owns the super-tile claim+broadcast, the SEGCNT clock, the +// barrier-free LDS init, the completion live++/live-- (harness occ[0]==0 gate), AND -- being a +// B-feed-class wave -- stages B for the current super-tile each iteration (A4 body via BSTAGE). +// +// CLAIM-COUNTER & completion occ-buffer offsets (occ base = s0:s1; host zero-inits the whole buffer): +// occ[0] (offset 0) = live counter (claimer +1 at entry, -1 at terminal; harness polls ==0) +// occ[20] (offset 20) = GLOBAL super-tile claim counter (mirrors coop's tile-claim at offset:20) +// (offsets 4/8/12/16 stay reserved for the coop-style maxlive/timers/total bookkeeping; unused here.) +// ============================================================================================ +.Lclaimer: +.if DYNVGPR +.Lclaimer_alloc: + s_alloc_vgpr 32 // commit lean (dyn WG-allocator consistency); SCC-retry guard + s_cbranch_scc0 .Lclaimer_alloc +.endif + // live++ : lane0 occ[0] += 1 + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lclaimer_live + v_mov_b32 v3, 1 + global_atomic_add_u32 v4, v3, s[0:1] scope:SCOPE_DEV +.Lclaimer_live: + s_mov_b32 exec_lo, s16 + // --- barrier-free LDS-control init: zero all control words; INITFLAG = 0xACED LAST --- + lds_put STI_OFF, 0 + lds_put EPOCH_OFF, 0 + lds_put ROWBLK_NEXT_OFF, 0 + lds_put ROWBLK_DONE_OFF, 0 + lds_put BFRAG_NEXT_OFF, 0 + lds_put AROW_NEXT_OFF, 0 + lds_put BFRAG_DONE_OFF, 0 + lds_put AROW_DONE_OFF, 0 + lds_put NCOMP_SLOT, NCOMP + lds_put NAFEED_SLOT, NAFEED + lds_put NBFEED_SLOT, NBFEED + lds_put GATE_OFF, 0 + lds_put (GATE_OFF+4), 0 + lds_put (GATE_OFF+8), 0 + lds_put (GATE_OFF+12), 0 + lds_put VRESV_OFF, (NCOMP*NFV + (NAFEED+NBFEED)*VLEAN) + lds_put SEGCNT_OFF, 0 + lds_put INITFLAG_OFF, 0xACED // LAST: publishes "LDS ready" to all follower waves + // FIX 1(e): load this dispatch's chunk terminal bound from occ[24] (host writes occW[6] per chunk; + // FIX 1j on the host side). All lanes read the same address -> no exec masking needed, just a + // plain broadcast load; stable for the WHOLE chunk, so load it ONCE here, not per-claim. + global_load_b32 v6, v4, s[0:1] offset:24 scope:SCOPE_DEV + s_wait_loadcnt 0x0 + v_readfirstlane_b32 s69, v6 // s69 = chunkHi (this dispatch's terminal sti bound) + s_mov_b32 s35, 0 // claimer local epoch +.Lclaim_loop: + // claim next sti: lane0 global_atomic_add occ[20] += 1, return old + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lclaim_grabbed + v_mov_b32 v3, 1 + global_atomic_add_u32 v5, v4, v3, s[0:1] offset:20 th:TH_ATOMIC_RETURN scope:SCOPE_DEV + s_wait_loadcnt 0x0 +.Lclaim_grabbed: + s_mov_b32 exec_lo, s16 + v_readfirstlane_b32 s17, v5 // sti + s_cmp_ge_u32 s17, s69 // sti >= chunkHi (occ[24]) -> terminal + s_cbranch_scc1 .Lclaimer_terminal + DECODE_STI // -> s19=mblk s30=tcol s31=ksi + // reset per-super-tile claim/completion counters BEFORE the epoch bump (followers see them reset) + lds_put ROWBLK_NEXT_OFF, 0 + lds_put ROWBLK_DONE_OFF, 0 + lds_put BFRAG_NEXT_OFF, 0 + lds_put AROW_NEXT_OFF, 0 + lds_put BFRAG_DONE_OFF, 0 + lds_put AROW_DONE_OFF, 0 + lds_put STI_OFF, s17 // publish STI FIRST... + lds_get s44, SEGCNT_OFF // bump SEGCNT (controller clock; +1/super-tile) + s_add_u32 s44, s44, 1 + lds_put SEGCNT_OFF, s44 + lds_get s44, EPOCH_OFF // ...then bump EPOCH LAST + s_add_u32 s44, s44, 1 + s_mov_b32 s35, s44 + lds_put EPOCH_OFF, s44 + BSTAGE // claimer helps stage B for this super-tile (s30,s31) + // A7 advance gate: free resident A/B only when ALL G rowblks are computed+flushed +.Lclaimer_wait_done: + s_sleep SLEEPN + lds_get s44, ROWBLK_DONE_OFF // (a) all G rowblks computed + flushed + s_cmp_lt_u32 s44, G + s_cbranch_scc1 .Lclaimer_wait_done + lds_get s44, BFRAG_DONE_OFF // all B frags stored + s_cmp_lt_u32 s44, FN + s_cbranch_scc1 .Lclaimer_wait_done + lds_get s44, AROW_DONE_OFF // all A rowblks staged + s_cmp_lt_u32 s44, G + s_cbranch_scc1 .Lclaimer_wait_done + // (b) QUIESCE the CLAIM counters before reset: each role wave must have executed its terminal + // over-claim (fetch_add returns >=threshold, then bails) BEFORE we reset, else a descheduled + // straggler's next fetch_add returns 0 and claims index 0 of the NEXT super-tile against stale + // decode/resident state (round-table finding #1). Sentinels = threshold + #role-waves (each does + // exactly one terminal bail). NOTE: compile-time NCOMP/NAFEED/NBFEED is correct for STATIC roles; + // Phase-B conversion must switch these to live role counts / epoch-snapshot drained counters. + lds_get s44, ROWBLK_NEXT_OFF // G claims + NCOMP terminal bails + s_cmp_lt_u32 s44, (G + NCOMP) + s_cbranch_scc1 .Lclaimer_wait_done + lds_get s44, BFRAG_NEXT_OFF // FN claims + NBFEED terminal bails + s_cmp_lt_u32 s44, (FN + NBFEED) + s_cbranch_scc1 .Lclaimer_wait_done + lds_get s44, AROW_NEXT_OFF // G claims + NAFEED terminal bails + s_cmp_lt_u32 s44, (G + NAFEED) + s_cbranch_scc1 .Lclaimer_wait_done + s_branch .Lclaim_loop +.Lclaimer_terminal: + lds_put STI_OFF, 0xFFFFFFFF // FIX 1(e): publish SENTINEL (not the raw over-claimed sti)... + lds_get s44, EPOCH_OFF // ...bump epoch so followers wake + retire (A7 terminal) + s_add_u32 s44, s44, 1 + lds_put EPOCH_OFF, s44 + // live-- : lane0 occ[0] -= 1 (harness completion gate fires) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lclaimer_dead + v_mov_b32 v3, -1 + global_atomic_add_u32 v4, v3, s[0:1] scope:SCOPE_DEV +.Lclaimer_dead: + s_mov_b32 exec_lo, s16 + s_endpgm + +// ============================================================================================ +// A4 -- .Lbfeed : B-feed wave. Follows EPOCH/STI, decodes (tcol,ksi), stages its claimed B frags. +// ============================================================================================ +.Lbfeed: +.if DYNVGPR +.Lbfeed_alloc: + s_alloc_vgpr 32 + s_cbranch_scc0 .Lbfeed_alloc +.endif +.Lbfeed_init: + s_sleep 1 + lds_get s44, INITFLAG_OFF + s_cmp_eq_u32 s44, 0xACED + s_cbranch_scc0 .Lbfeed_init // wait for the claimer's barrier-free LDS init + s_mov_b32 s35, 0 // local epoch +.Lbfeed_follow: + s_sleep SLEEPN + lds_get s44, EPOCH_OFF + s_cmp_eq_u32 s44, s35 + s_cbranch_scc1 .Lbfeed_follow // wait next super-tile (epoch change) + s_mov_b32 s35, s44 + lds_get s17, STI_OFF + s_cmp_eq_u32 s17, 0xFFFFFFFF // FIX 1(f): sentinel (A7) -> retire (was: STI>=TOTAL_super) + s_cbranch_scc1 .Lretire + DECODE_STI // s30=tcol s31=ksi (mblk unused) + BSTAGE + s_branch .Lbfeed_follow + +// ============================================================================================ +// A5 -- .Lafeed : A-feed wave. Follows EPOCH/STI, decodes (mblk,ksi), stages its claimed A rowblks. +// ============================================================================================ +.Lafeed: +.if DYNVGPR +.Lafeed_alloc: + s_alloc_vgpr 32 + s_cbranch_scc0 .Lafeed_alloc +.endif +.Lafeed_init: + s_sleep 1 + lds_get s44, INITFLAG_OFF + s_cmp_eq_u32 s44, 0xACED + s_cbranch_scc0 .Lafeed_init + s_mov_b32 s35, 0 +.Lafeed_follow: + s_sleep SLEEPN + lds_get s44, EPOCH_OFF + s_cmp_eq_u32 s44, s35 + s_cbranch_scc1 .Lafeed_follow + s_mov_b32 s35, s44 + lds_get s17, STI_OFF + s_cmp_eq_u32 s17, 0xFFFFFFFF // FIX 1(f): sentinel (A7) -> retire (was: STI>=TOTAL_super) + s_cbranch_scc1 .Lretire + DECODE_STI // s19=mblk s31=ksi (tcol unused) + ASTAGE + s_branch .Lafeed_follow + +// ============================================================================================ +// A6 -- .Lcompute : compute wave. Follows EPOCH/STI, decodes (mblk,tcol,ksi), waits resident A/B +// fully staged (DONE counters), then claims rowblks, runs WMMA over the SEGK segment, and flushes +// fp32 partials into C via global_atomic_add_f32 (split-K segments accumulate into the same C cell). +// ============================================================================================ +.Lcompute: +.if DYNVGPR +.Lcompute_alloc: + s_alloc_vgpr 32 // lean baseline; grow per rowblk + s_cbranch_scc0 .Lcompute_alloc +.endif +.Lcompute_init: + s_sleep 1 + lds_get s44, INITFLAG_OFF + s_cmp_eq_u32 s44, 0xACED + s_cbranch_scc0 .Lcompute_init + s_mov_b32 s35, 0 +.Lcompute_follow: + s_sleep SLEEPN + lds_get s44, EPOCH_OFF + s_cmp_eq_u32 s44, s35 + s_cbranch_scc1 .Lcompute_follow + s_mov_b32 s35, s44 + lds_get s17, STI_OFF + s_cmp_eq_u32 s17, 0xFFFFFFFF // FIX 1(f): sentinel (A7) -> retire (was: STI>=TOTAL_super) + s_cbranch_scc1 .Lretire + DECODE_STI // s19=mblk s30=tcol s31=ksi + // wait until resident A AND B fully STAGED (B: FN frags stored, A: G rowblks stored) +.Lcompute_staged: + s_sleep SLEEPN + lds_get s44, BFRAG_DONE_OFF + s_cmp_lt_u32 s44, FN + s_cbranch_scc1 .Lcompute_staged + lds_get s44, AROW_DONE_OFF + s_cmp_lt_u32 s44, G + s_cbranch_scc1 .Lcompute_staged + // C tile-term: ti = mblk*NTL + tcol ; ti*(G*FM*FN*1024) (ksi-INDEPENDENT -> split-K accumulates) + s_mul_i32 s38, s19, s13 + s_add_u32 s38, s38, s30 + s_mul_i32 s38, s38, (G*FM*FN*1024) +.Lcompute_claim: + lds_fetch_add s33, ROWBLK_NEXT_OFF, 1 // claim rowblk r in [0,G) + s_cmp_ge_u32 s33, G + s_cbranch_scc1 .Lcompute_drained +.if DYNVGPR + s_wait_loadcnt 0x0 + s_wait_storecnt 0x0 +.Lcompute_grow: + s_alloc_vgpr NFV // grow (SCC-retry guarded, brick-class rule) + s_cbranch_scc0 .Lcompute_grow +.endif + // zero FM*FN fp32 accumulators + .set idx, 0 + .rept FM*FN + v_mov_b32 v[ACC+idx*8+0], 0 + v_mov_b32 v[ACC+idx*8+1], 0 + v_mov_b32 v[ACC+idx*8+2], 0 + v_mov_b32 v[ACC+idx*8+3], 0 + v_mov_b32 v[ACC+idx*8+4], 0 + v_mov_b32 v[ACC+idx*8+5], 0 + v_mov_b32 v[ACC+idx*8+6], 0 + v_mov_b32 v[ACC+idx*8+7], 0 + .set idx, idx+1 + .endr + // resident operand bases (lane*8 + region [+ r*FM*256 for A]) + v_add_nc_u32 v12, v9, BRES_OFF + s_mul_i32 s37, s33, (FM*256) + v_add_nc_u32 v13, v9, ARES_OFF + v_add_nc_u32 v13, v13, s37 + // WMMA over the SEGK segment (KSEG_STEPS k-steps); read resident B(ks) + A(ks,r) from LDS + .set ks, 0 + .rept KSEG_STEPS + .set ni, 0 + .rept FN + ds_load_b64 v[FB+ni*2:FB+ni*2+1], v12 offset:((ks*FN+ni)*256) + .set ni, ni+1 + .endr + .set mi, 0 + .rept FM + ds_load_b64 v[FA+mi*2:FA+mi*2+1], v13 offset:((ks*G*FM+mi)*256) + .set mi, mi+1 + .endr + s_wait_dscnt 0x0 + .set mi, 0 + .rept FM + .set ni, 0 + .rept FN + v_wmma_f32_16x16x16_fp8_fp8 v[ACC+(mi*FN+ni)*8:ACC+(mi*FN+ni)*8+7], v[FA+mi*2:FA+mi*2+1], v[FB+ni*2:FB+ni*2+1], v[ACC+(mi*FN+ni)*8:ACC+(mi*FN+ni)*8+7] + .set ni, ni+1 + .endr + .set mi, mi+1 + .endr + .set ks, ks+1 + .endr + // flush: C base = C + ti-term + r*(FM*FN*1024) ; per (frag,elem) atomic-add one fp32 (vaddr v10=lane*32) + s_mul_i32 s39, s33, (FM*FN*1024) + s_add_u32 s39, s38, s39 + s_add_u32 s28, s6, s39 + s_addc_u32 s29, s7, 0 + .set frag, 0 + .rept FM*FN + .set e, 0 + .rept 8 + global_atomic_add_f32 v10, v[ACC+frag*8+e], s[28:29] offset:(frag*1024 + e*4) scope:SCOPE_DEV + .set e, e+1 + .endr + .set frag, frag+1 + .endr + s_wait_storecnt 0x0 // atomic-adds READ ACC -> must drain before shrink frees ACC +.if DYNVGPR +.Lcompute_shrink: + s_alloc_vgpr 32 // shrink (SCC-retry guarded) + s_cbranch_scc0 .Lcompute_shrink +.endif + lds_inc ROWBLK_DONE_OFF // rowblk r computed + flushed (frees the A7 advance gate) + s_branch .Lcompute_claim +.Lcompute_drained: + s_branch .Lcompute_follow // this super-tile's compute drained -> re-check epoch/terminal + +// ---- A7 role-agnostic terminal (followers): retire. (Claimer retires via .Lclaimer_terminal.) ---- +.Lretire: + s_endpgm +.else + s_endpgm // DSWS2=0 has no v2 body (this file is always built DSWS2=1) +.endif + .size occ_kernel, .-occ_kernel + +// ---- RGADESC: analysis-only descriptor so `rga -s bin --co` can enumerate + livereg this kernel. +// vgpr 256 ceiling so livereg reports the true s_alloc-grown peak-live. NOT emitted for the PM4 .bin. ---- +.if RGADESC +.amdhsa_kernel occ_kernel + .amdhsa_next_free_vgpr 256 + .amdhsa_next_free_sgpr 72 // body uses up to s69 (s66=n_kseg s67=mask s68=shift s69=chunkHi, FIX 1) + .amdhsa_group_segment_fixed_size 32768 + .amdhsa_user_sgpr_count 15 // FIX 1(g): v2 contract now s0..s14 only (n_kseg/TOTAL_super/ + // magic_kseg dropped -- derived in-kernel / memory-carried) + .amdhsa_wavefront_size32 1 +.end_amdhsa_kernel +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: occ_kernel + .symbol: occ_kernel.kd + .kernarg_segment_size: 60 + .kernarg_segment_align: 8 + .group_segment_fixed_size: 32768 + .private_segment_fixed_size: 0 + .wavefront_size: 32 + .sgpr_count: 72 + .vgpr_count: 256 + .max_flat_workgroup_size: 256 + .args: [] +.end_amdgpu_metadata +.endif diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/test_dsws_ctrl_model.cpp b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/test_dsws_ctrl_model.cpp new file mode 100644 index 000000000000..731a7c6678ab --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/test_dsws_ctrl_model.cpp @@ -0,0 +1,51 @@ +// TDD for the DSWS control-law logic (SPEC_DSWS_CONTROLLER.md). These four pure functions +// are the *reference semantics* the Phase-3 gfx1201 asm must match 1:1 (gate_try_win -> +// ds_cmpst_b32 LDS CAS; reserve_grow -> atomic_add/sub on the vgpr_reserved LDS counter; +// watermark_decision/epoch_of -> the boundary decision math). Locking the logic here, under a +// real multi-thread race, proves the protocol before it's transcribed into hand-asm. +#include "dsws_ctrl_model.cpp" +#include +#include +#include +#include +#include + +int main() { + // ---- watermark bands: occhigh over-served(-1), else dead-zone(0) ---- + assert(watermark_decision(0, 2, 6) == +1); // empty ring -> starved + assert(watermark_decision(7, 2, 6) == -1); // full ring -> over-served + assert(watermark_decision(4, 2, 6) == 0); // mid -> dead-zone + assert(watermark_decision(2, 2, 6) == 0); // on the low edge is NOT starved (occhigh is strict) + + // ---- epoch clock: E = segments_processed >> EPOCH_SHIFT ---- + assert(epoch_of(0, 3) == 0 && epoch_of(7, 3) == 0 && epoch_of(8, 3) == 1 && + epoch_of(15, 3) == 1 && epoch_of(16, 3) == 2); + assert(epoch_of(5, 0) == 5); // shift 0 -> every segment is its own epoch + + // ---- gate: exactly ONE winner per epoch among many racing waves (the anti-thrash CAS) ---- + for (uint32_t E = 1; E < 50; ++E) { + std::atomic g{E - 1}; // gate last fired at epoch E-1; a fresh epoch E is open + std::atomic wins{0}; + std::vector ts; + for (int i = 0; i < 64; ++i) ts.emplace_back([&] { if (gate_try_win(g, E)) wins++; }); + for (auto& t : ts) t.join(); + assert(wins.load() == 1); // single-winner invariant + assert(g.load() == E); // gate advanced to E + } + // a SECOND attempt at the same epoch must lose (already fired this epoch) + { std::atomic g{5}; assert(!gate_try_win(g, 5)); assert(!gate_try_win(g, 4)); } + + // ---- reservation: concurrent grows never exceed budget; over-budget grows cleanly undo ---- + { std::atomic r{0}; std::atomic ok{0}; + std::vector ts; + for (int i = 0; i < 10; ++i) ts.emplace_back([&] { if (reserve_grow(r, 30, 100)) ok++; }); + for (auto& t : ts) t.join(); + assert(r.load() <= 100); // envelope never blown + assert(ok.load() == 3); // 3*30=90<=100; a 4th would be 120>100 -> rejected+undone + assert(r.load() == 90); // exactly the 3 winners' reservations remain + } + + printf("dsws_ctrl_model: ALL PASS\n"); + return 0; +} From 54217513742b67a5061429f09a0ca92db0260546 Mon Sep 17 00:00:00 2001 From: mad-lab-kbando <69054773+kmbandy@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:40:29 -0400 Subject: [PATCH 02/19] feat(dsws): CPU model for Phase-B snapshot/quiesce + N-1 cross-check Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0132aDSBLwusCJ4KzHQTnvdu --- .../spike/dvgpr_occ/dsws_ctrl_model.cpp | 21 +++++++++++ .../spike/dvgpr_occ/test_dsws_ctrl_model.cpp | 35 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/dsws_ctrl_model.cpp b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/dsws_ctrl_model.cpp index 4766045ec531..5fed956d9fa3 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/dsws_ctrl_model.cpp +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/dsws_ctrl_model.cpp @@ -52,3 +52,24 @@ static inline bool reserve_grow(std::atomic& resv, uint32_t delta, uin } return true; } + +struct WgSnap { uint32_t nC, nA, nB; }; + +static inline WgSnap snapshot_counts(uint32_t nC, uint32_t nA, uint32_t nB) { + return WgSnap{nC, nA, nB}; +} + +// Sentinels = work-threshold + snapshot role-count terminal bails (Phase A arithmetic, +// with compile-time constants replaced by the per-epoch snapshot). +static inline bool quiesce_ready(uint32_t rowblk_next, uint32_t bfrag_next, + uint32_t arow_next, const WgSnap& s, + uint32_t G, uint32_t FN) { + return rowblk_next >= (G + s.nC) + && bfrag_next >= (FN + s.nB) + && arow_next >= (G + s.nA); +} + +// Role-agnostic safety net: fixed N waves, wid0 claimer never bails -> exactly N-1 bails. +static inline bool quiesce_ready_nm1(uint32_t quiesce_cnt, uint32_t N) { + return quiesce_cnt >= (N - 1); +} diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/test_dsws_ctrl_model.cpp b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/test_dsws_ctrl_model.cpp index 731a7c6678ab..4c62c06a872d 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/test_dsws_ctrl_model.cpp +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/test_dsws_ctrl_model.cpp @@ -46,6 +46,41 @@ int main() { assert(r.load() == 90); // exactly the 3 winners' reservations remain } + // ---- snapshot/quiesce (Phase B Decision 1) ---- + { + // snapshot freezes the counts used to size the quiesce sentinels + WgSnap s = snapshot_counts(4, 2, 2); // G=6, FN=4 + // not ready: rowblk short of G + nC terminal bails + assert(!quiesce_ready(6 + 3, 4 + 2, 6 + 2, s, 6, 4)); // rowblk 9 < 6+4 + // ready: every counter reached threshold + snapshot bails + assert( quiesce_ready(6 + 4, 4 + 2, 6 + 2, s, 6, 4)); + // a moved partition (3c3a2b) needs different sentinels; old snapshot is wrong high + WgSnap s2 = snapshot_counts(3, 3, 2); // sentinels: rowblk>=9, bfrag>=6, arow>=9 + assert( quiesce_ready(6 + 3, 4 + 2, 6 + 3, s2, 6, 4)); // 9,6,9 all meet -> ready + assert(!quiesce_ready(6 + 3, 4 + 2, 6 + 2, s2, 6, 4)); // arow 8 < 9 -> NOT ready + // N-1 cross-check agrees at the ready point (N=8 -> 7 bails) + assert( quiesce_ready_nm1(7, 8)); + assert(!quiesce_ready_nm1(6, 8)); + } + + { + // Under any interleaving of N-1 bails, quiesce_ready_nm1 must not fire before the last bail. + for (uint32_t trial = 0; trial < 64; ++trial) { + std::atomic cnt{0}; + std::atomic early{false}; + std::vector ts; + const uint32_t N = 8; + for (uint32_t w = 0; w < N - 1; ++w) + ts.emplace_back([&]{ + if (quiesce_ready_nm1(cnt.load(), N)) early.store(true); // read BEFORE our bump + cnt.fetch_add(1, std::memory_order_acq_rel); + }); + for (auto& t : ts) t.join(); + assert(!early.load()); // never ready with a bail still outstanding + assert(quiesce_ready_nm1(cnt.load(), N)); // ready once all N-1 landed + } + } + printf("dsws_ctrl_model: ALL PASS\n"); return 0; } From 3f62cb5f9a7ede4eac84f2689ed622eeb9c4930e Mon Sep 17 00:00:00 2001 From: mad-lab-kbando <69054773+kmbandy@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:54:18 -0400 Subject: [PATCH 03/19] feat(dsws): Phase-B LDS snapshot + quiesce slots (inert at DSWS2_CONV=0) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0132aDSBLwusCJ4KzHQTnvdu --- .../spike/dvgpr_occ/occ_kernel_dsws.s | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s index 4bf185b06788..90fa4587445b 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s @@ -114,6 +114,20 @@ .set BFRAG_DONE_OFF, 60 // B-frag STORE-completion counter (compute gates on this, NOT the claim ctr) .set AROW_DONE_OFF, 64 // A-rowblk STORE-completion counter (compute gates on this) .set INITFLAG_OFF, 68 // barrier-free LDS-init publish flag (claimer writes 0xACED LAST) +// ---- Phase-B (DSWS2_CONV) control state: role-mix snapshot slots + quiesce counter ---- +// Based at INITFLAG_OFF+4 (NOT the brief's SEGCNT_OFF+4): the brief predates the A3..A7 control +// words (BFRAG_DONE/AROW_DONE/INITFLAG at 60/64/68), so SEGCNT_OFF+4=60 would collide with them. +// Basing after the LAST control word keeps the new state inside the 0..255 control gap BELOW the +// fixed resident region (BRES_OFF=256), so NO resident-region repoint is needed -- the resident +// BRES_OFF/ARES_OFF immediates (emitted unconditionally in the kernel body) stay untouched, which +// is what keeps the DSWS2_CONV=0 binary byte-identical to the Phase-A green bin. All `.set`s here +// are inert (emit no bytes); the only new code (claimer init) is gated under `.if DSWS2_CONV`. +.ifndef DSWS2_CONV + .set DSWS2_CONV, 0 // 0 = pre-conversion static substrate (Phase A green); 1 = Phase B +.endif +.set SNAP_BASE, (INITFLAG_OFF + 4) // u32[6]: [parity*3 + {0:nC,1:nA,2:nB}] role-mix snapshots +.set QUIESCE_CNT_OFF,(SNAP_BASE + 6*4) // u32 role-agnostic bail counter +.set DSWS2_STATE_END,(QUIESCE_CNT_OFF + 4) .set KSEG_STEPS, (SEGK/16) // K16-steps per split-K segment = SEGK K-elements / 16 // FIX 1(b): NKSEG_SHIFT = log2(KSEG_STEPS), so the prologue can derive n_kseg = KT >> NKSEG_SHIFT instead // of receiving it as a (now-dropped) kernarg. SEGK is always a power-of-two multiple of 16 in every @@ -141,6 +155,10 @@ .if LDS_TOTAL_DSWS2 > 32768 .error "DSWS2 LDS layout exceeds 32768B group segment" .endif +// Phase-B state must fit in the control gap below the resident region (inert compile check, no bytes). +.if DSWS2_STATE_END > BRES_OFF + .error "DSWS2 Phase-B state (SNAP_BASE/QUIESCE_CNT) overlaps resident B region (BRES_OFF)" +.endif .if DSWS2 // ---- role counts (lifted from coop's `.ifndef NCOMP` etc., gated under DSWS2) ---- @@ -430,6 +448,17 @@ occ_kernel: lds_put (GATE_OFF+12), 0 lds_put VRESV_OFF, (NCOMP*NFV + (NAFEED+NBFEED)*VLEAN) lds_put SEGCNT_OFF, 0 +.if DSWS2_CONV + // Phase-B: seed BOTH epoch-parity role-mix snapshots with the launch mix, zero the quiesce counter. + // Gated so DSWS2_CONV=0 emits ZERO new bytes -> byte-identical to the Phase-A green bin. + lds_put QUIESCE_CNT_OFF, 0 + lds_put (SNAP_BASE + 0), NCOMP // parity-0 snapshot = launch mix + lds_put (SNAP_BASE + 4), NAFEED + lds_put (SNAP_BASE + 8), NBFEED + lds_put (SNAP_BASE + 12), NCOMP // parity-1 = launch mix too (init) + lds_put (SNAP_BASE + 16), NAFEED + lds_put (SNAP_BASE + 20), NBFEED +.endif lds_put INITFLAG_OFF, 0xACED // LAST: publishes "LDS ready" to all follower waves // FIX 1(e): load this dispatch's chunk terminal bound from occ[24] (host writes occW[6] per chunk; // FIX 1j on the host side). All lanes read the same address -> no exec masking needed, just a From c714001420ff4a8d23de0594c43161ca59059c6b Mon Sep 17 00:00:00 2001 From: mad-lab-kbando <69054773+kmbandy@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:07:31 -0400 Subject: [PATCH 04/19] feat(dsws): Phase-B consume-point sensing (read-only) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0132aDSBLwusCJ4KzHQTnvdu --- .../spike/dvgpr_occ/occ_kernel_dsws.s | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s index 90fa4587445b..c3b1cd31df90 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s @@ -353,6 +353,41 @@ .Lasd\@: .endm +// ============================================================================================ +// Phase-B (DSWS2_CONV) consume-point ring-occupancy sensor -- Task 3, READ-ONLY (actuation is Task 5). +// Mirrors the coop occ_a/occ_b sensor (occ = producer - consumer, sampled where the value is +// CONSUMED, not at the segment boundary). The claimer's A7 wait-done spin runs CONCURRENTLY with the +// compute drain, so it observes the ring mid-flight; at the segment boundary the resident region has +// fully drained and occ would read a stuck ~0 (permanent false-starvation) -- exactly what SPEC warns. +// +// COUNTER IDENTITIES (confirmed against the live claim/consume sites -- see report): +// producer = the STORE-completion counters the compute wave actually gates on: +// A-ring: AROW_DONE_OFF (A rowblks resident, monotonic in [0,G]; lds_inc @ ASTAGE) +// B-ring: BFRAG_DONE_OFF (B frags resident, monotonic in [0,FN]; lds_inc @ BSTAGE) +// NOT the *_NEXT claim counters: AROW_NEXT/BFRAG_NEXT overshoot the ring depth by the role +// terminal-bails (G+NAFEED / FN+NBFEED), which would break the occ <= depth bound. +// consumer = ROWBLK_NEXT_OFF, the compute rowblk-claim clock (consume progress through the super-tile: +// each claimed rowblk r consumes A(r) and re-reads all FN shared B frags). +// min-clamp: cons is clamped to prod before the subtract so the u32 result cannot underflow when the +// consume clock outruns a shallower ring (G=6 > FN=4 -> ROWBLK_NEXT can exceed BFRAG_DONE). +// INVARIANT preserved: occ_A in [0,G], occ_B in [0,FN] (nonnegative, bounded by ring depth). +// +// REGISTER DISCIPLINE (brick-critical; this path is reachable pre-grow -- a >v15 vector temp is +// OOR-poison under dyn-VGPR, SPEC S4): scalars <= s65 only (s60/s61 scratch; callers pass dst in +// [s62,s65]); the only vector temps are inside lds_get, which uses v11/v14 (INTERIOR to the launch +// 16-VGPR block) -- NO >v15 temp is introduced here. +.if DSWS2_CONV +.macro occ_sample dst_a, dst_b // out: \dst_a=occ_A in [0,G], \dst_b=occ_B in [0,FN]; clob s60,s61 + lds_get \dst_a, AROW_DONE_OFF // prod_a: A rowblks resident (store-completion) + lds_get \dst_b, BFRAG_DONE_OFF // prod_b: B frags resident (store-completion) + lds_get s60, ROWBLK_NEXT_OFF // cons : compute rowblk-claim consume clock + s_min_u32 s61, s60, \dst_a // cons_a = min(clock, prod_a) (clamp -> no u32 underflow) + s_sub_u32 \dst_a, \dst_a, s61 // occ_A = prod_a - cons_a in [0,G] + s_min_u32 s61, s60, \dst_b // cons_b = min(clock, prod_b) + s_sub_u32 \dst_b, \dst_b, s61 // occ_B = prod_b - cons_b in [0,FN] +.endm +.endif + // ============================================================================================ // KERNEL // ============================================================================================ @@ -501,6 +536,26 @@ occ_kernel: // A7 advance gate: free resident A/B only when ALL G rowblks are computed+flushed .Lclaimer_wait_done: s_sleep SLEEPN +.if DSWS2_CONV +.if DIAG + // Phase-B DIAG probe (Task 3): wid 0 samples the LIVE ring occupancy (compute is mid-drain here) + // and publishes the last-sampled occ_A/occ_B so a GPU run can confirm the sensor OSCILLATES + // rather than reading a stuck 0. Written every wait-done spin -> a live poller observes it vary. + // Spare word-indexed slots occ[26]/occ[27] -> byte offsets 104/108 (well clear of the + // byte-indexed control words occ[0]/occ[20]/occ[24]). READ-ONLY sensing, NO actuation. + occ_sample s62, s63 + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Locc_diag_skip + v_mov_b32 v14, s62 // occ_A (v14 <= v15: pre-grow safe) + v_mov_b32 v15, s63 // occ_B + global_store_b32 v4, v14, s[0:1] offset:104 scope:SCOPE_DEV // occ[26] = last occ_A + global_store_b32 v4, v15, s[0:1] offset:108 scope:SCOPE_DEV // occ[27] = last occ_B +.Locc_diag_skip: + s_mov_b32 exec_lo, s16 +.endif +.endif lds_get s44, ROWBLK_DONE_OFF // (a) all G rowblks computed + flushed s_cmp_lt_u32 s44, G s_cbranch_scc1 .Lclaimer_wait_done From 6f3e36f45a331f06a3e702382b63cd1e3674341d Mon Sep 17 00:00:00 2001 From: mad-lab-kbando <69054773+kmbandy@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:24:26 -0400 Subject: [PATCH 05/19] feat(dsws): port try_gate ticket + reservation envelope Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0132aDSBLwusCJ4KzHQTnvdu --- .../spike/dvgpr_occ/occ_kernel_dsws.s | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s index c3b1cd31df90..d222df6fb3a8 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s @@ -125,6 +125,9 @@ .ifndef DSWS2_CONV .set DSWS2_CONV, 0 // 0 = pre-conversion static substrate (Phase A green); 1 = Phase B .endif +.ifndef DSWS2_TICKET_SELFTEST + .set DSWS2_TICKET_SELFTEST, 0 // DIAG-only try_gate single-winner smoke (Task 4 Step 3); default 0 = no bytes +.endif .set SNAP_BASE, (INITFLAG_OFF + 4) // u32[6]: [parity*3 + {0:nC,1:nA,2:nB}] role-mix snapshots .set QUIESCE_CNT_OFF,(SNAP_BASE + 6*4) // u32 role-agnostic bail counter .set DSWS2_STATE_END,(QUIESCE_CNT_OFF + 4) @@ -386,6 +389,76 @@ s_min_u32 s61, s60, \dst_b // cons_b = min(clock, prod_b) s_sub_u32 \dst_b, \dst_b, s61 // occ_B = prod_b - cons_b in [0,FN] .endm + +// -------------------------------------------------------------------------------------------- +// Phase-B controller thresholds + sum-envelope budget (Task 4). EPOCH_SHIFT mirrors coop / +// occ_dispatch (epoch = segcnt >> EPOCH_SHIFT). BUDGET is the per-WG VGPR sum-envelope ceiling +// the reservation counter must never exceed; default = the launch reservation, which makes the +// envelope a strict conservation law (a feed->compute grow can only fit if a compute->feed shrink +// already freed the delta). Task 5 may re-tune via `-defsym BUDGET=` if per-SIMD headroom exists. +// -------------------------------------------------------------------------------------------- +.ifndef EPOCH_SHIFT + .set EPOCH_SHIFT, 3 // decision clock: epoch = segcnt >> EPOCH_SHIFT (small = reactive) +.endif +.ifndef BUDGET + .set BUDGET, (NCOMP*NFV + (NAFEED+NBFEED)*VLEAN) // = VRESV_OFF init (conservation ceiling) +.endif + +// try_gate: the lock-free single-winner conversion ticket (transcribed VERBATIM from occ_kernel_coop.s, +// which transcribes dsws_ctrl_model.cpp gate_try_win + epoch_of EXACTLY). E = segcnt>>EPOCH_SHIFT. +// gate[dir] holds the last epoch dir fired. Among many waves racing the same (gtry_gate->reserve_try point -- +// occ_sample's s62/s63 result is consumed into `dir` BEFORE this runs), v5/v6/v7 (<=v15: pre-grow / +// lean-safe). CAS operand order (gfx1201, GCN order -- NOT flipped, KG 9ed04f3c): +// ds_cmpstore_rtn_b32 vdst,vaddr,vNEW,vCMP -> MEM=(MEM==vCMP)?vNEW:MEM, vdst<-old. So vsrc0=E (new), +// vsrc1=g (compare). WIN iff returned-old == g. (Swapping them leaves gate stuck so old==g for ALL +// racers -> every racer "wins" -> would-win ~= NCOMP*epochs instead of ~= epochs.) +.macro try_gate dir, swin + lds_get s62, SEGCNT_OFF // E = epoch_of(segcnt, EPOCH_SHIFT) + s_lshr_b32 s62, s62, EPOCH_SHIFT + lds_get s63, (GATE_OFF + (\dir)*4) // g = gate[dir] + s_mov_b32 \swin, 0 + s_cmp_ge_u32 s63, s62 // g >= E -> dir already fired this/later epoch -> lose + s_cbranch_scc1 .Ltg_done\@ + s_mov_b32 s65, exec_lo // lane0-only CAS (one ticket attempt per WAVE) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltg_restore\@ + v_mov_b32 v5, (GATE_OFF + (\dir)*4) // vaddr = &gate[dir] + v_mov_b32 v6, s63 // v6 = g (vsrc1 = CMP/expected) + v_mov_b32 v7, s62 // v7 = E (vsrc0 = NEW value to store) + ds_cmpstore_rtn_b32 v6, v5, v7, v6 // gate[dir] = (gate[dir]==g) ? E : gate[dir]; v6 <- old + s_wait_dscnt 0x0 +.Ltg_restore\@: + s_mov_b32 exec_lo, s65 + v_readfirstlane_b32 s64, v6 // s64 = old (lane0's CAS result, broadcast) + s_cmp_eq_u32 s64, s63 // WIN iff old == g (we were the swapper) + s_cbranch_scc0 .Ltg_done\@ + s_mov_b32 \swin, 1 +.Ltg_done\@: +.endm + +// reserve_try: the VGPR sum-envelope reservation (transcribes reserve_grow, dsws_ctrl_model.cpp:47). +// Reserve first (atomic add of SIGNED \delta on vgpr_reserved), then validate prev+delta <= BUDGET; +// on over-budget cleanly UNDO (atomic add of -\delta) and reject. The LDS atomic serializes the <=2 +// concurrent grows an epoch permits: the second to validate sees the first's reservation and backs off. +// GROW (feed->compute): pass \delta = +(NFV-VLEAN). Over-budget -> undo, \won=0 (stay in role). +// SHRINK(compute->feed): pass \delta = -(NFV-VLEAN). new = prev+delta < prev <= BUDGET, so the +// validate branch is a proven no-op -> \won=1 ALWAYS (shrink never fails). +// One macro, one call site (Task 5 `reserve_try delta, s_ok`); direction is the sign of \delta. +// Scratch: s62/s63 (free at the bail-commit point -- try_gate's s62..s65 are long dead by then). +.macro reserve_try delta, won + lds_fetch_add s62, VRESV_OFF, (\delta) // s62 = prev reserved; vgpr_reserved += delta + s_add_u32 s63, s62, (\delta) // s63 = new reservation = prev + delta + s_mov_b32 \won, 1 + s_cmp_le_u32 s63, BUDGET // new <= BUDGET -> commit (win); shrink always passes + s_cbranch_scc1 .Lrt_done\@ + lds_fetch_add s62, VRESV_OFF, -(\delta) // over-budget: undo the reservation, reject + s_mov_b32 \won, 0 +.Lrt_done\@: +.endm .endif // ============================================================================================ @@ -434,6 +507,24 @@ occ_kernel: v_readfirstlane_b32 s24, v1 // wid (uniform per wave) s_cmp_eq_u32 s24, 0 s_cbranch_scc1 .Lclaimer +.if DSWS2_CONV && DIAG && DSWS2_TICKET_SELFTEST + // Task 4 Step 3 -- try_gate single-winner SMOKE (assemble-only stub; default off). Every non-claimer + // wave races the (dir=0) ticket ONCE and atomic-adds its win (0/1) into occ[28] (byte offset 112, + // clear of the 0/20/24/104/108 control+probe words). On GPU (Task 6, if enabled) the sum should land + // near #epochs, NOT NCOMP*#epochs -- the harness-side proof the LDS-CAS yields <=1 winner/(dir,epoch). + // v4=0 (set in prologue), v2=lane; try_gate temps v5/v6/v7 are <=v15 (pre-grow safe). wid (s24) + // survives -- try_gate touches only s62..s65 / s16. NOTE: pre-init-rendezvous placement -> a real run + // reads gate/segcnt before the claimer publishes them; fine for an assemble/smoke stub. + try_gate 0, s50 + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltg_selftest_skip + v_mov_b32 v5, s50 // win flag (0/1) for THIS wave + global_atomic_add_u32 v4, v5, s[0:1] offset:112 scope:SCOPE_DEV // occ[28] += win +.Ltg_selftest_skip: + s_mov_b32 exec_lo, s16 +.endif s_cmp_lt_u32 s24, NBFEED s_cbranch_scc1 .Lbfeed s_cmp_lt_u32 s24, (NBFEED+NAFEED) From 8e4a1447c72c8f2acbbcb9f6f2557dc76f70b8f4 Mon Sep 17 00:00:00 2001 From: mad-lab-kbando <69054773+kmbandy@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:59:41 -0400 Subject: [PATCH 06/19] =?UTF-8?q?feat(dsws):=20Phase-B=20Task=205=20?= =?UTF-8?q?=E2=80=94=20bail-time=20commit=20+=20claimer=20snapshot/quiesce?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the conversion crux into occ_kernel_dsws.s (all under .if DSWS2_CONV, byte-identical at DSWS2_CONV=0): - conv_dec_floor: floor-guarded ds_cmpstore CAS decrement of a role slot (never drives a role below 1 wave; two same-source converters/epoch safe). - conv_apply: bail-time COMMIT — floor-dec source, reserve_try envelope, dest inc, s59 role flip, s_alloc_vgpr GROW/SHRINK (SCC-retry), ordered strictly BEFORE the QUIESCE_CNT bump (snapshot handshake, SPEC 3.4 dec 2). - Per-role decision + commit at all 3 terminal bails (compute/afeed/bfeed), dir a compile-time immediate per site. - Quiesce sentinels switched to the epoch-parity snapshot (G+snap.nC etc.) with a QUIESCE_CNT >= WAVES-1 cross-check; DIAG occ[29] mismatch flag. - Broadcast snapshot of the live mix into the next epoch parity + QUIESCE reset, epoch published last. Offline gates green: 3/3 mixes ASSEMBLE_OK, RGA 0 spills; DSWS2_CONV=0 sha256 byte-identical; dry-print NCOMP=4/2/2 no REFUSE; CPU model ALL PASS. Reviewed clean (spec + quality): OOR window safe at all 3 grow sites, commit-before-QUIESCE ordering, floor-guard CAS, reserve_try undo verified. rga_check.sh: make KSRC overridable (${KSRC:-default}) to gate the dsws kernel; default unchanged. PLAN: fix a tautological inline test-assertion example. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0132aDSBLwusCJ4KzHQTnvdu --- .../dvgpr_occ/PLAN_DSWS_PHASEB_CONVERSION.md | 6 +- .../spike/dvgpr_occ/occ_kernel_dsws.s | 233 ++++++++++++++++++ .../spike/dvgpr_occ/rga_check.sh | 2 +- 3 files changed, 237 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_PHASEB_CONVERSION.md b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_PHASEB_CONVERSION.md index c643c00423f7..48e459864b6f 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_PHASEB_CONVERSION.md +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/PLAN_DSWS_PHASEB_CONVERSION.md @@ -64,9 +64,9 @@ The claimer already publishes `NCOMP/NAFEED/NBFEED_SLOT`, `GATE_OFF[4]=0`, `VRES // ready: every counter reached threshold + snapshot bails assert( quiesce_ready(6 + 4, 4 + 2, 6 + 2, s, 6, 4)); // a moved partition (3c3a2b) needs different sentinels; old snapshot is wrong high - WgSnap s2 = snapshot_counts(3, 3, 2); - assert( quiesce_ready(6 + 3, 4 + 2, 6 + 3, s2, 6, 4)); - assert(!quiesce_ready(6 + 4, 4 + 2, 6 + 3, s2, 6, 4) == false); // 6+4>=6+3 ok -> ready; sanity + WgSnap s2 = snapshot_counts(3, 3, 2); // sentinels: rowblk>=9, bfrag>=6, arow>=9 + assert( quiesce_ready(6 + 3, 4 + 2, 6 + 3, s2, 6, 4)); // 9,6,9 all meet -> ready + assert(!quiesce_ready(6 + 3, 4 + 2, 6 + 2, s2, 6, 4)); // arow 8 < 9 -> NOT ready // N-1 cross-check agrees at the ready point (N=8 -> 7 bails) assert( quiesce_ready_nm1(7, 8)); assert(!quiesce_ready_nm1(6, 8)); diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s index d222df6fb3a8..841cbdd371dc 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s @@ -459,6 +459,103 @@ s_mov_b32 \won, 0 .Lrt_done\@: .endm + +// -------------------------------------------------------------------------------------------- +// Phase-B (Task 5) watermark thresholds + LDS put-runtime helper + bail-time commit macros. +// Watermark decision (SPEC; mirrors coop CTRL_LOW/CTRL_HIGH, occ_dispatch DSWS_LOW/HIGH): +// occ_X < CTRL_LOW -> compute STARVED for X -> shrink a compute wave into feed-X. +// occ_X > CTRL_HIGH_X -> feed-X OVER-SERVING -> grow a feed-X wave into compute. +// occ_A in [0,G], occ_B in [0,FN] (occ_sample bounds), so the HIGH marks are per-ring-depth. +// -------------------------------------------------------------------------------------------- +.ifndef CTRL_LOW + .set CTRL_LOW, 1 // occ_X < 1 (== 0, ring empty at consume) -> starved +.endif +.ifndef CTRL_HIGH_A + .set CTRL_HIGH_A, (G-1) // occ_A > G-1 -> A-ring saturated -> A-feed over-serving +.endif +.ifndef CTRL_HIGH_B + .set CTRL_HIGH_B, (FN-1) // occ_B > FN-1 -> B-ring saturated -> B-feed over-serving +.endif + +// lds_put_r: lane-0-of-wave write scalar \ssrc -> LDS[\saddr] (RUNTIME byte offset in a sreg). Mirrors +// the coop lds_put_v idiom but takes a SCALAR address (symmetry with lds_get_r). Used by the claimer's +// Step-4 snapshot write into the runtime parity half of SNAP_BASE. Temps RP_A/RP_D are v11/v14 (<=v15, +// pre-grow safe); s49 is the exec save (matches lds_put). +.macro lds_put_r saddr, ssrc + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lputr_skip\@ + v_mov_b32 v[RP_A], \saddr + v_mov_b32 v[RP_D], \ssrc + ds_store_b32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lputr_skip\@: + s_mov_b32 exec_lo, s49 +.endm + +// conv_dec_floor: floor-guarded ATOMIC decrement of a role slot -- \ok <- 1 iff it decremented \slot_off +// (only when the current value was > 1), else 0 (floor hit; source role must keep >= 1 wave). A +// ds_cmpstore_rtn_b32 CAS loop (re-reads on a lost race), so two same-source converters in one epoch +// (e.g. compute->Afeed and compute->Bfeed both dec NCOMP_SLOT) can never drive the slot below 1. +// Clob: s52 (read value), s53 (new/CAS-return), s65 (exec save); v5/v6/v7 (<=v15, pre-grow safe). +.macro conv_dec_floor slot_off, ok + s_mov_b32 \ok, 0 +.Lcdf_retry\@: + lds_get s52, \slot_off // s52 = current source-slot count + s_cmp_le_u32 s52, 1 + s_cbranch_scc1 .Lcdf_done\@ // <=1 -> at floor, cannot convert away (ok stays 0) + s_sub_u32 s53, s52, 1 // new = old - 1 + s_mov_b32 s65, exec_lo // lane0-only CAS (one attempt per WAVE) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lcdf_restore\@ + v_mov_b32 v5, \slot_off // vaddr = &slot + v_mov_b32 v6, s52 // v6 = expected old (CMP) + v_mov_b32 v7, s53 // v7 = new value (NEW) + ds_cmpstore_rtn_b32 v6, v5, v7, v6 // slot = (slot==old)? new : slot ; v6 <- prior + s_wait_dscnt 0x0 +.Lcdf_restore\@: + s_mov_b32 exec_lo, s65 + v_readfirstlane_b32 s53, v6 // s53 = prior (lane0 CAS result, broadcast) + s_cmp_eq_u32 s53, s52 // success iff prior == expected (we were the swapper) + s_cbranch_scc0 .Lcdf_retry\@ // lost the race -> re-read and retry + s_mov_b32 \ok, 1 +.Lcdf_done\@: +.endm + +// conv_apply: the bail-time role-conversion COMMIT (SPEC 3.4 Approach A). Precondition: s58 = s_win +// (1 iff this wave won the (dir,epoch) ticket). Ordered strictly BEFORE the QUIESCE_CNT bump the +// CALLER emits after this macro (the quiesce counter is the snapshot handshake). +// ORDER: (a) floor-guarded dec of \src_slot -> (b) reserve the VGPR sum-envelope \delta (shrink +// always ok; grow may abort over BUDGET) -> (c) on ok: inc \dst_slot, flip private role reg (s59), +// s_alloc_vgpr \alloc_sz (GROW=NFV feed->compute / SHRINK=32 compute->feed) with SCC-retry -> +// (d) on floor-fail or reserve-abort: cancel, remain current role (undo the source dec if a +// reservation abort happened after the dec). +// PRE-GROW OOR WINDOW (SPEC 4, #1 brick risk): the wave is lean-32 on entry; every LDS/atomic temp +// read before the s_alloc_vgpr GROW is <=v15 (occ_sample/try_gate v5/v6/v7 + v11/v14; conv_dec_floor +// v5/v6/v7; lds_fetch_add v11/v14) and every carried scalar is <=s65. NO >v15 source before GROW. +// Clob: s52,s53,s54 (+ conv_dec_floor / reserve_try scratch); s59 = new role slot id (record). +.macro conv_apply src_slot, dst_slot, delta, alloc_sz + s_cmp_eq_u32 s58, 0 + s_cbranch_scc1 .Lca_skip\@ // lost the ticket -> no conversion this bail + conv_dec_floor \src_slot, s54 // (a) floor-guarded atomic dec of source slot + s_cmp_eq_u32 s54, 0 + s_cbranch_scc1 .Lca_skip\@ // floor-fail (source at 1) -> cancel, remain current role + reserve_try (\delta), s53 // (b) reserve VGPR envelope (grow may abort; shrink ok) + s_cmp_eq_u32 s53, 0 + s_cbranch_scc0 .Lca_commit\@ + lds_fetch_add s52, \src_slot, 1 // (d) reserve aborted: UNDO the source dec, cancel + s_branch .Lca_skip\@ +.Lca_commit\@: + lds_fetch_add s52, \dst_slot, 1 // (c) inc dest slot (unbounded -> plain atomic add) + s_mov_b32 s59, \dst_slot // flip private current-role reg (records new role slot id) + // ---- s_alloc_vgpr resize: THE pre-grow OOR window closes here; all reads above were <=v15 ---- +.Lca_alloc\@: + s_alloc_vgpr \alloc_sz // GROW(NFV) / SHRINK(32); SCC-retry (brick-class rule) + s_cbranch_scc0 .Lca_alloc\@ +.Lca_skip\@: +.endm .endif // ============================================================================================ @@ -622,6 +719,24 @@ occ_kernel: lds_get s44, EPOCH_OFF // ...then bump EPOCH LAST s_add_u32 s44, s44, 1 s_mov_b32 s35, s44 +.if DSWS2_CONV + // ---- Step 4 (SPEC 3.4 decision 1): snapshot the LIVE role mix into the NEXT epoch's parity half of + // SNAP_BASE, and reset QUIESCE_CNT, BEFORE the epoch bump below (which is published LAST). Followers + // and the claimer's own wait-done for THIS super-tile then size their quiesce sentinels from + // parity(newEpoch) -- so the quiesce counter is the snapshot handshake (Step 3 reads it). ---- + s_and_b32 s46, s44, 1 // parity(newEpoch) + s_mul_i32 s46, s46, 12 // parity*12 (3-word half) -- matches SNAP_BASE comment + s_add_u32 s46, s46, SNAP_BASE // s46 = byte offset of parity half + lds_get s47, NCOMP_SLOT + lds_put_r s46, s47 // snap.nC = live NCOMP_SLOT + s_add_u32 s46, s46, 4 + lds_get s47, NAFEED_SLOT + lds_put_r s46, s47 // snap.nA = live NAFEED_SLOT + s_add_u32 s46, s46, 4 + lds_get s47, NBFEED_SLOT + lds_put_r s46, s47 // snap.nB = live NBFEED_SLOT + lds_put QUIESCE_CNT_OFF, 0 // reset the per-super-tile bail counter +.endif lds_put EPOCH_OFF, s44 BSTAGE // claimer helps stage B for this super-tile (s30,s31) // A7 advance gate: free resident A/B only when ALL G rowblks are computed+flushed @@ -662,6 +777,65 @@ occ_kernel: // decode/resident state (round-table finding #1). Sentinels = threshold + #role-waves (each does // exactly one terminal bail). NOTE: compile-time NCOMP/NAFEED/NBFEED is correct for STATIC roles; // Phase-B conversion must switch these to live role counts / epoch-snapshot drained counters. +.if DSWS2_CONV + // ---- Step 3 (SPEC 3.4 decision 1): size the three claim-counter sentinels from THIS epoch's parity + // snapshot (live role mix written at broadcast, Step 4) instead of compile-time NCOMP/NAFEED/NBFEED, + // so conversions re-tune the quiesce thresholds. A SEPARATE QUIESCE_CNT >= WAVES-1 cross-check is the + // DIAG safety net; under DIAG a mismatch between the two is flagged to occ[29] (byte 116). ---- + lds_get s45, EPOCH_OFF + s_and_b32 s45, s45, 1 // parity(thisEpoch) + s_mul_i32 s45, s45, 12 // parity*12 (3-word half) + s_add_u32 s45, s45, SNAP_BASE + lds_get_r s46, s45 // snap.nC + s_add_u32 s45, s45, 4 + lds_get_r s47, s45 // snap.nA + s_add_u32 s45, s45, 4 + lds_get_r s48, s45 // snap.nB + s_add_u32 s46, s46, G // thr_rowblk = G + snap.nC + s_add_u32 s47, s47, G // thr_arow = G + snap.nA + s_add_u32 s48, s48, FN // thr_bfrag = FN + snap.nB + // sentinels_done (s50) = (ROWBLK_NEXT>=thr_rowblk) & (BFRAG_NEXT>=thr_bfrag) & (AROW_NEXT>=thr_arow) + s_mov_b32 s50, 1 + lds_get s44, ROWBLK_NEXT_OFF + s_cmp_lt_u32 s44, s46 + s_cbranch_scc0 .Lqc_rb_ok + s_mov_b32 s50, 0 +.Lqc_rb_ok: + lds_get s44, BFRAG_NEXT_OFF + s_cmp_lt_u32 s44, s48 + s_cbranch_scc0 .Lqc_bf_ok + s_mov_b32 s50, 0 +.Lqc_bf_ok: + lds_get s44, AROW_NEXT_OFF + s_cmp_lt_u32 s44, s47 + s_cbranch_scc0 .Lqc_ar_ok + s_mov_b32 s50, 0 +.Lqc_ar_ok: + // quiesce_done (s51) = QUIESCE_CNT >= WAVES-1 (each of the WAVES-1 non-claimer waves bumped once) + s_mov_b32 s51, 1 + lds_get s44, QUIESCE_CNT_OFF + s_cmp_lt_u32 s44, (WAVES-1) + s_cbranch_scc0 .Lqc_q_ok + s_mov_b32 s51, 0 +.Lqc_q_ok: +.if DIAG + // DIAG cross-check: the snapshot sentinels and the QUIESCE handshake must AGREE. Publish their XOR + // (1 = mismatch) to occ[29] (byte 116; clear of occ[26]/27=104/108 Task3, occ[28]=112 Task4). + s_xor_b32 s52, s50, s51 + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lqc_diag_skip + v_mov_b32 v14, s52 // v14 <= v15: pre-grow safe (claimer is lean-32 anyway) + global_store_b32 v4, v14, s[0:1] offset:116 scope:SCOPE_DEV // occ[29] = quiesce/snapshot mismatch +.Lqc_diag_skip: + s_mov_b32 exec_lo, s16 +.endif + // advance only when BOTH the snapshot sentinels AND the QUIESCE cross-check agree they are done + s_and_b32 s50, s50, s51 + s_cmp_eq_u32 s50, 0 + s_cbranch_scc1 .Lclaimer_wait_done +.else lds_get s44, ROWBLK_NEXT_OFF // G claims + NCOMP terminal bails s_cmp_lt_u32 s44, (G + NCOMP) s_cbranch_scc1 .Lclaimer_wait_done @@ -671,6 +845,7 @@ occ_kernel: lds_get s44, AROW_NEXT_OFF // G claims + NAFEED terminal bails s_cmp_lt_u32 s44, (G + NAFEED) s_cbranch_scc1 .Lclaimer_wait_done +.endif s_branch .Lclaim_loop .Lclaimer_terminal: lds_put STI_OFF, 0xFFFFFFFF // FIX 1(e): publish SENTINEL (not the raw over-claimed sti)... @@ -714,6 +889,21 @@ occ_kernel: s_cbranch_scc1 .Lretire DECODE_STI // s30=tcol s31=ksi (mblk unused) BSTAGE +.if DSWS2_CONV + // ==== Phase-B decision (Step 1) + bail-time commit (Step 2), B-feed wave ==== + // Lean-32 -> PRE-GROW window (the feed->compute GROW closes it inside conv_apply, post all <=v15 + // reads). If the B-ring is OVER-SERVED (occ_B>CTRL_HIGH_B) grow one B-feed->compute (dir 3): + // reserve delta +(NFV-VLEAN) (may abort over BUDGET -> stay B-feed). + s_mov_b32 s58, 0 + occ_sample s55, s56 // s55=occ_A, s56=occ_B + s_cmp_gt_u32 s56, CTRL_HIGH_B + s_cbranch_scc0 .Lbfeed_quiesce + s_mov_b32 s57, 3 // dir 3: B-feed -> compute + try_gate 3, s58 + conv_apply NBFEED_SLOT, NCOMP_SLOT, +(NFV-VLEAN), NFV +.Lbfeed_quiesce: + lds_fetch_add s61, QUIESCE_CNT_OFF, 1 // commit-before-bump ordering (SPEC 3.4 decision 2) +.endif s_branch .Lbfeed_follow // ============================================================================================ @@ -742,6 +932,22 @@ occ_kernel: s_cbranch_scc1 .Lretire DECODE_STI // s19=mblk s31=ksi (tcol unused) ASTAGE +.if DSWS2_CONV + // ==== Phase-B decision (Step 1) + bail-time commit (Step 2), A-feed wave ==== + // Wave is lean-32 here (feeds never grow), so this is a PRE-GROW window; the conversion GROW + // (feed->compute, s_alloc_vgpr NFV) is the ONLY grow and closes the window inside conv_apply, after + // all <=v15 LDS/atomic reads. If the A-ring is OVER-SERVED (occ_A>CTRL_HIGH_A) grow one A-feed-> + // compute (dir 2): reserve delta +(NFV-VLEAN) (may abort over BUDGET -> stay A-feed). + s_mov_b32 s58, 0 + occ_sample s55, s56 // s55=occ_A, s56=occ_B + s_cmp_gt_u32 s55, CTRL_HIGH_A + s_cbranch_scc0 .Lafeed_quiesce + s_mov_b32 s57, 2 // dir 2: A-feed -> compute + try_gate 2, s58 + conv_apply NAFEED_SLOT, NCOMP_SLOT, +(NFV-VLEAN), NFV +.Lafeed_quiesce: + lds_fetch_add s61, QUIESCE_CNT_OFF, 1 // commit-before-bump ordering (SPEC 3.4 decision 2) +.endif s_branch .Lafeed_follow // ============================================================================================ @@ -861,6 +1067,33 @@ occ_kernel: lds_inc ROWBLK_DONE_OFF // rowblk r computed + flushed (frees the A7 advance gate) s_branch .Lcompute_claim .Lcompute_drained: +.if DSWS2_CONV + // ==== Phase-B role-boundary decision (Step 1) + bail-time commit (Step 2), compute wave ==== + // Wave is lean-32 here (the last claimed rowblk shrank to 32; a zero-claim drain never grew), so the + // whole decision+commit runs in the PRE-GROW window -- every temp <=v15 / scalar <=s65 (see conv_apply + // OOR note). If compute is STARVED for A (occ_AA-feed (dir 0); + // else if starved for B, compute->B-feed (dir 1). Both are SHRINKs: reserve delta -(NFV-VLEAN), + // s_alloc_vgpr 32 (already lean -> no-op). Persistent: s57=dir, s58=s_win, s59=new-role (all outside + // s60..s65 so occ_sample/try_gate/reserve_try/conv_* cannot clobber them while live). + s_mov_b32 s58, 0 // s_win = 0 (default: raced no ticket) + occ_sample s55, s56 // s55=occ_A in [0,G], s56=occ_B in [0,FN] (clob s60,s61) + s_cmp_lt_u32 s55, CTRL_LOW + s_cbranch_scc0 .Lcmp_try_b + s_mov_b32 s57, 0 // dir 0: compute -> A-feed + try_gate 0, s58 // s58 = s_win (single-winner per (dir,epoch)) + conv_apply NCOMP_SLOT, NAFEED_SLOT, -(NFV-VLEAN), 32 + s_branch .Lcmp_quiesce +.Lcmp_try_b: + s_cmp_lt_u32 s56, CTRL_LOW + s_cbranch_scc0 .Lcmp_quiesce + s_mov_b32 s57, 1 // dir 1: compute -> B-feed + try_gate 1, s58 + conv_apply NCOMP_SLOT, NBFEED_SLOT, -(NFV-VLEAN), 32 +.Lcmp_quiesce: + // ORDERING (SPEC 3.4 decision 2): the commit above fully completed (role-slot CAS + reservation + + // s_alloc_vgpr) BEFORE this QUIESCE_CNT bump -- the bump is the snapshot handshake the claimer reads. + lds_fetch_add s61, QUIESCE_CNT_OFF, 1 // exactly one bump per non-claimer wave / super-tile +.endif s_branch .Lcompute_follow // this super-tile's compute drained -> re-check epoch/terminal // ---- A7 role-agnostic terminal (followers): retire. (Claimer retires via .Lclaimer_terminal.) ---- diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/rga_check.sh b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/rga_check.sh index cdb81ed0ce19..585b2a152d51 100755 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/rga_check.sh +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/rga_check.sh @@ -14,7 +14,7 @@ set -euo pipefail ROCM=/opt/rocm L="$ROCM/llvm/bin" RGA=/home/kmbandy/Downloads/rdts/RadeonDeveloperToolSuite-2026-05-28-1806/rga -KSRC=occ_kernel_wggemm2.s +KSRC="${KSRC:-occ_kernel_wggemm2.s}" # override to gate a different kernel, e.g. KSRC=occ_kernel_coop.s LABEL="${1:?usage: rga_check.sh NBFEED= DSWS2_CONV=0` +- **Run (per run):** `ML8_POOL=16 ML8_COOP_CHUNK=0 DSWS2_SEGK=64 DSWS2_NKSEG=32 DSWS2_ORACLE_MTL=3 + DSWS2_ORACLE_NTL=8 ./occ_dispatch --dsws2 ` — **single pass** (no `DSWS2_TARGET_SECS`). +- **Shape:** `576×512×2048`, super-tile `192×64` (G=6 FM=2 FN=4), `n_kseg=32`, `TOTAL_super=768`, + `POOL=16` WGs. (M=576 = the proven-brick-safe regime; the distribution is shape-robust.) +- Logs on real disk: `~/dsws_gpu_logs/phase_ab_flow_gd.bin +# OFFLINE/CPU only. Usage: ./build_flow.sh [NCOMP NAFEED NBFEED] +# Env: POOL_N=3 PHASEPROBE={0|1} NOCFLUSH={0|1} CSTORE={0|1} SLEEPN=N COORD_PERIOD=N DIAG=0 +set -e +cd "$(dirname "$0")" +L=/opt/rocm/llvm/bin +fail=0 +mkflow() { # EMERGENT economy: no mix args. Env: WAVES VBUDGET G SEGK POOL_N ACC_N ... + local tag="occ_dsws2_w${WAVES:-16}_flow_gd" + nice -19 ionice -c3 "$L/clang" -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx1201 \ + -Wa,-defsym,DSWS2=1 -Wa,-defsym,FM=2 -Wa,-defsym,FN=4 -Wa,-defsym,G=${G:-6} -Wa,-defsym,SEGK=${SEGK:-64} \ + -Wa,-defsym,SAFEPROBE=1 -Wa,-defsym,DIAG=${DIAG:-0} -Wa,-defsym,POOL_N=${POOL_N:-3} -Wa,-defsym,ACC_N=${ACC_N:-1} \ + -Wa,-defsym,WAVES=${WAVES:-16} -Wa,-defsym,VBUDGET=${VBUDGET:-1536} \ + -Wa,-defsym,PHASEPROBE=${PHASEPROBE:-0} -Wa,-defsym,NOCFLUSH=${NOCFLUSH:-0} -Wa,-defsym,CSTORE=${CSTORE:-0} \ + -Wa,-defsym,SLEEPN=${SLEEPN:-2} -Wa,-defsym,COORD_PERIOD=${COORD_PERIOD:-64} -Wa,-defsym,TFPROBE=${TFPROBE:-0} -Wa,-defsym,DEADMAN=${DEADMAN:-1} -Wa,-defsym,DEADMAN_TICKS=${DEADMAN_TICKS:-50000000} -Wa,-defsym,STAGINSTR=${STAGINSTR:-0} -Wa,-defsym,TRACE=${TRACE:-0} \ + -c occ_kernel_dsws_flow.s -o "$tag.o" 2>/tmp/flow_build.err \ + && { "$L/llvm-objcopy" -O binary --only-section=.text "$tag.o" "$tag.bin"; \ + echo " OK $tag.bin ($(wc -c < "$tag.bin")B .text) [POOL_N=${POOL_N:-3} PHASEPROBE=${PHASEPROBE:-0}]"; } \ + || { echo " FAIL $tag"; sed -n '1,25p' /tmp/flow_build.err; fail=1; } +} +echo "== flow bin (occ_kernel_dsws_flow.s; EMERGENT mix; WAVES=${WAVES:-16} G=${G:-6} SEGK=${SEGK:-64} POOL_N=${POOL_N:-3} VBUDGET=${VBUDGET:-1536}) ==" +mkflow +echo "flow build done. fail=$fail" +exit $fail diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_ring.sh b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_ring.sh new file mode 100755 index 000000000000..bfba9eced325 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_ring.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# build_ring.sh — FIX 1a D=2 double-buffered ring-of-slots bin (occ_kernel_dsws_ring.s). +# Bin name MUST match occ_dispatch.cpp DSWS2_RING path: occ_dsws2_cab_ring_gd.bin +# OFFLINE/CPU only — assemble + objcopy, NO GPU. Usage: ./build_ring.sh [NCOMP NAFEED NBFEED] +# Env: PHASEPROBE={0|1} NOCFLUSH={0|1} CSTORE={0|1} SLEEPN=N DIAG=0 +set -e +cd "$(dirname "$0")" +L=/opt/rocm/llvm/bin +fail=0 +mkring() { # $1=NCOMP $2=NAFEED $3=NBFEED + local tag="occ_dsws2_${1}c${2}a${3}b_ring_gd" + nice -19 ionice -c3 "$L/clang" -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx1201 \ + -Wa,-defsym,DSWS2=1 -Wa,-defsym,FM=2 -Wa,-defsym,FN=4 -Wa,-defsym,G=6 -Wa,-defsym,SEGK=64 \ + -Wa,-defsym,SAFEPROBE=1 -Wa,-defsym,DIAG=${DIAG:-0} \ + -Wa,-defsym,NCOMP=$1 -Wa,-defsym,NAFEED=$2 -Wa,-defsym,NBFEED=$3 \ + -Wa,-defsym,PHASEPROBE=${PHASEPROBE:-0} -Wa,-defsym,NOCFLUSH=${NOCFLUSH:-0} -Wa,-defsym,CSTORE=${CSTORE:-0} \ + -Wa,-defsym,SLEEPN=${SLEEPN:-2} -Wa,-defsym,TFPROBE=${TFPROBE:-0} \ + -c occ_kernel_dsws_ring.s -o "$tag.o" 2>/tmp/ring_build.err \ + && { "$L/llvm-objcopy" -O binary --only-section=.text "$tag.o" "$tag.bin"; \ + echo " OK $tag.bin ($(wc -c < "$tag.bin")B .text) [PHASEPROBE=${PHASEPROBE:-0} NOCFLUSH=${NOCFLUSH:-0}]"; } \ + || { echo " FAIL $tag"; sed -n '1,20p' /tmp/ring_build.err; fail=1; } +} +c=${1:-4}; a=${2:-2}; b=${3:-2} +echo "== FIX 1a ring bin (occ_kernel_dsws_ring.s; G=6 SEGK=64 FM=2 FN=4 D=2) ==" +mkring "$c" "$a" "$b" +echo "ring build done. fail=$fail" +exit $fail diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/fp8_oracle.cpp b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/fp8_oracle.cpp index aeadac94ca5d..6abea4b85fa7 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/fp8_oracle.cpp +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/fp8_oracle.cpp @@ -29,3 +29,42 @@ void wmma_ref_16x16x16(const uint8_t* A, const uint8_t* B, const float* C, float D[i * 16 + j] = acc; } } + +#include +// Tiered oracle comparison. bad = #elements exceeding rel*|ref|+abs_; max_rel = worst |got-ref|/|ref|. +OracleCmp oracle_compare(const float* got, const float* ref, long n, float rel, float abs_) { + OracleCmp r{true, 0, 0.0}; + for (long i = 0; i < n; ++i) { + float d = std::fabs(got[i] - ref[i]); + float thr = rel * std::fabs(ref[i]) + abs_; + double rl = (double)d / ((double)std::fabs(ref[i]) + 1e-30); + if (rl > r.max_rel) r.max_rel = rl; + if (d > thr) { r.ok = false; ++r.bad; } + } + return r; +} + +#ifdef ORACLE_SELFTEST +#include +#include +#include +int main() { + const long n = 256; + // ref ~ O(100) so the abs term doesn't dominate the rel term in the test. + std::vector ref(n), id(n), p01(n), p1(n), p5(n); + for (long i = 0; i < n; ++i) { + float v = 100.0f + 50.0f * (float)(i % 7); + ref[i] = v; id[i] = v; + p01[i] = v * 1.001f; // 0.1% + p1[i] = v * 1.01f; // 1% + p5[i] = v * 1.05f; // 5% + } + assert( oracle_compare(id.data(), ref.data(), n, 5e-3f, 1e-2f).ok); // identical -> tight ok + assert( oracle_compare(p01.data(), ref.data(), n, 5e-3f, 1e-2f).ok); // 0.1% -> tight ok + assert(!oracle_compare(p1.data(), ref.data(), n, 5e-3f, 1e-2f).ok); // 1% -> tight REJECTS + assert( oracle_compare(p1.data(), ref.data(), n, 3e-2f, 2e-2f).ok); // 1% -> loose ok + assert(!oracle_compare(p5.data(), ref.data(), n, 3e-2f, 2e-2f).ok); // 5% -> loose REJECTS + printf("ORACLE_SELFTEST all pass\n"); + return 0; +} +#endif diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/fp8_oracle.h b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/fp8_oracle.h index 826006d4c892..fd2c25b99cb2 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/fp8_oracle.h +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/fp8_oracle.h @@ -5,3 +5,9 @@ float fp8_e4m3_to_float(uint8_t b); // Reference D = A*B + C. A,B are 16x16 row-major e4m3 bytes; C,D are 16x16 row-major f32. void wmma_ref_16x16x16(const uint8_t* A, const uint8_t* B, const float* C, float* D); + +// Tiered oracle comparison (DSWS v2). Generalizes the inline gate fabs(got-ref) > rel*fabs(ref)+abs_. +// Tier 1 (n_kseg==1): TIGHT = {rel 5e-3, abs 1e-2} (the proven gate). +// Tier 2 (n_kseg>1): LOOSE = {rel 3e-2, abs 2e-2} (absorbs split-K reassociation). +struct OracleCmp { bool ok; long bad; double max_rel; }; +OracleCmp oracle_compare(const float* got, const float* ref, long n, float rel, float abs_); diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_dispatch.cpp b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_dispatch.cpp index bdc44865b9f9..ece9f3fa2ef2 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_dispatch.cpp +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_dispatch.cpp @@ -1321,9 +1321,13 @@ struct CoopResult { bool ok=false; uint32_t maxlive=0, total=0; uint64_t okFrags static CoopResult run_mbcoop(uint32_t node, const char* isaPath, bool dynvgpr, uint32_t pool, int M, int N, int K, int FM, int FN, int P, int RINGD, bool fullCheck, bool useGenDiv=false, - uint32_t reps=1, double targetSecs=0.0) { + uint32_t reps=1, double targetSecs=0.0, + int totalWaves=0, uint32_t ldsBytesOverride=0) { + // DSWS (MAD-305): totalWaves>0 launches N=NCOMP+NAFEED+NBFEED waves/WG (P=NCOMP for the C-store/oracle + // partition); ldsBytesOverride carries the larger DSWS LDS_TOTAL_DSWS. Both default to the proven 2-role + // coop behavior (1+P waves, coop LDS) so every existing caller is byte-identical. CoopResult res; - const int WAVES_LAUNCH = 1 + P; // 1 feed/claim wave + P compute waves + const int WAVES_LAUNCH = totalWaves > 0 ? totalWaves : (1 + P); // 1 feed + P compute (coop) | N (DSWS) const int TM = P*FM*16, TN = FN*16; // WG tile: P M-bands (each FM rows-of-16) x shared FN N-cols int MTL = M / TM, NTL = N / TN, NT = N / 16, KT = K / 16; uint32_t TOTAL = (uint32_t)MTL * NTL; @@ -1411,7 +1415,8 @@ static CoopResult run_mbcoop(uint32_t node, const char* isaPath, bool dynvgpr, u uint32_t vgprField = dynvgpr ? 4u : ((fatregs / 8) & 0x3fu); uint32_t rsrc1 = BuildPgmRsrc1(dynvgpr); rsrc1 = (rsrc1 & ~0x3fu) | (vgprField & 0x3fu); // LDS (Step 3 byte layout): B_ring[RINGD*FN*256] + prod_count(u32) + cons_count[P](u32) + tile_slot[3](u32) - uint32_t ldsBytes = (uint32_t)(RINGD * FN * 256 + 4 * (1 + P + 3)); + uint32_t ldsBytes = ldsBytesOverride > 0 ? ldsBytesOverride // DSWS: the full LDS_TOTAL_DSWS + : (uint32_t)(RINGD * FN * 256 + 4 * (1 + P + 3)); // coop: B-ring + prod/cons/ti/epoch/initflag ldsBytes = (ldsBytes + 0x1FFu) & ~0x1FFu; // round to 512B LDS granule uint32_t ldsU=0,ldsA=0,ldsG=0; uint32_t ldsBits = ldsRsrc2Bits(ldsBytes, &ldsU, &ldsA, &ldsG); uint32_t rsrc2 = (BuildPgmRsrc2(dynvgpr) & ~0x3eu) | (15u << RSRC2_USER_SGPR_SHIFT) | ldsBits; @@ -1458,7 +1463,7 @@ static CoopResult run_mbcoop(uint32_t node, const char* isaPath, bool dynvgpr, u uint32_t chunkHi = (base + chunkTiles < TOTAL) ? (base + chunkTiles) : TOTAL; userdata[11] = chunkHi; // s11 = this chunk's terminal tile (feed exits at claim>=hi; POOLTERM compute follows) occW[0]=0; occW[1]=0; occW[2]=0xFFFFFFFFu; occW[3]=0; occW[4]=0; occW[5]=base; *fenceW=0; occW[6]=occW[7]=occW[8]=occW[9]=occW[10]=occW[11]=occW[12]=occW[13]=0; - for (int qi=14; qi<28; ++qi) occW[qi]=0; // DIAGFINE markers: 14-21 hot-step, 22 raw-ti, 23-27 init-window + for (int qi=14; qi<52; ++qi) occW[qi]=0; // DIAGFINE: 14-21 hot-step, 22 raw-ti, 23-27 init, 28-31 A-feed, 32-36 sensors, 39 gate would-win count, 40-47 DSWS2_BAILMARK per-wave bail epochs, 48 conversion-commit count RingPlace(ring, PM4AcquireMemoryPacket(FAMILY_GFX12)); RingPlace(ring, PM4SetShaderRegPacket(mmCOMPUTE_START_X, dims, 8)); RingPlace(ring, PM4SetShaderRegPacket(mmCOMPUTE_PGM_LO, pgm, 6)); @@ -1484,10 +1489,12 @@ static CoopResult run_mbcoop(uint32_t node, const char* isaPath, bool dynvgpr, u while (true) { double now = now_s(); if (streamOn && (now - lastSnap) >= 0.2) { lastSnap = now; fprintf(stderr, "[occ +%5.2fs] live%u maxlive%u claim%u end%u | INIT adm%u tmr%u lds%u flag%u rdv%u | " - "feedPh%u compPh%u cons%u tiles%u | feed:tr%u pub%u comp:dsB%u wm%u rawTi%u | fence=%s\n", + "feedPh%u compPh%u cons%u tiles%u | feed:tr%u pub%u comp:dsB%u wm%u rawTi%u | SENS occ_b%u occ_a%u roles[%u/%u/%u] | BAIL[w1=%u w2=%u w3=%u w4=%u w5=%u w6=%u w7=%u] | fence=%s\n", now-t0, occW[0],occW[1],occW[5],occW[3], occW[23],occW[24],occW[25],occW[26],occW[27], occW[6],occW[7],occW[10],occW[11], occW[19],occW[21],occW[15],occW[17], occW[22], + occW[32],occW[33],occW[34],occW[35],occW[36], + occW[41],occW[42],occW[43],occW[44],occW[45],occW[46],occW[47], // DSWS2_BAILMARK per-wave bail epochs (wid1..7; wid0=claimer=occ[40]) (*fenceW==FENCE_VALUE)?"FIRED":"--"); fflush(stderr); } if (occW[1] > 0) admitted = true; @@ -1510,6 +1517,14 @@ static CoopResult run_mbcoop(uint32_t node, const char* isaPath, bool dynvgpr, u occW[18], occW[19], occW[20], occW[21]); fprintf(stderr, " DIAGFINE compute: prodwait[14]=%u dsloadB[15]=%u Aload[16]=%u wmma[17]=%u consRel[10]=%u rawTiMax[22]=%u\n", occW[14], occW[15], occW[16], occW[17], occW[10], occW[22]); + fprintf(stderr, " DIAGFINE Afeed : reached[28]=%u Aload[29]=%u dsstore[30]=%u publish[31]=%u | Bphase(occ6)=%u Bti(occ8)=%u\n", + occW[28], occW[29], occW[30], occW[31], occW[6], occW[8]); + fprintf(stderr, " DIAGINIT : adm[23]=%u tmr[24]=%u ldsinit[25]=%u initflag[26]=%u rdv[27]=%u\n", + occW[23], occW[24], occW[25], occW[26], occW[27]); + fprintf(stderr, " DSWS sensors : occ_b[32]=%u occ_a[33]=%u nComp[34]=%u nAfeed[35]=%u nBfeed[36]=%u gateWin[39]=%u\n", + occW[32], occW[33], occW[34], occW[35], occW[36], occW[39]); + fprintf(stderr, " DSWS2 BAILMARK : per-wave last-bailed epoch w1=%u w2=%u w3=%u w4=%u w5=%u w6=%u w7=%u (all==hung epoch => visibility; ONE stale => that wave is the STRAGGLER)\n", + occW[41], occW[42], occW[43], occW[44], occW[45], occW[46], occW[47]); repFail=true; break; } uint32_t gs=occW[2], ge=occW[3]; @@ -1546,6 +1561,15 @@ static CoopResult run_mbcoop(uint32_t node, const char* isaPath, bool dynvgpr, u while (*fenceW != FENCE_VALUE && (now_s() - tw) < 5.0) nanosleep(&ts, nullptr); } bool queueIdle = (*fenceW == FENCE_VALUE); if (!queueIdle) fprintf(stderr, " [teardown] WARN: EOP fence never fired in 5s; queue NON-IDLE -> NOT destroying (process-exit reclaims). Brick-avoidance.\n"); + // DSWS Phase-2 sensor readback (always print on clean exit -- a fast oracle finishes before the 200ms + // stream OR the timeout dump ever fires, so this is the guaranteed observation of the LAST chunk's + // final sensor sample). occ_b/occ_a must sit in [0,RINGD] (not pinned 0/RINGD); roles must = launch mix. + if (totalWaves > 0) { + fprintf(stderr, " [dsws sensors @clean-exit] occ_b=%u occ_a=%u roles[nComp=%u nAfeed=%u nBfeed=%u] gateWin(c->Afeed)[39]=%u (last chunk)\n", + occW[32], occW[33], occW[34], occW[35], occW[36], occW[39]); + fprintf(stderr, " [dsws CONVERSIONS] committed role-switches this run = %u (occ[48]; >0 => waves ADAPTIVELY switched role)\n", + occW[48]); + } res.ok = true; res.maxlive = lastMaxlive; res.total = lastTotal; res.wall = spanSum / (res.repsDone ? res.repsDone : 1); // mean per-rep span res.wallSum = spanSum; res.wallMin = spanMin; res.wallMax = spanMax; @@ -1611,6 +1635,652 @@ static CoopResult run_mbcoop(uint32_t node, const char* isaPath, bool dynvgpr, u return res; } +// --------------------------------------------------------------------------- +// MAD-305 DSWS v2 substrate launch (Task A8, PLAN_DSWS_SUBSTRATE_V2.md). Dispatches occ_kernel_dsws.s: a +// pinned wid-0 claimer broadcasts super-tiles from a pool; B-feed/A-feed/compute waves drain LDS atomic +// claim counters against resident-in-LDS A/B for the current super-tile; compute flushes fp32 partials via +// global_atomic_add_f32 (split-K segments accumulate into the SAME C cell, ksi-independent address). +// +// This MIRRORS run_mbcoop's PM4 launch/chunk/teardown/canary infrastructure byte-for-byte where the +// protocols line up (buffer alloc, VRAM guard, address bounds gate, compositor-safe chunking, fence/settle +// poll, guarded teardown, C-guard-tail canary) -- re-keyed to the v2 contract: +// * occ[20] (=occW[5], byte offset 20) is the SAME global claim counter coop uses for output-tile claims; +// v2 claims super-tile ids (`sti`) through it instead. +// * The v2 C address formula (ti*(G*FM*FN*1024) + r*(FM*FN*1024) + frag*1024 + lane*32 + e*4) is coop's +// formula with P (compute-wave count) replaced by G (super-tile M-extent) and cid replaced by r +// (claimed rowblk) -- so the oracle reuses unpack_D/oracle_compare exactly, decoding r where coop +// decoded cid. +// +// *** FIX 1 (round-table Opus+Codex pass) *** +// The original v2 kernarg contract called for COMPUTE_PGM_RSRC2.USER_SGPR=18 (s0..s17 hardware-preloaded) +// so n_kseg/TOTAL_super/magic_kseg could ride in as s15/s16/s17, written via a SECOND SET_SH_REG packet +// at register (COMPUTE_USER_DATA_0 + 16). That was never deliverable: every OTHER kernel in this harness +// uses <=15 user SGPRs, and the project's own pinned PM4 register reference (dvgpr_pm4/ref/gfx_7_2_d.h) +// defines COMPUTE_USER_DATA_0..15 ONLY (16 registers, covering s0..s15) -- there is no register defined +// for s16/s17 anywhere in this raw-PM4 path, and RESULT_WGGEMM.md's "raw-PM4 TGID is unavailable" probe +// already found SGPR delivery beyond s15 under raw-PM4 (CP-direct dispatch, MES bypassed) reads constant +// garbage, not a controllable value. On top of that, the per-chunk override of kernarg slot s16 (the +// chunk terminal bound) silently collided with s16 also being the would-be TOTAL_super kernarg -- two +// unrelated meanings on the same undeliverable slot. +// FIX 1 drops s15/s16/s17 from the kernarg contract entirely (USER_SGPR=15, s0..s14 only, ONE SET_SH_REG +// packet of 16 registers like every other proven path here, index 15 unused/padding). n_kseg is now +// DERIVED in-kernel from KT (s8); the chunk terminal bound is now MEMORY-CARRIED via occ[24] (occW[6], +// written once per chunk below) instead of riding in an undeliverable kernarg slot. See the KERNARG +// CONTRACT block at the top of occ_kernel_dsws.s for the full new scheme. +// --------------------------------------------------------------------------- +struct Dsws2Result { + bool ok = false; // true iff the run completed cleanly (fence fired, occ[0]==0) AND badFrags==0 + uint64_t okFrags = 0, badFrags = 0; + double maxRel = 0.0; + uint32_t occ0 = 0; // live-counter readback at last clean completion (expect 0) + uint32_t occClaim = 0; // occ[20] readback at last clean completion (global claim counter) + double tf = 0.0; // TFPROBE throughput (2*M*N*K / summed GPU-tick span); 0 if the bin has no tick capture + uint64_t wall = 0; // summed per-chunk GPU-tick span (occ[3]-occ[2], device busy ticks, excl host gaps) +}; + +static Dsws2Result run_dsws2(uint32_t node, const char* isaPath, + uint32_t nComp, uint32_t nAfeed, uint32_t nBfeed, + int Gv, int SEGKv, int FMc, int FNc, + int Mo, int No, int Ko, + float orel, float oabs, double freq_hz) { + Dsws2Result res; + const uint32_t WAVES_LAUNCH = nComp + nAfeed + nBfeed; + const int TMsuper = Gv*16*FMc, TN = FNc*16; // super-tile M rows, N-panel cols + if (TMsuper == 0 || TN == 0 || SEGKv <= 0 || Ko <= 0 || (Mo % TMsuper) || (No % TN) || (Ko % SEGKv)) { + fprintf(stderr, " [dsws2] geometry %dx%dx%d does not divide cleanly (G=%d SEGK=%d FM=%d FN=%d)\n", + Mo, No, Ko, Gv, SEGKv, FMc, FNc); + return res; + } + const int MTLsuper = Mo / TMsuper, NTL = No / TN, NT = No / 16, KT = Ko / 16; + const int KSEG_STEPS = SEGKv / 16; + const int n_kseg = Ko / SEGKv; + if (MTLsuper == 0 || NTL == 0 || n_kseg == 0 || KSEG_STEPS == 0) { + fprintf(stderr, " [dsws2] degenerate geometry (MTLsuper=%d NTL=%d n_kseg=%d KSEG_STEPS=%d)\n", + MTLsuper, NTL, n_kseg, KSEG_STEPS); + return res; + } + // FIX 1(k): the kernel now DERIVES n_kseg in-kernel as KT >> NKSEG_SHIFT (a plain shift) and uses it + // as a shift/mask pair (shift=ff1(n_kseg), mask=n_kseg-1) to split sti -> (t,ksi) -- both of those + // are only correct when n_kseg is a power of two (ff1 finds "the" single set bit; a non-power-of-two + // mask drops bits). Refuse rather than silently mis-decode every super-tile. + if ((n_kseg & (n_kseg - 1)) != 0) { + fprintf(stderr, " [dsws2] *** REFUSE: n_kseg=%d is not a power of two (required: the kernel derives " + "shift/mask from n_kseg via s_ff1_i32_b32) ***\n", n_kseg); + return res; + } + const uint32_t TOTAL = (uint32_t)MTLsuper * (uint32_t)NTL; // coop-compat output-tile count (C sizing) + const uint64_t TOTAL_super = (uint64_t)MTLsuper * (uint64_t)NTL * (uint64_t)n_kseg; // super-tile pool size + const uint32_t magic = (uint32_t)((0x100000000ULL + (uint64_t)NTL - 1) / (uint64_t)NTL); // ceil(2^32/NTL) + const uint32_t magicTotal = (uint32_t)((0x100000000ULL + (uint64_t)TOTAL - 1) / (uint64_t)TOTAL); // ceil(2^32/TOTAL) for KMAJOR ksi=sti/TOTAL + + static const uint8_t NICE[6] = {0x38,0x40,0x30,0xB8,0xC0,0xB0}; + std::vector Ah((size_t)Mo*Ko), Bh((size_t)Ko*No), Bshufh((size_t)Ko*No); + for (size_t i = 0; i < Ah.size(); ++i) Ah[i] = NICE[(i*7 + i/(size_t)Ko) % 6]; + for (size_t i = 0; i < Bh.size(); ++i) Bh[i] = NICE[(i*5 + (i/(size_t)No)*3) % 6]; + mbg_preshuffle_B(Bh.data(), Bshufh.data(), Ko, No); + + size_t isaLen = 0; uint8_t* isaBytes = ReadFile(isaPath, &isaLen); + if (!isaBytes) { fprintf(stderr, " [dsws2] cannot read kernel bin '%s'\n", isaPath); return res; } + GpuBuf isa = AllocGpu(node, (isaLen + 0xFFF) & ~0xFFFull, true, false); + GpuBuf occ = AllocGpu(node, 0x1000, false, true); + // ---- TRACE: per-super-tile time-series buffer (DSWS2_TRACE=1; requires a TRACE=1 kernel bin + single chunk). + // The claimer appends a 16-u32 row per super-tile (indexed by SEGCNT) capturing the live role mix, ring + // occupancy peak, conversions, and vresv. Host reads it back to CSV after the run. ---- + const bool traceOn = getenv("DSWS2_TRACE") != nullptr; + uint32_t traceMaxRows = 0; GpuBuf traceBuf{}; volatile uint32_t* traceW = nullptr; + if (traceOn) { + uint64_t want = TOTAL_super + 64; traceMaxRows = (uint32_t)(want > (1u<<21) ? (1u<<21) : want); + traceBuf = AllocGpu(node, ((uint64_t)traceMaxRows*64 + 0xFFF)&~0xFFFull, false, true); + traceW = (volatile uint32_t*)traceBuf.ptr; + if (!traceBuf.ptr) { fprintf(stderr, " [dsws2 trace] buffer alloc FAILED -> trace disabled\n"); traceW = nullptr; } + else memset((void*)traceW, 0, traceBuf.size); + } + // SAFETY PADDING (mirrors run_mbcoop): a guard tail after each operand so a small dyn off-by-one global + // access lands in mapped VRAM (observable wrong answer) instead of a page-fault brick. + uint64_t padB = (uint64_t)(getenv("ML8_COOP_PAD_MB") ? atoi(getenv("ML8_COOP_PAD_MB")) : 64) * 1024ull * 1024ull; + GpuBuf Ad = AllocGpu(node, ((Ah.size()+0xFFF)&~0xFFFull) + padB, false, true, /*deviceLocal*/true); + GpuBuf Bd = AllocGpu(node, ((Bshufh.size()+0xFFF)&~0xFFFull) + padB, false, true, /*deviceLocal*/true); + uint64_t cbytes = ((uint64_t)TOTAL * (uint64_t)((uint32_t)Gv*FMc*FNc*1024) + 0xFFF) & ~0xFFFull; // TOTAL output tiles x G*FM*FN frags x 256 f32 + GpuBuf C = AllocGpu(node, cbytes + padB, false, true, /*deviceLocal*/true); + GpuBuf fence = AllocGpu(node, 0x1000, false, true); + if (!(Ad.vram && Bd.vram && C.vram)) { + fprintf(stderr, "\n*** DSWS2 VRAM GUARD FAILED (%s): operands not device-local -> PERF/SAFETY INVALID ***\n", isaPath); + abort(); + } + // ---- ADDRESS BOUNDS GATE (MANDATORY, mirrors run_mbcoop's gate). Formulas re-derived from + // occ_kernel_dsws.s's ASTAGE/BSTAGE/.Lcompute address math (G replaces coop's P; r replaces cid). ---- + { + uint64_t Asize = (uint64_t)Ah.size(), Bsize = (uint64_t)Bshufh.size(), Csz = cbytes; + uint64_t rowblkAbsMax = (uint64_t)MTLsuper * (uint64_t)Gv - 1ull; + uint64_t Amax = rowblkAbsMax*(uint64_t)16*FMc*Ko + (uint64_t)(FMc-1)*16*Ko + + (uint64_t)(n_kseg-1)*SEGKv + (uint64_t)(KSEG_STEPS-1)*16 + + (uint64_t)15*Ko + 8 + 7; + uint64_t Bmax = (uint64_t)(NTL-1)*FNc*256 + (uint64_t)(n_kseg-1)*KSEG_STEPS*(uint64_t)NT*256 + + (uint64_t)(FNc-1)*256 + (uint64_t)(KSEG_STEPS-1)*(uint64_t)NT*256 + + (uint64_t)31*8 + 7; + uint64_t Cmax = (uint64_t)(TOTAL-1)*(uint64_t)Gv*FMc*FNc*1024 + (uint64_t)(Gv-1)*(uint64_t)FMc*FNc*1024 + + (uint64_t)(FMc*FNc-1)*1024 + (uint64_t)31*32 + (uint64_t)7*4 + 3; + bool aok = Amax < Asize, bok = Bmax < Bsize, cok = Cmax < Csz; + printf(" [dsws2 bounds] A last=%llu/%llu %s B last=%llu/%llu %s C last=%llu/%llu %s\n", + (unsigned long long)Amax,(unsigned long long)Asize, aok?"OK":"*OOB*", + (unsigned long long)Bmax,(unsigned long long)Bsize, bok?"OK":"*OOB*", + (unsigned long long)Cmax,(unsigned long long)Csz, cok?"OK":"*OOB*"); + if (!(aok && bok && cok)) { + fprintf(stderr, "\n*** DSWS2 ADDRESS BOUNDS GATE FAILED (%s) -> REFUSING to dispatch. ***\n", isaPath); + FreeGpu(fence); FreeGpu(C); FreeGpu(Bd); FreeGpu(Ad); FreeGpu(occ); FreeGpu(isa); + return res; + } + } + memcpy(isa.ptr, isaBytes, isaLen); free(isaBytes); + memcpy(Ad.ptr, Ah.data(), Ah.size()); + memcpy(Bd.ptr, Bshufh.data(), Bshufh.size()); + memset((char*)Ad.ptr + ((Ah.size()+0xFFF)&~0xFFFull), 0, padB); + memset((char*)Bd.ptr + ((Bshufh.size()+0xFFF)&~0xFFFull), 0, padB); + volatile uint32_t* occW = (volatile uint32_t*)occ.ptr; + volatile uint32_t* fenceW = (volatile uint32_t*)fence.ptr; + memset((void*)occW, 0, occ.size); // host zero-init: occ[0] live-count, occ[20] claim-counter, all reserved words + *fenceW = 0; + memset((char*)C.ptr + cbytes, 0, padB); // CANARY: zero the C guard tail (any nonzero after run = an OOB store) + // FIX 2: the kernel's compute role accumulates into C via global_atomic_add_f32 (split-K segments add + // into the SAME C cell) -- it never initializes a cell, so the host MUST zero the C data region before + // ANY dispatch (occ_kernel_dsws.s's KERNARG CONTRACT comment: "HOST MUST MEMSET C=0"). This was + // missing entirely (only the guard-tail canary was zeroed above). ONCE here, before the chunk loop -- + // NOT per chunk, so split-K segments claimed across separate chunk dispatches still accumulate. + memset((char*)C.ptr, 0, cbytes); + + Ring ring; ring.buf = AllocGpu(node, 0x10000, true, true); ring.dw = (uint32_t*)ring.buf.ptr; + ring.sizeDw = (uint32_t)(ring.buf.size / sizeof(uint32_t)); + CHECK(hsaKmtCreateQueue(node, HSA_QUEUE_COMPUTE, 100, HSA_QUEUE_PRIORITY_NORMAL, ring.buf.ptr, ring.buf.size, nullptr, &ring.res)); + + uint64_t shiftedIsa = ((uint64_t)isa.ptr) >> 8; + uint64_t occVa=(uint64_t)occ.ptr, aVa=(uint64_t)Ad.ptr, bVa=(uint64_t)Bd.ptr, cVa=(uint64_t)C.ptr, fenceVa=(uint64_t)fence.ptr; + uint32_t dims[8] = {0,0,0,(uint32_t)(WAVES_LAUNCH*32),1,1,0,0}; // NUM_THREAD_X = WAVES_LAUNCH*32 -> WAVES_LAUNCH waves/WG + uint32_t pgm[6] = {(uint32_t)shiftedIsa,(uint32_t)(shiftedIsa>>32)|(g_is_dgpu?0u:(1u<<8)),0,0,0,0}; + // DYNVGPR is baked DYNVGPR=1 into this bin's compute role (build_dsws.sh mk2 never overrides it) -- dyn-VGPR + // MUST be armed (RSRC2 bit6) to match; there is no static v2 bin to fall back to if it weren't. + const uint32_t vgprField = 4u; // lean 32-VGPR launch; compute waves s_alloc_vgpr-grow per claimed rowblk + uint32_t rsrc1 = BuildPgmRsrc1(true); rsrc1 = (rsrc1 & ~0x3fu) | (vgprField & 0x3fu); + // FIX 1 pools: DSWS2_FLOW=1 -> N-deep flow pool (LDS_TOTAL_FLOW = 256 + POOL_N*OPSTRIDE); + // DSWS2_RING=1 -> D=2 ring (33024); neither -> single-slot occ_kernel_dsws.s (16640, byte-identical). + uint32_t poolSlots = 1u; + if (getenv("DSWS2_FLOW")) poolSlots = getenv("FLOW_POOL_N") ? (uint32_t)atoi(getenv("FLOW_POOL_N")) : 3u; + else if (getenv("DSWS2_RING")) poolSlots = 2u; + uint32_t operandBytes = (uint32_t)(FNc*16*SEGKv) + (uint32_t)((uint32_t)Gv*16*FMc*SEGKv); // per-slot = 16384 + // FIX 1 STAGGER: flow adds a per-rowblk fp32 reduction accumulator pool (ACC_N banks x FM*FN*1024B) AFTER + // the operand pool. Must match the kernel's ACC_BASE/ACC_STRIDE/ACC_N (DSWS2_ACC_N, default 1; 0 for ring/single). + uint32_t accN = getenv("DSWS2_FLOW") ? (getenv("DSWS2_ACC_N") ? (uint32_t)atoi(getenv("DSWS2_ACC_N")) : 1u) : 0u; + uint32_t accBytes = accN * (uint32_t)(FMc*FNc*1024); + uint32_t ldsBytesRaw = 256u + poolSlots * operandBytes + accBytes; // flow POOL3/ACC1:57600 POOL2/ACC2:49408 + uint32_t ldsU=0, ldsA=0, ldsG=0; uint32_t ldsBits = ldsRsrc2Bits(ldsBytesRaw, &ldsU, &ldsA, &ldsG); + uint32_t rsrc2 = (BuildPgmRsrc2(true) & ~0x3eu) | (15u << RSRC2_USER_SGPR_SHIFT) | ldsBits; // USER_SGPR=15 (FIX 1h: dropped s15..s17) + uint32_t rsrc[2] = {rsrc1, rsrc2}; + printf(" [dsws2] %dx%dx%d super-tile=%dx%d (G=%d FM=%d FN=%d) TOTAL=%u TOTAL_super=%llu n_kseg=%d " + "waves/WG=%u(=%uc%ua%ub) LDS=%uB(alloc %uB) VGPR~%u dyn=1 RSRC2=0x%x\n", + Mo,No,Ko, TMsuper,TN, Gv,FMc,FNc, TOTAL, (unsigned long long)TOTAL_super, n_kseg, + WAVES_LAUNCH, nComp,nAfeed,nBfeed, ldsBytesRaw, ldsA, vgprField*8, rsrc2); + + // FIX 1(i): 15 kernargs (s0..s14) only -- n_kseg/TOTAL_super/magic_kseg dropped (derived in-kernel / + // memory-carried via occ[24], see occ_kernel_dsws.s KERNARG CONTRACT). Array is still 16 wide to match + // every other proven path's single 16-register SET_SH_REG packet; index 15 is unused padding (lands + // in the hardware's TGID_X slot, which this kernel does not read). + uint32_t userdata[16] = { + (uint32_t)occVa,(uint32_t)(occVa>>32), (uint32_t)aVa,(uint32_t)(aVa>>32), // s0:1 occ, s2:3 A + (uint32_t)bVa,(uint32_t)(bVa>>32), (uint32_t)cVa,(uint32_t)(cVa>>32), // s4:5 Bshuf, s6:7 C + (uint32_t)KT, (uint32_t)Ko, (uint32_t)(NT*256), TOTAL, // s8 KT, s9 K(bytes/row), s10 NTx256, s11 TOTAL + magic, (uint32_t)NTL, (uint32_t)(FNc*256), 0u }; // s12 magic, s13 NTL, s14 FNx256, [15] unused + uint32_t dispInit = BuildDispatchInitiator(); + + const uint32_t poolD = getenv("ML8_POOL") ? (uint32_t)atoi(getenv("ML8_POOL")) : 64u; + const uint32_t pool = poolD < 64u ? poolD : 64u; + const char* ydis = getenv("ML8_YIELD_DISABLE"); bool yieldOff = ydis && ydis[0]=='1'; + int yieldMs = getenv("ML8_YIELD_MS") ? atoi(getenv("ML8_YIELD_MS")) : 5; + if (yieldMs < 0) yieldMs = 0; + double yieldEvery = (getenv("ML8_YIELD_EVERY_MS") ? atoi(getenv("ML8_YIELD_EVERY_MS")) : 100) / 1000.0; // proven run_mbgemm cadence + if (yieldEvery <= 0.0) yieldEvery = 0.1; + const double timeoutS = 25.0; + // COMPOSITOR-SAFE CHUNKING (mirrors run_mbcoop): bound each dispatch to ML8_COOP_CHUNK super-tiles + // (claim starts at occ[20]=base, terminal bound occ[24]/occW[6]=chunkHi -- FIX 1(j), memory-carried + // since there is no deliverable kernarg slot for it) and yield between dispatches. + uint64_t chunkTilesEnv = getenv("ML8_COOP_CHUNK") ? (uint64_t)atoll(getenv("ML8_COOP_CHUNK")) : 0ull; + // FIX 1 STAGGER: the flow write-once kernel claims occ[20] as whole TILES (a WG owns a tile's n_kseg + // segments so its per-WG LDS banks sum a full tile); every other dsws2 path claims super-tiles. + const uint64_t claimTotal = getenv("DSWS2_FLOW") ? (uint64_t)TOTAL : TOTAL_super; + uint64_t chunkTiles = (chunkTilesEnv == 0ull || chunkTilesEnv > claimTotal) ? claimTotal : chunkTilesEnv; + uint64_t nChunks = (claimTotal + chunkTiles - 1ull) / chunkTiles; + double chunkMaxS = getenv("ML8_COOP_CHUNK_MAXS") ? atof(getenv("ML8_COOP_CHUNK_MAXS")) : 0.75; + if (chunkTiles < TOTAL_super) printf(" [dsws2] compositor-safe: %llu super-tiles/dispatch x %llu chunks (yield %dms between; abort chunk > %.2fs)\n", + (unsigned long long)chunkTiles, (unsigned long long)nChunks, yieldMs, chunkMaxS); + bool streamOn = getenv("ML8_COOP_STREAM") != nullptr; + uint32_t reslim[1]={0}, tmpring[1]={0}, restart[4]={0,0,0,0}; + bool allok = true; uint32_t lastOcc0 = 0, lastOcc20 = 0; uint32_t totalConv = 0; // occ[48] conv-commit count, summed across chunks (reset per chunk) + uint64_t sumSpan = 0; uint32_t spanChunks = 0; bool tfMissing = false; // TFPROBE: summed per-chunk GPU-tick span (occ[3]-occ[2]); tfMissing => bin has no tick capture + // SUSTAINED (DSWS2_REPS>1): re-run the whole chunked GEMM back-to-back, buffers reused, C re-zeroed per rep + // (split-K atomic-adds into C, so a repeated pass without reset would double it). Spans sum across ALL + // reps -> TF is over reps*(2MNK) work / total busy ticks (warm-clock steady state, not a cold ms blip). + // Per-rep span min/max -> the TF spread (glass-flat vs jittery), the trustworthiness signal. + uint32_t dswsReps = getenv("DSWS2_REPS") ? (uint32_t)atoi(getenv("DSWS2_REPS")) : 1u; + if (dswsReps < 1u) dswsReps = 1u; + double dswsTarget = getenv("DSWS2_TARGET_SECS") ? atof(getenv("DSWS2_TARGET_SECS")) : 0.0; // >0: rep until this many wall-secs + double repT0 = now_s(); + uint64_t repSpanMin = ~0ull, repSpanMax = 0; uint32_t repsDone = 0; + for (uint32_t rep = 0; ; ++rep) { + if (dswsTarget > 0.0) { if (rep > 0 && (now_s() - repT0) >= dswsTarget) break; } // duration-bounded + else { if (rep >= dswsReps) break; } // count-bounded + if (rep > 0) memset((char*)C.ptr, 0, cbytes); // split-K accumulation reset before each repeated pass + uint64_t repSpanBase = sumSpan; + // ML8_CHUNK_DIAG: per-chunk wall + STAGINSTR delta (coast/computed/feed/grow-fail). occ[70..73] are + // OUTSIDE the per-chunk memset (occ[0..63]) so they accumulate; snapshot before each chunk for the delta. + // A slow chunk with grow-fail/coast spiking + computed crawling == VGPR-starvation churn (compositor + // held the SIMD pool during the inter-chunk yield); a slow chunk that is mostly `computed` == real work. + const bool chunkDiag = getenv("ML8_CHUNK_DIAG") != nullptr; + for (uint64_t base = 0; base < claimTotal; base += chunkTiles) { + uint64_t chunkHi = (base + chunkTiles < claimTotal) ? (base + chunkTiles) : claimTotal; + uint32_t diagPrevCoast = occW[70], diagPrevComp = occW[71], diagPrevFeed = occW[72], diagPrevGF = occW[73]; + memset((void*)occW, 0, 0x100); // re-zero the control region (occ[0] live, occ[20] claim, reserved) each chunk + occW[5] = (uint32_t)base; // occ[20] (=occW[5]) claim counter starts at this chunk's base sti + occW[6] = (uint32_t)chunkHi; // FIX 1(j): occ[24] (=occW[6]) = this chunk's terminal sti bound (memory-carried) + occW[2] = 0xFFFFFFFFu; // TFPROBE: min-sentinel for the entry-tick atomic_min (occ[2]); occ[3] stays 0 (max) + occW[62] = magicTotal; // KMAJOR: magic(TOTAL) for the ksi=sti/TOTAL decode (ignored unless KMAJOR bin) + if (traceW) { uint64_t tva=(uint64_t)traceBuf.ptr; // TRACE: (re-)publish buffer VA + cap (memset above wiped occ[52..54]) + occW[52]=(uint32_t)tva; occW[53]=(uint32_t)(tva>>32); occW[54]=traceMaxRows; } + *fenceW = 0; + RingPlace(ring, PM4AcquireMemoryPacket(FAMILY_GFX12)); + RingPlace(ring, PM4SetShaderRegPacket(mmCOMPUTE_START_X, dims, 8)); + RingPlace(ring, PM4SetShaderRegPacket(mmCOMPUTE_PGM_LO, pgm, 6)); + RingPlace(ring, PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC1, rsrc, 2)); + RingPlace(ring, PM4SetShaderRegPacket(mmCOMPUTE_RESOURCE_LIMITS, reslim, 1)); + RingPlace(ring, PM4SetShaderRegPacket(mmCOMPUTE_TMPRING_SIZE, tmpring, 1)); + RingPlace(ring, PM4SetShaderRegPacket(mmCOMPUTE_RESTART_X, restart, 4)); + RingPlace(ring, PM4SetShaderRegPacket(mmCOMPUTE_USER_DATA_0, userdata, 16)); // s0..s14 (FIX 1i: ONE packet, like every proven path) + RingPlace(ring, PM4DispatchDirectPacket(pool * (uint32_t)(WAVES_LAUNCH*32), 1, 1, dispInit)); // grid = pool WGs + // NOTE: CS_PARTIAL_FLUSH here was tried (2026-07-05) and STALLS too -- the CP-level terminal-store/wave + // drain is genuinely stuck at 16 waves; no host packet (drop-EOP, drop-ACQUIRE, PARTIAL_FLUSH) fixes it. + // The fix is shader-side (terminal store / dyn-VGPR wave retirement). See KG. + // Codex fix (2026-07-05): EOP RELEASE_MEM only on the FINAL chunk. The terminal-store-drain quirk stalls + // the EOP fence with no post-kernel traffic; a per-chunk stalled EOP sits in the in-order queue and + // BLOCKS the next chunk's DispatchDirect from ever launching (that was the chunk1 "occ20 stuck at base, + // 0 claims" hang). Non-last chunks serialize via the kernel-done gate (occ0==0 + settle) in the poll + // loop below; only the last chunk arms the real fence (guarded teardown handles it if it too stalls). + const bool lastChunk = (chunkHi >= claimTotal); + if (lastChunk) RingPlace(ring, PM4ReleaseMemoryPacket(FAMILY_GFX12, true, fenceVa, FENCE_VALUE)); + double t0 = now_s(); RingSubmit(ring); + bool done = false, admitted = false; double lastSnap = t0, lastYield = t0; + // Complete on the KERNEL'S OWN done-signal (occ0==0 + settle), NOT the EOP fence. The terminal C store's + // s_endpgm implicit drain stalls the EOP fence on this raw-PM4 path (COOP_STATUS.md; the proven coop + // path completes the same way via ML8_COOP_NOFENCE). The store IS issued -- the settle lets it land + // before the oracle reads C (a stale read fails the oracle, never a false CLEAN). The guarded teardown + // below still refuses to destroy a non-idle queue (brick-avoidance), so a lingering wave is reclaimed + // by process-exit, never a forced destroy. A real hang still trips the timeoutS bail -> forensics. + uint32_t lastEnd = 0; double lastEndChange = t0; + double settle = getenv("DSWS2_SETTLE") ? atof(getenv("DSWS2_SETTLE")) : 0.30; + while (true) { double now = now_s(); + // COMPOSITOR YIELD (proven run_mbgemm mechanism, was MISSING on the flow path): hand the gfx ring + // (Hyprland) an unconditional render+VGPR window every yieldEvery ms DURING the wait -- so a long or + // stuck dispatch can't starve the desktop's gfx ring into a ring-timeout MODE1 reset. Host sleep + // only; never enters the in-kernel TF span. This is the "let hyprland through every so often" logic. + if (!yieldOff && yieldMs > 0 && (now - lastYield) >= yieldEvery) { + struct timespec yts = { yieldMs/1000, (long)(yieldMs%1000)*1000000L }; + nanosleep(&yts, nullptr); lastYield = now_s(); + } + if (streamOn && (now - lastSnap) >= 0.2) { lastSnap = now; + fprintf(stderr, "[dsws2 +%5.2fs] occ0(live)=%u occ20(claim)=%u fence=%s\n", + now-t0, occW[0], occW[5], (*fenceW==FENCE_VALUE)?"FIRED":"--"); fflush(stderr); } + if (occW[0] > 0) admitted = true; + uint32_t end = occW[3]; if (end != lastEnd) { lastEnd = end; lastEndChange = now; } + bool ff = (*fenceW == FENCE_VALUE); + // done = fence fired, OR kernel-done: occ0==0 (all counted waves retired) + a wave stamped its exit + // tick (occ[3]!=0) + settled (store landed). Matches the coop gate. + if (admitted && occW[0]==0 && (ff || (end != 0 && (now - lastEndChange) > settle))) { done = true; break; } + if (now - t0 > timeoutS) break; + } + if (!done) { + fprintf(stderr, "\n*** DSWS2 TIMEOUT (chunk base=%llu hi=%llu): occ0(live)=%u occ20(claim)=%u fence=%s ***\n", + (unsigned long long)base, (unsigned long long)chunkHi, occW[0], occW[5], (*fenceW==FENCE_VALUE)?"FIRED":"--"); + // EMERGENT-economy timeout forensics: WHERE did it stall? (all slots stream live during the run) + // computed>0 & rising -> compute progressed (slow/contention); computed~0 & coast huge -> LIVELOCK. + fprintf(stderr, " [timeout forensics] residentPeak occ[1]=%u fatPeak occ[58]=%u fatResidual occ[57]=%u alllive-net occ[60]=%u (TRACE: >0 w/ occ0=0 == waves stuck PRE-LIVE at .Lflow_alloc)\n", + occW[1], occW[58], occW[57], occW[60]); + fprintf(stderr, " [timeout forensics] STAGINSTR coast occ[70]=%u computed occ[71]=%u feed-stages occ[72]=%u grow-fail occ[73]=%u\n", + occW[70], occW[71], occW[72], occW[73]); + allok = false; break; + } + if (chunkDiag) { + double cwall = now_s() - t0; + fprintf(stderr, " [chunk diag] base=%llu hi=%llu wall=%.3fs claim=%u STAGINSTR d: coast=%u computed=%u feed=%u grow-fail=%u%s\n", + (unsigned long long)base, (unsigned long long)chunkHi, cwall, occW[5], + occW[70]-diagPrevCoast, occW[71]-diagPrevComp, occW[72]-diagPrevFeed, occW[73]-diagPrevGF, + (cwall > 0.5) ? " <-- SLOW" : ""); + fflush(stderr); + } + lastOcc0 = occW[0]; lastOcc20 = occW[5]; totalConv += occW[48]; // accumulate this chunk's role-switch commits (DIAG conv counter) + // TFPROBE: read this chunk's device-busy span BEFORE the next iteration re-zeros occ[2]/occ[3]. A stamped + // chunk has occ[2] != 0xFFFFFFFF (entry min written) AND occ[3] != 0 (exit max written). Sum spans across + // chunks -> total GPU busy ticks for the whole GEMM (host inter-chunk gaps excluded). If unstamped, the + // bin lacks TFPROBE tick capture -> flag and skip (no bogus TF from the 0xFFFFFFFF sentinel). + { uint32_t gs = occW[2], ge = occW[3]; + if (gs != 0xFFFFFFFFu && ge != 0) { + sumSpan += (ge >= gs) ? (uint64_t)(ge - gs) : ((uint64_t)ge + 0x100000000ull - (uint64_t)gs); + spanChunks++; + } else tfMissing = true; } + if (!yieldOff && yieldMs > 0) { struct timespec ts = { yieldMs/1000, (long)(yieldMs%1000)*1000000L }; nanosleep(&ts, nullptr); } + if ((now_s() - t0) > chunkMaxS) { + fprintf(stderr, " [dsws2] WARN chunk @base%llu wall %.2fs > %.2fs cap -> ABORT remaining chunks\n", + (unsigned long long)base, now_s()-t0, chunkMaxS); + allok = false; break; + } + } + if (!allok) break; // rep loop: bail on any chunk failure/timeout + { uint64_t rs = sumSpan - repSpanBase; // this rep's busy-tick span (across its chunks) + if (rs > 0) { if (rs < repSpanMin) repSpanMin = rs; if (rs > repSpanMax) repSpanMax = rs; repsDone++; } } + } // ---- end SUSTAINED rep loop ---- + if (!allok) { + fprintf(stderr, " [teardown] dsws2 run did not complete cleanly -> NOT destroying queue (brick-avoidance; process-exit reclaims).\n"); + return res; + } + { double tw = now_s(); struct timespec ts = {0, 2000000L}; + while (*fenceW != FENCE_VALUE && (now_s() - tw) < 5.0) nanosleep(&ts, nullptr); } + bool queueIdle = (*fenceW == FENCE_VALUE); + if (!queueIdle) fprintf(stderr, " [teardown] WARN: EOP fence never fired in 5s; queue NON-IDLE -> NOT destroying (process-exit reclaims).\n"); + res.occ0 = lastOcc0; res.occClaim = lastOcc20; + printf(" [dsws2 alllive-net] occ[60]=%u peak-resident occ[1]=%u (TRACE build: occ[60]>0 w/ occ0=0 == waves stuck PRE-LIVE at .Lflow_alloc)\n", occW[60], occW[1]); + printf(" [dsws2 completion] occ[0](live)=%u (0=clean) occ[20](claim)=%u (NOTE: with pool>=1, each WG's pinned\n" + " claimer makes exactly one extra terminal over-claim past the bound, so the expected clean value is\n" + " chunkHi(last chunk)+#WGs-that-raced-the-last-claim, NOT exactly TOTAL_super=%llu -- treat occ[20] as a\n" + " 'did every WG's claimer reach a terminal claim' liveness signal, occ[0]==0 as the real completion gate)\n", + lastOcc0, lastOcc20, (unsigned long long)TOTAL_super); + printf(" [dsws2 CONVERSIONS] committed role-switches (occ[48], summed over chunks) = %u (>0 => waves ADAPTIVELY switched role)\n", totalConv); + { // STAGINSTR write-once diag: feed-vs-compute-bound. occ[70]=coast iters, [71]=computed, [72]=feed stages. + uint32_t coastIt = occW[70], compIt = occW[71], feedIt = occW[72], growFail = occW[73]; + if (coastIt + compIt > 0) { + double starve = 100.0 * (double)coastIt / (double)(coastIt + compIt); + printf(" [dsws2 STAGINSTR] compute-wave iters: coast=%u computed=%u feed-stages=%u grow-fail=%u\n" + " -> coast-frac=%.1f%% grow-fail(stagger-repulsion)=%u (%.1f%% of coasts)\n", + coastIt, compIt, feedIt, growFail, starve, growFail, + coastIt > 0 ? 100.0 * (double)growFail / (double)coastIt : 0.0); + } + } + if (traceOn) { + uint32_t fatPeak = occW[58], fatResidual = occW[57]; // FATMAX / FATLIVE (should end ~0 if balanced) + printf(" [dsws2 VGPR-BUDGET PROBE] peak concurrent FAT compute waves (occ[58]) = %u -> ~%u VGPR in flight (x NFV=112)" + " [residual live=%d]\n", fatPeak, fatPeak*112u, (int)fatResidual); + printf(" (per-SIMD B estimate = peak/128 SIMDs x 112; raise DSWS2_BUDGET/pool until this plateaus or s_alloc stalls)\n"); + uint32_t peakWaves = occW[1]; // occ[1] = peak concurrent RESIDENT waves (all roles), vs 2048 HW ceiling (16/SIMD) + printf(" [dsws2 OCCUPANCY] peak concurrent resident waves (occ[1]) = %u of 2048 HW max (%.1f%%, %.2f/SIMD) " + "launched = %u WGs x %u waves = %u\n", peakWaves, peakWaves/2048.0*100.0, peakWaves/128.0, + pool, WAVES_LAUNCH, pool*WAVES_LAUNCH); + } + + // ---- TFPROBE THROUGHPUT: total useful work / total device-busy span. Work = 2*M*N*K (split-K independent; + // the n_kseg segments reduce the SAME K, so total MACs = M*N*K regardless of how K is partitioned). Span + // = summed per-chunk (occ[3]-occ[2]) GPU ticks -> the on-chip busy time, immune to host launch/fence/poll + // overhead (the reason a host wall-clock is useless at these <1ms shapes). TF = 2*M*N*K*freq / span / 1e12. ---- + if (spanChunks > 0 && sumSpan > 0) { + res.wall = sumSpan; + double reps_eff = (repsDone > 0) ? (double)repsDone : 1.0; // work = reps_eff * (2MNK); span = sum over reps + double workAll = 2.0 * (double)Mo * (double)No * (double)Ko * reps_eff; + res.tf = workAll * freq_hz / (double)sumSpan / 1e12; + double perRepWork = 2.0 * (double)Mo * (double)No * (double)Ko; + double tfHi = (repSpanMax > 0) ? perRepWork * freq_hz / (double)repSpanMin / 1e12 : res.tf; // min span -> peak TF + double tfLo = (repSpanMax > 0) ? perRepWork * freq_hz / (double)repSpanMax / 1e12 : res.tf; // max span -> trough TF + printf(" [dsws2 THROUGHPUT] %dx%dx%d TF=%.1f (%.1f%% of 307 TF fp8 peak) span=%llu ticks / %u chunk(s) @ %.0f MHz\n", + Mo, No, Ko, res.tf, res.tf / 307.0 * 100.0, (unsigned long long)sumSpan, spanChunks, freq_hz / 1e6); + if (repsDone > 1) + printf(" [dsws2 SUSTAINED] reps=%u TF=%.1f mean (per-rep %.1f-%.1f, spread %.1f%%) -- glass-flat=trustworthy\n", + repsDone, res.tf, tfLo, tfHi, (tfHi > 0 ? (tfHi - tfLo) / tfHi * 100.0 : 0.0)); + } else { + printf(" [dsws2 THROUGHPUT] n/a -- bin has no TFPROBE tick capture (occ[2]/occ[3] unstamped%s). " + "Rebuild the bin with -Wa,-defsym,TFPROBE=1 to measure TF.\n", tfMissing ? "" : "; no chunks completed"); + } + + // ---- PHASEPROBE: in-kernel per-phase tick breakdown of the COMPUTE wave (the critical path). + // Accumulators at occ[64..69] (bytes 256..276) live ABOVE the 0x100 per-chunk memset -> they SUM + // over the whole run. u32 slots -> keep PHASEPROBE runs short (single/few passes) to avoid wrap; the + // DISTRIBUTION (%) is stable regardless. Ticks are summed across ALL compute waves (aggregate time + // in each phase), so % shows WHERE compute-wave time goes -- measured, not inferred. ---- + { + const char* phName[6] = {"FOLLOW_WAIT","STAGE_WAIT","GROW","WMMA","FLUSH","SHRINK"}; + uint64_t ph[6] = {0,0,0,0,0,0}, phSum = 0; + for (int i = 0; i < 6; i++) { ph[i] = (uint64_t)occW[64 + i]; phSum += ph[i]; } + if (phSum > 0) { + printf(" [dsws2 PHASE breakdown] compute-wave ticks by phase (summed over all waves+chunks):\n"); + printf(" %-12s %14s %6s %s\n", "phase", "ticks", "share", "what"); + const char* phWhat[6] = {"idle: waiting for claimer to publish next super-tile", + "idle: waiting for A/B feeds to stage operands", + "dyn-VGPR grow 32->112 (+ rowblk claim)", + "the actual fp8 WMMA compute", + "split-K C reduction (global_atomic_add_f32)", + "dyn-VGPR shrink 112->32"}; + for (int i = 0; i < 6; i++) { + double pct = 100.0 * (double)ph[i] / (double)phSum; + char bar[41]; int nb = (int)(pct / 2.5 + 0.5); if (nb > 40) nb = 40; + for (int k = 0; k < nb; k++) bar[k] = '#'; bar[nb] = 0; + printf(" %-12s %14llu %5.1f%% %-40s %s\n", phName[i], + (unsigned long long)ph[i], pct, bar, phWhat[i]); + } + printf(" %-12s %14llu\n", "TOTAL", (unsigned long long)phSum); + } + } + + // ---- TRACE dump: read the per-super-tile rows back to CSV (real disk). Rows are indexed by SEGCNT + // (1-based), so row 0 stays zero; skip all-zero (unwritten) rows. Single-chunk runs only (chunked + // runs reset SEGCNT per chunk -> rows overwrite). ---- + if (traceW) { + const char* csv = getenv("DSWS2_TRACE_CSV"); + char path[600]; + if (!csv) { snprintf(path, sizeof path, "/home/kmbandy/dsws_gpu_logs/trace_%dx%dx%d.csv", Mo, No, Ko); csv = path; } + FILE* tf = fopen(csv, "w"); + if (tf) { + fprintf(tf, "row,tick_lo,segcnt,epoch,nComp,nAfeed,nBfeed,occA,occB,convCount,vresv,sti,quiesce,tick_hi,chunkHi,wg_id\n"); + uint32_t nrows = 0; + for (uint32_t r = 0; r < traceMaxRows; ++r) { + const volatile uint32_t* row = traceW + (size_t)r*16; + bool nz = false; for (int i = 0; i < 16; ++i) if (row[i]) { nz = true; break; } + if (!nz) continue; + fprintf(tf, "%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u\n", + r, row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9],row[10],row[11],row[12],row[13],row[14]); + ++nrows; + } + fclose(tf); + printf(" [dsws2 trace] wrote %u rows -> %s\n", nrows, csv); + } else fprintf(stderr, " [dsws2 trace] cannot open '%s' for write\n", csv); + } + + // ---- CANARY scan (verbatim pattern from run_mbcoop): did any C store land past the data region? ---- + { const uint32_t* tail = (const uint32_t*)((const char*)C.ptr + cbytes); + uint64_t words = padB/4, firstNZ = ~0ull, lastNZ = 0, nzCount = 0; + for (uint64_t w=0; wr and P->G (the v2 G-extent). ---- + const float* Cf = (const float*)C.ptr; + // SAMPLED ORACLE: full-K CPU reference is ~O(TOTAL*G*KT) MACs -> minutes at training M (640 tiles, KT=128). + // DSWS2_ORACLE_STRIDE>1 checks every Nth output tile (still every rowblk/frag within it) so big-shape + // perf runs verify a representative subset cheaply. Default 1 = full check (unchanged for small shapes). + int ostride = getenv("DSWS2_ORACLE_STRIDE") ? atoi(getenv("DSWS2_ORACLE_STRIDE")) : 1; + if (ostride < 1) ostride = 1; + int nTilesChecked = 0; + for (int ti = 0; ti < (int)TOTAL; ti += ostride) { + nTilesChecked++; + int mblk = ti / NTL, tcol = ti % NTL; + for (int r = 0; r < Gv; ++r) { + for (int mi = 0; mi < FMc; ++mi) for (int ni = 0; ni < FNc; ++ni) { + int rowbase = mblk*TMsuper + r*(FMc*16) + mi*16; + int colbase = tcol*TN + ni*16; + float Cacc[256]; for (int i=0;i<256;i++) Cacc[i]=0.f; + uint8_t Ablk[256], Bblk[256]; float Dout[256]; + for (int kt = 0; kt < KT; ++kt) { + for (int i=0;i<16;i++) for (int j=0;j<16;j++) { + Ablk[i*16+j] = Ah[(size_t)(rowbase+i)*Ko + (kt*16+j)]; + Bblk[i*16+j] = Bh[(size_t)(kt*16+i)*No + (colbase+j)]; + } + wmma_ref_16x16x16(Ablk, Bblk, Cacc, Dout); + for (int i=0;i<256;i++) Cacc[i]=Dout[i]; + } + int frag = mi*FNc + ni; + size_t foff = (size_t)ti*(size_t)((uint32_t)Gv*FMc*FNc*256) + (size_t)r*(size_t)(FMc*FNc*256) + (size_t)frag*256; + float D[256]; unpack_D(Cf + foff, D); + OracleCmp cmp = oracle_compare(D, Cacc, 256, orel, oabs); + if (cmp.ok) res.okFrags++; else res.badFrags++; + if (cmp.max_rel > res.maxRel) res.maxRel = cmp.max_rel; + } + } + } + printf(" [dsws2 oracle] ok=%llu bad=%llu max_rel=%.4g tier=%s (rel=%.0e abs=%.0e) [%d/%u tiles checked, stride=%d]\n", + (unsigned long long)res.okFrags, (unsigned long long)res.badFrags, res.maxRel, + n_kseg==1?"TIGHT":"LOOSE", orel, oabs, nTilesChecked, TOTAL, ostride); + + if (queueIdle) { + CHECK(hsaKmtDestroyQueue(ring.res.QueueId)); + if (traceOn && traceBuf.ptr) FreeGpu(traceBuf); + FreeGpu(ring.buf); FreeGpu(fence); FreeGpu(C); FreeGpu(Bd); FreeGpu(Ad); FreeGpu(occ); FreeGpu(isa); + } + res.ok = allok && (res.badFrags == 0); + return res; +} + +// =========================================================================================== +// run_grind: the NON-split-K CONTROL. Launches occ_kernel_grind.bin (1 wave = 1 WG owns one +// FM*16 x FN*16 output tile, full-K in registers, writes C ONCE with plain global_store, NO +// split-K, NO C-atomic reduction). Directly comparable to run_dsws2's TF: does avoiding split-K's +// 32x C-write amplification beat the split-K kernel's ~2.1 TF? Static VGPR (the anti-moat control). +// Self-instrumented tick span (occ[2]/occ[3]) + maxlive (occ[1]) live in the kernel already. +// =========================================================================================== +struct GrindResult { bool ok=false; uint64_t okFrags=0, badFrags=0; double maxRel=0.0, tf=0.0; uint64_t wall=0; uint32_t maxlive=0, occ0=0; }; + +static GrindResult run_grind(uint32_t node, const char* isaPath, int FMc, int FNc, + int Mo, int No, int Ko, float orel, float oabs, double freq_hz) { + GrindResult res; + const int TM = FMc*16, TN = FNc*16; // grind output tile = FM*16 x FN*16 (32x64 @ 2x4) + if (TM==0 || TN==0 || Ko<=0 || (Mo%TM) || (No%TN) || (Ko%16)) { + fprintf(stderr, " [grind] geometry %dx%dx%d not tile-aligned (TM=%d TN=%d, K%%16)\n", Mo,No,Ko,TM,TN); return res; } + const int MTL = Mo/TM, NTL = No/TN, KT = Ko/16, NT = No/16; + const uint32_t TOTAL = (uint32_t)MTL*(uint32_t)NTL; // grind tile count (finer: no G, no ksi) + const uint32_t magic = (uint32_t)((0x100000000ULL + (uint64_t)NTL - 1)/(uint64_t)NTL); + const int KCHUNK = getenv("GRIND_KCHUNK") ? atoi(getenv("GRIND_KCHUNK")) : 4; + + static const uint8_t NICE[6] = {0x38,0x40,0x30,0xB8,0xC0,0xB0}; + std::vector Ah((size_t)Mo*Ko), Bh((size_t)Ko*No), Bshufh((size_t)Ko*No); + for (size_t i=0;i abort ***\n"); abort(); } + // address bounds gate: last A/B/C element the kernel can touch must be in-buffer. + { uint64_t Amax = (uint64_t)(MTL*FMc*16-1)*Ko + (uint64_t)(KT-1)*16 + 8 + 7; // max row=(MTL*FM*16-1), max kcol=(KT-1)*16+8+7 + uint64_t Bmax = (uint64_t)(NTL-1)*FNc*256 + (uint64_t)(KT-1)*(uint64_t)NT*256 + (uint64_t)(FNc-1)*256 + (uint64_t)31*8 + 7; + uint64_t Cmax = (uint64_t)(TOTAL-1)*(uint64_t)(FMc*FNc*1024) + (uint64_t)(FMc*FNc-1)*1024 + (uint64_t)31*32 + 7*4 + 3; + bool aok=Amax REFUSE ***\n"); + FreeGpu(fence);FreeGpu(C);FreeGpu(Bd);FreeGpu(Ad);FreeGpu(occ);FreeGpu(isa); return res; } } + memcpy(isa.ptr,isaBytes,isaLen); free(isaBytes); + memcpy(Ad.ptr,Ah.data(),Ah.size()); memcpy(Bd.ptr,Bshufh.data(),Bshufh.size()); + memset((char*)Ad.ptr+((Ah.size()+0xFFF)&~0xFFFull),0,padB); + memset((char*)Bd.ptr+((Bshufh.size()+0xFFF)&~0xFFFull),0,padB); + volatile uint32_t* occW=(volatile uint32_t*)occ.ptr; volatile uint32_t* fenceW=(volatile uint32_t*)fence.ptr; + memset((void*)occW,0,occ.size); *fenceW=0; + memset((char*)C.ptr,0,cbytes+padB); // grind writes each cell once; zero anyway (canary tail + clean) + + Ring ring; ring.buf=AllocGpu(node,0x10000,true,true); ring.dw=(uint32_t*)ring.buf.ptr; ring.sizeDw=(uint32_t)(ring.buf.size/sizeof(uint32_t)); + CHECK(hsaKmtCreateQueue(node,HSA_QUEUE_COMPUTE,100,HSA_QUEUE_PRIORITY_NORMAL,ring.buf.ptr,ring.buf.size,nullptr,&ring.res)); + uint64_t shiftedIsa=((uint64_t)isa.ptr)>>8; + uint64_t occVa=(uint64_t)occ.ptr,aVa=(uint64_t)Ad.ptr,bVa=(uint64_t)Bd.ptr,cVa=(uint64_t)C.ptr,fenceVa=(uint64_t)fence.ptr; + const uint32_t WAVES_LAUNCH=1u; // 1 wave/WG (the control) + uint32_t dims[8]={0,0,0,WAVES_LAUNCH*32,1,1,0,0}; + uint32_t pgm[6]={(uint32_t)shiftedIsa,(uint32_t)(shiftedIsa>>32)|(g_is_dgpu?0u:(1u<<8)),0,0,0,0}; + const uint32_t vgprField=15u; // static 120 VGPR (NFV~108); anti-moat control + uint32_t rsrc1=BuildPgmRsrc1(false); rsrc1=(rsrc1 & ~0x3fu)|(vgprField & 0x3fu); + uint32_t ldsBytesRaw=(uint32_t)(KCHUNK*(FMc+FNc)*256); // 6144 @ KCHUNK=4,FM=2,FN=4 + uint32_t ldsU=0,ldsA=0,ldsG=0; uint32_t ldsBits=ldsRsrc2Bits(ldsBytesRaw,&ldsU,&ldsA,&ldsG); + uint32_t rsrc2=(BuildPgmRsrc2(false) & ~0x3eu)|(15u<>32),(uint32_t)aVa,(uint32_t)(aVa>>32), + (uint32_t)bVa,(uint32_t)(bVa>>32),(uint32_t)cVa,(uint32_t)(cVa>>32), + (uint32_t)KT,(uint32_t)Ko,(uint32_t)(NT*256),TOTAL, + magic,(uint32_t)NTL,(uint32_t)(FNc*256),0u }; + uint32_t dispInit=BuildDispatchInitiator(); + const uint32_t pool=getenv("ML8_POOL")?(uint32_t)atoi(getenv("ML8_POOL")):256u; // NO 64-clamp: grind is 1 wave/WG, needs many WGs + printf(" [grind] %dx%dx%d tile=%dx%d (FM=%d FN=%d) TOTAL=%u tiles waves/WG=1 pool=%u WGs LDS=%uB(alloc %uB) VGPR=%u static RSRC2=0x%x\n", + Mo,No,Ko,TM,TN,FMc,FNc,TOTAL,pool,ldsBytesRaw,ldsA,vgprField*8,rsrc2); + uint32_t reslim[1]={0},tmpring[1]={0},restart[4]={0,0,0,0}; + memset((void*)occW,0,0x100); occW[20]=0; occW[2]=0xFFFFFFFFu; *fenceW=0; // claim base 0; min-tick sentinel + RingPlace(ring,PM4AcquireMemoryPacket(FAMILY_GFX12)); + RingPlace(ring,PM4SetShaderRegPacket(mmCOMPUTE_START_X,dims,8)); + RingPlace(ring,PM4SetShaderRegPacket(mmCOMPUTE_PGM_LO,pgm,6)); + RingPlace(ring,PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC1,rsrc,2)); + RingPlace(ring,PM4SetShaderRegPacket(mmCOMPUTE_RESOURCE_LIMITS,reslim,1)); + RingPlace(ring,PM4SetShaderRegPacket(mmCOMPUTE_TMPRING_SIZE,tmpring,1)); + RingPlace(ring,PM4SetShaderRegPacket(mmCOMPUTE_RESTART_X,restart,4)); + RingPlace(ring,PM4SetShaderRegPacket(mmCOMPUTE_USER_DATA_0,userdata,16)); + RingPlace(ring,PM4DispatchDirectPacket(pool*WAVES_LAUNCH*32,1,1,dispInit)); + RingPlace(ring,PM4ReleaseMemoryPacket(FAMILY_GFX12,true,fenceVa,FENCE_VALUE)); + double t0=now_s(); RingSubmit(ring); + bool done=false, admitted=false; const double timeoutS=25.0; + while (true) { double now=now_s(); + if (occW[0]>0) admitted=true; + if (admitted && occW[0]==0 && *fenceW==FENCE_VALUE) { done=true; break; } + if (now-t0>timeoutS) break; } + if (!done) { fprintf(stderr,"\n*** GRIND TIMEOUT: occ0=%u fence=%s ***\n",occW[0],(*fenceW==FENCE_VALUE)?"FIRED":"--"); + fprintf(stderr," [teardown] grind did not complete -> NOT destroying queue (process-exit reclaims).\n"); return res; } + res.occ0=occW[0]; res.maxlive=occW[1]; + { uint32_t gs=occW[2],ge=occW[3]; + if (gs!=0xFFFFFFFFu && ge!=0) { res.wall=(ge>=gs)?(uint64_t)(ge-gs):((uint64_t)ge+0x100000000ull-(uint64_t)gs); + res.tf=2.0*(double)Mo*(double)No*(double)Ko*freq_hz/(double)res.wall/1e12; } } + printf(" [grind completion] occ[0]=%u (0=clean) maxlive=%u (of 2048, %.1f%%, %.2f/SIMD)\n", + res.occ0,res.maxlive,res.maxlive/2048.0*100.0,res.maxlive/128.0); + if (res.wall) printf(" [grind THROUGHPUT] %dx%dx%d TF=%.1f (%.1f%% of 307 TF fp8 peak) span=%llu ticks @ %.0f MHz\n", + Mo,No,Ko,res.tf,res.tf/307.0*100.0,(unsigned long long)res.wall,freq_hz/1e6); + else printf(" [grind THROUGHPUT] n/a (occ[2]/occ[3] unstamped)\n"); + // canary + { const uint32_t* tail=(const uint32_t*)((const char*)C.ptr+cbytes); uint64_t words=padB/4,nz=0; + for (uint64_t w=0;wres.maxRel) res.maxRel=cmp.max_rel; + } + } + printf(" [grind oracle] ok=%llu bad=%llu max_rel=%.4g (rel=%.0e abs=%.0e)\n", + (unsigned long long)res.okFrags,(unsigned long long)res.badFrags,res.maxRel,orel,oabs); + if (*fenceW==FENCE_VALUE) { CHECK(hsaKmtDestroyQueue(ring.res.QueueId)); + FreeGpu(ring.buf);FreeGpu(fence);FreeGpu(C);FreeGpu(Bd);FreeGpu(Ad);FreeGpu(occ);FreeGpu(isa); } + res.ok = (res.occ0==0) && (res.badFrags==0); + return res; +} + // --------------------------------------------------------------------------- // MAD-305 Step A phase-timer: dispatch the PROFILE build of the real BLADDER FEEDONLY kernel and read the // per-phase realtime tick-sums (occ[8..14]) + K-tile count (occ[15]) accumulated by the single profiler @@ -2667,9 +3337,39 @@ static void run_dynsmoke(uint32_t node) { FreeGpu(ring.buf); FreeGpu(fence); FreeGpu(occ); FreeGpu(isa); } +// ===== MAD-305 DSWS adaptive wave-role controller (SPEC_DSWS_CONTROLLER.md / PLAN_DSWS_CONTROLLER.md). +// v1 = static 3-role substrate {compute / A-feed / B-feed}; the controller layers on in Phases 2-4. +// Config is env-driven so build_dsws.sh / supervised runs stay parameterizable. ===== +struct DswsCfg { + uint32_t nComp, nAfeed, nBfeed; // role partition; N = nComp+nAfeed+nBfeed (wave count never changes) + uint32_t ringd; // A-ring / B-ring depth + uint32_t low, high; // watermark band: occhigh = over-served + uint32_t epochShift; // decision cadence: E = (segments_processed >> epochShift) + bool dyn; // arm s_alloc_vgpr dyn-VGPR (COMPUTE_PGM_RSRC2 bit6) + uint32_t N() const { return nComp + nAfeed + nBfeed; } +}; +static DswsCfg parse_dsws_cfg() { + DswsCfg c; + c.nComp = getenv("DSWS_NCOMP") ? (uint32_t)atoi(getenv("DSWS_NCOMP")) : 4u; + c.nAfeed = getenv("DSWS_NAFEED") ? (uint32_t)atoi(getenv("DSWS_NAFEED")) : 2u; + c.nBfeed = getenv("DSWS_NBFEED") ? (uint32_t)atoi(getenv("DSWS_NBFEED")) : 2u; + c.ringd = getenv("DSWS_RINGD") ? (uint32_t)atoi(getenv("DSWS_RINGD")) : 2u; + c.low = getenv("DSWS_LOW") ? (uint32_t)atoi(getenv("DSWS_LOW")) : 1u; + c.high = getenv("DSWS_HIGH") ? (uint32_t)atoi(getenv("DSWS_HIGH")) : (c.ringd > 1u ? c.ringd - 1u : 1u); + c.epochShift = getenv("DSWS_EPOCHSHIFT") ? (uint32_t)atoi(getenv("DSWS_EPOCHSHIFT")) : 3u; + c.dyn = getenv("DSWS_DYN") ? atoi(getenv("DSWS_DYN")) != 0 : false; + return c; +} + +// FIX 3(m): a positional non-flag argv token (e.g. "4c2a2b" after --dsws2/--dsws) used to be silently +// ignored by the argv loop below -- nothing ever read argv[i] once it failed every `--foo` strcmp. A user +// passing a role-mix that disagreed with the actual (env-derived) NCOMP/NAFEED/NBFEED got the WRONG +// config with zero warning. Captured here so the DSWS2 mode handler can validate it instead. +static char g_posMixArg[64] = {0}; + int main(int argc, char** argv) { setvbuf(stdout, NULL, _IONBF, 0); // unbuffered: if a raw-PM4 run hangs and gets SIGKILL'd, the log still shows WHERE it died - enum { CORRECT, PRONG1, PRONG2, PRONG3, COMBINED, TIMERCHECK, PROBE, MICROBATCH, MBGEMM, MBSAT, DYNFAT1, MBPROF, MERGE, WGGEMM, SGPRPROBE, WGLDS, LDSBOUND, WGGEMM2, WGPERF, WG2X2, NFUNROLL, NFOCC, NFBF, BANDSWP, FEEDPIPE, FEEDLADDER, FEEDBTR, FEEDPROF, FEEDSTAG, FEEDPB, STACK, BW, BASELINES, SUSTAIN, KWIN, KWINORACLE, TILEPROBE, BLDSPROBE, BTR128, ANOLDS, ANOLDSTR, WAVESWEEP, OCCSWEEP, REUSE82, REUSE82TW2, REUSE82KW2, VGPR82, BLDS82, BPF82, SP82, ALD82, WALL82, TW8, TW4LEAN, B128MODE, TILEORDMODE, FP8EDGE, LDSTRIMMODE, VGPRPROBE, LEAN, DECOMP, WAVESPEC, MBML8, MBML8LONG, MBML8GAUNT, MBML8DYN, MBML8GATE, MBML8NF, MBML8PROF, MBML8BATCH, MBML8MATCH, MBML8COOP, DYNSMOKE } mode = CORRECT; + enum { CORRECT, PRONG1, PRONG2, PRONG3, COMBINED, TIMERCHECK, PROBE, MICROBATCH, MBGEMM, MBSAT, DYNFAT1, MBPROF, MERGE, WGGEMM, SGPRPROBE, WGLDS, LDSBOUND, WGGEMM2, WGPERF, WG2X2, NFUNROLL, NFOCC, NFBF, BANDSWP, FEEDPIPE, FEEDLADDER, FEEDBTR, FEEDPROF, FEEDSTAG, FEEDPB, STACK, BW, BASELINES, SUSTAIN, KWIN, KWINORACLE, TILEPROBE, BLDSPROBE, BTR128, ANOLDS, ANOLDSTR, WAVESWEEP, OCCSWEEP, REUSE82, REUSE82TW2, REUSE82KW2, VGPR82, BLDS82, BPF82, SP82, ALD82, WALL82, TW8, TW4LEAN, B128MODE, TILEORDMODE, FP8EDGE, LDSTRIMMODE, VGPRPROBE, LEAN, DECOMP, WAVESPEC, MBML8, MBML8LONG, MBML8GAUNT, MBML8DYN, MBML8GATE, MBML8NF, MBML8PROF, MBML8BATCH, MBML8MATCH, MBML8COOP, DYNSMOKE, DSWS, DSWS2, GRIND } mode = CORRECT; bool fat = false; // --fat: include >128-VGPR shapes (require umr SQ_DYN_VGPR.BLOCK_SIZE=1, cap 256) for (int i = 1; i < argc; ++i) { if (!strcmp(argv[i], "--prong1")) mode = PRONG1; @@ -2690,6 +3390,9 @@ int main(int argc, char** argv) { else if (!strcmp(argv[i], "--mbml8match")) mode = MBML8MATCH; else if (!strcmp(argv[i], "--mbml8coop")) mode = MBML8COOP; else if (!strcmp(argv[i], "--dynsmoke")) mode = DYNSMOKE; + else if (!strcmp(argv[i], "--dsws")) mode = DSWS; + else if (!strcmp(argv[i], "--dsws2")) mode = DSWS2; + else if (!strcmp(argv[i], "--grind")) mode = GRIND; else if (!strcmp(argv[i], "--mbml8gate")) mode = MBML8GATE; else if (!strcmp(argv[i], "--mbsat")) mode = MBSAT; else if (!strcmp(argv[i], "--dynfat1")) mode = DYNFAT1; @@ -2746,6 +3449,11 @@ int main(int argc, char** argv) { else if (!strcmp(argv[i], "--lean")) mode = LEAN; else if (!strcmp(argv[i], "--decomp")) mode = DECOMP; else if (!strcmp(argv[i], "--fat")) fat = true; + else if (argv[i][0] != '-') { + // FIX 3(m): a non-flag positional token (e.g. a "4c2a2b" role-mix string) -- capture it for + // DSWS2's validation instead of silently dropping it. + snprintf(g_posMixArg, sizeof g_posMixArg, "%s", argv[i]); + } } // Test matrices A,B (16x16 e4m3, non-trivial) and the CPU oracle D = A.B. @@ -5253,6 +5961,249 @@ int main(int argc, char** argv) { c.name, r.maxlive, r.okMarks, expect, r.badMarks, r.missMarks, pass ? "PASS" : "FAIL"); if (!pass) rc = 3; } + } else if (mode == DSWS) { + // ===== MAD-305 DSWS adaptive wave-role controller. Phase 1 = STATIC 3-role substrate. + // T1.1 (this) = config + validation refuse-path ONLY; the actual --dsws dispatch + oracle gate + // wire in at T1.3. Mirrors the WAVESPEC / MBML8COOP brick-guard discipline: a geometry/bin + // mismatch under bit6-armed dyn is exactly what bricks gfx1201, so we REFUSE (rc=4, no dispatch) + // on any invalid config or missing bin. ===== + DswsCfg c = parse_dsws_cfg(); + printf("\n=== MAD-305 DSWS 3-role substrate (nComp=%u nAfeed=%u nBfeed=%u N=%u RINGD=%u LOW=%u HIGH=%u EPOCH_SHIFT=%u dyn=%d) ===\n", + c.nComp, c.nAfeed, c.nBfeed, c.N(), c.ringd, c.low, c.high, c.epochShift, c.dyn); + // ---- validation refuse-path (spec floors: compute>=1, A-feed>=1, B-feed>=1; band sanity) ---- + if (c.nComp < 1) { printf(" *** REFUSE: nComp>=1 required (compute floor); got %u ***\n", c.nComp); rc = 4; } + else if (c.nAfeed < 1) { printf(" *** REFUSE: nAfeed>=1 required (A-feed floor); got %u ***\n", c.nAfeed); rc = 4; } + else if (c.nBfeed < 1) { printf(" *** REFUSE: nBfeed>=1 required (B-feed floor); got %u ***\n", c.nBfeed); rc = 4; } + else if (c.ringd < 1) { printf(" *** REFUSE: RINGD>=1 required; got %u ***\n", c.ringd); rc = 4; } + else if (c.low > c.high) { printf(" *** REFUSE: LOW(%u) > HIGH(%u) — invalid watermark band ***\n", c.low, c.high); rc = 4; } + else if (c.high > c.ringd) { printf(" *** REFUSE: HIGH(%u) > RINGD(%u) — occ is clamped to [0,RINGD] ***\n", c.high, c.ringd); rc = 4; } + // ---- bin-presence brick-guard (T1.2/T1.3 build it via ./build_dsws.sh). Absent now by design. ---- + char dswsBin[160]; + snprintf(dswsBin, sizeof dswsBin, "occ_dsws_%uc%ua%ub_r%u%s_gd.bin", + c.nComp, c.nAfeed, c.nBfeed, c.ringd, c.dyn ? "_dyn" : ""); + if (rc == 0) { + FILE* fb = fopen(dswsBin, "rb"); + if (fb) fclose(fb); + else { printf(" *** DSWS kernel bin '%s' NOT BUILT — REFUSING to dispatch " + "(build it via ./build_dsws.sh; T1.2/T1.3) ***\n", dswsBin); + rc = 4; } + } + if (rc == 0) { + // ===== T1.3 static 3-role dispatch + oracle gate. Role counts are baked into the bin (defsyms), + // so the harness only sets WG threads = N*32 (totalWaves) and the bigger DSWS LDS; the C-store + // /oracle partition uses P=NCOMP (DSWS compute count). Small tile-multiple oracle shape first + // (sub-second, brick-safe); GENDIV (ml8 N are non-pow2). [SUPERVISED at T1.4.] ===== + const int FMc = 2, FNc = 4; // DSWS v1 fixed coop tile (baked into the kernel + bin name) + const int Nwaves = (int)c.N(); // launch N = NCOMP+NAFEED+NBFEED waves/WG + // Replicate the kernel's LDS_TOTAL_DSWS EXACTLY (RINGD_A defaults to RINGD): + // BRING + cons[NCOMP] + (prod,ti,epoch,initflag) + prod_b_hi[NBFEED-1] + A-ring + prod_a[NCOMP] + cons_a[NCOMP] + uint32_t BRING = (uint32_t)c.ringd * FNc * 256; + uint32_t ldsBase = BRING + 4u*c.nComp + 16u; // = LDS_TOTAL (the DSWS=0 prefix) + uint32_t aring = (uint32_t)c.ringd * c.nComp * FMc * 256; // RINGD_A * NCOMP * FM * 256 + uint32_t ldsDsws = ldsBase + 4u*(c.nBfeed - 1u) + aring + 8u*c.nComp; + const uint32_t poolD = getenv("ML8_POOL") ? (uint32_t)atoi(getenv("ML8_POOL")) : 64u; + const char* onlyShape = getenv("DSWS_ONLY"); + int oMTL = getenv("DSWS_ORACLE_MTL") ? atoi(getenv("DSWS_ORACLE_MTL")) : 4; + int oNTL = getenv("DSWS_ORACLE_NTL") ? atoi(getenv("DSWS_ORACLE_NTL")) : 8; + int TM = (int)c.nComp*FMc*16, TN = FNc*16; // WG tile + struct SH { const char* name; int M, K, N; }; + SH shapes[] = { {"down ", 2048, 9216, 2560}, {"down_pf", 512, 9216, 2560} }; + printf(" [dsws] N=%d waves/WG WGtile=%dx%d LDS=%uB bin=%s\n", Nwaves, TM, TN, ldsDsws, dswsBin); + for (auto& s : shapes) { + if (onlyShape && !strstr(s.name, onlyShape)) continue; + int Mo = TM*oMTL, No = TN*oNTL, Ko = 512; // small tile-multiple oracle shape + printf("\n #### DSWS %s oracle %dx%dx%d (STORE=1, GENDIV) ####\n", s.name, Mo, No, Ko); + CoopResult o = run_mbcoop(node, dswsBin, c.dyn, poolD < 64u ? poolD : 64u, Mo, No, Ko, + FMc, FNc, (int)c.nComp, (int)c.ringd, /*fullCheck*/true, + /*useGenDiv*/true, /*reps*/1, /*targetSecs*/0.0, + /*totalWaves*/Nwaves, /*ldsBytesOverride*/ldsDsws); + if (!o.ok) { printf(" oracle INCOMPLETE (hang/timeout) -> protocol/grow bug; STOP\n"); rc = 3; break; } + bool clean = (o.badFrags == 0 && o.okFrags > 0); + printf(" oracle %s ok=%llu bad=%llu maxlive=%u\n", + clean ? "CLEAN" : "*** BAD (race/math) ***", + (unsigned long long)o.okFrags, (unsigned long long)o.badFrags, o.maxlive); + if (!clean) { rc = 3; break; } + } + } + } else if (mode == GRIND) { + const int FMc=2, FNc=4; + const int Mo = getenv("GRIND_M") ? atoi(getenv("GRIND_M")) : 576; + const int No = getenv("GRIND_N") ? atoi(getenv("GRIND_N")) : 512; + const int Ko = getenv("GRIND_K") ? atoi(getenv("GRIND_K")) : 2048; + const float orel = getenv("GRIND_REL") ? atof(getenv("GRIND_REL")) : 5e-3f; // full-K single write -> TIGHT tier + const float oabs = getenv("GRIND_ABS") ? atof(getenv("GRIND_ABS")) : 1e-2f; + const char* gbin = getenv("GRIND_BIN") ? getenv("GRIND_BIN") : "occ_kernel_grind.bin"; + printf("\n=== GRIND control (non-split-K, one-tile-per-WG, full-K, write-once C) ===\n"); + FILE* fb=fopen(gbin,"rb"); if (fb) fclose(fb); else { printf(" *** grind bin '%s' NOT BUILT -> refuse ***\n",gbin); rc=4; } + if (rc==0) { + GrindResult o = run_grind(node,gbin,FMc,FNc,Mo,No,Ko,orel,oabs,freq_hz); + if (!o.ok && o.okFrags==0 && o.badFrags==0) { printf(" grind INCOMPLETE (hang/refuse) -> STOP\n"); rc=3; } + else if (o.badFrags>0) { printf(" grind oracle *** BAD *** -> STOP\n"); rc=3; } + else { printf(" grind oracle CLEAN\n"); rc=0; } + } + } else if (mode == DSWS2) { + // ===== MAD-305 DSWS v2 substrate (PLAN_DSWS_SUBSTRATE_V2.md, Task A8). Computes/dry-prints the + // super-tile pool params for occ_kernel_dsws.s and, when DSWS2_DRYRUN is unset, launches it via + // run_dsws2 (the v2 PM4 launch + tiered-oracle path). DSWS2_DRYRUN=1 -> print params + return + // rc=0 WITHOUT touching the GPU (gate 2 of A8: must still dry-print, never dispatch). ===== + DswsCfg c = parse_dsws_cfg(); + const int FMc = 2, FNc = 4; // v2 fixed coop tile (matches build_dsws.sh mk2) + const int Gv = getenv("DSWS2_G") ? atoi(getenv("DSWS2_G")) : 6; // M-extent (rowblks/super-tile) = NCOMP_MAX + const int SEGKv = getenv("DSWS2_SEGK") ? atoi(getenv("DSWS2_SEGK")) : 64; // split-K segment (K-elements) + // super-tile geometry (tile-multiple oracle shape, mirroring the --dsws oracle defaults: oMTL/oNTL/Ko). + const int TMsuper = Gv*16*FMc; // super-tile M rows = G*16*FM + const int TN = FNc*16; // N-panel cols = FN*16 + const int oMTL = getenv("DSWS2_ORACLE_MTL") ? atoi(getenv("DSWS2_ORACLE_MTL")) : 4; + const int oNTL = getenv("DSWS2_ORACLE_NTL") ? atoi(getenv("DSWS2_ORACLE_NTL")) : 8; + // n_kseg/Ko: DSWS2_NKSEG (when set) is the PRIMARY lever -- it derives Ko = SEGKv*n_kseg so the + // pool always covers the FULL K range in exactly n_kseg segments (SEGK is a compile-time defsym + // baked into the .bin's KSEG_STEPS-unrolled WMMA loop; Ko/SEGKv must stay exact or the resident + // A/B staging silently undercounts K). This is how the A8 command forces n_kseg=1 (TIGHT tier) + // without needing a separate DSWS2_K=64: `DSWS2_NKSEG=1` -> Ko=SEGKv*1=64 automatically. + // Without DSWS2_NKSEG, Ko comes from DSWS2_K (default 512) and n_kseg = Ko/SEGKv as before. + int n_kseg, Ko; + if (getenv("DSWS2_NKSEG")) { + n_kseg = atoi(getenv("DSWS2_NKSEG")); + Ko = (n_kseg > 0) ? SEGKv * n_kseg : 0; + } else { + Ko = getenv("DSWS2_K") ? atoi(getenv("DSWS2_K")) : 512; + n_kseg = (SEGKv > 0) ? (Ko / SEGKv) : 0; // = KT/(SEGK/16) (same K-units as the --dsws oracle) + } + const int Mo = TMsuper*oMTL, No = TN*oNTL; // tile-multiple oracle shape + const int KT = (SEGKv > 0) ? Ko/16 : 0; + const int NTL = No / TN; + const int MTLsuper = Mo / TMsuper; + const long long TOTAL_super = (long long)MTLsuper * NTL * n_kseg; // (M/(G*16*FM)) * NTL * n_kseg + const uint64_t TOTAL64 = (uint64_t)MTLsuper * (uint64_t)NTL; // coop-compat output-tile count (C sizing) + uint32_t poolSlots_h = 1u; // FIX 1: flow N-deep pool / ring D=2 / single-slot + if (getenv("DSWS2_FLOW")) poolSlots_h = getenv("FLOW_POOL_N") ? (uint32_t)atoi(getenv("FLOW_POOL_N")) : 3u; + else if (getenv("DSWS2_RING")) poolSlots_h = 2u; + // FIX 1 STAGGER: flow per-rowblk accumulator pool (ACC_N banks x FM*FN*1024B), matches kernel ACC_*. + const uint32_t accN_h = getenv("DSWS2_FLOW") ? (getenv("DSWS2_ACC_N") ? (uint32_t)atoi(getenv("DSWS2_ACC_N")) : 1u) : 0u; + const uint32_t ldsBytes = 256u + poolSlots_h * ((uint32_t)(FNc*16*SEGKv) + (uint32_t)(Gv*16*FMc*SEGKv)) + + accN_h * (uint32_t)(FMc*FNc*1024); + // A2 tiered oracle thresholds: TIGHT (proven gate) for n_kseg==1, LOOSE (split-K reassoc) for n_kseg>1. + // The A8 compare calls oracle_compare(got, ref, n, orel, oabs). + const float orel = (n_kseg == 1) ? 5e-3f : 3e-2f; + const float oabs = (n_kseg == 1) ? 1e-2f : 2e-2f; + const bool dry = getenv("DSWS2_DRYRUN") != nullptr; + printf("\n=== MAD-305 DSWS v2 substrate (A8 launch path; PLAN_DSWS_SUBSTRATE_V2.md) ===\n"); + printf(" G=%d SEGK=%d FM=%d FN=%d NCOMP=%u NAFEED=%u NBFEED=%u\n", + Gv, SEGKv, FMc, FNc, c.nComp, c.nAfeed, c.nBfeed); + printf(" oracle shape %dx%dx%d (super-tile %dx%d, KT=%d, NTL=%d, MTLsuper=%d)\n", + Mo, No, Ko, TMsuper, TN, KT, NTL, MTLsuper); + printf(" n_kseg=%d TOTAL_super=%lld LDS=%uB\n", n_kseg, TOTAL_super, ldsBytes); + printf(" oracle tier: %s (rel=%.0e abs=%.0e)\n", n_kseg == 1 ? "TIGHT" : "LOOSE", orel, oabs); + // FIX 3(m): DSWS2 input validation refuse-paths (mirror the --dsws T1.1 refuse-path discipline). + // Resolve the positional-mix-arg check to a single bool+message BEFORE the dry/refuse chain below, + // so it can sit as one `else if` link in that chain (dry-run must keep bypassing ALL of these + // checks -- including this one -- exactly like it already bypasses the degenerate-geometry and + // Gv/SEGK checks; that's an existing, load-bearing contract: DSWS2_DRYRUN never touches the GPU + // AND never refuses, it just prints whatever params were computed). + bool posMixBad = false; char posMixMsg[256] = {0}; + if (g_posMixArg[0]) { + uint32_t pc = 0, pa = 0, pb = 0; + if (sscanf(g_posMixArg, "%uc%ua%ub", &pc, &pa, &pb) != 3) { + posMixBad = true; + snprintf(posMixMsg, sizeof posMixMsg, + "unrecognized positional arg '%s' (expected a role-mix like '4c2a2b', or no " + "positional arg at all -- role counts come from DSWS_NCOMP/DSWS_NAFEED/DSWS_NBFEED)", + g_posMixArg); + } else if (pc != c.nComp || pa != c.nAfeed || pb != c.nBfeed) { + // a positional role-mix token (e.g. "4c2a2b") was given on the command line. The only built + // v2 bin is compile-time-fixed at NCOMP=4/NAFEED=2/NBFEED=2 (build_dsws.sh mk2); the bin + // filename is actually picked from c.nComp/nAfeed/nBfeed (the DSWS_NCOMP/AFEED/BFEED env + // config), NOT from this positional token -- previously the token was silently ignored, so + // a user passing a mix that disagreed with the active env config got the WRONG config with + // no warning. + posMixBad = true; + snprintf(posMixMsg, sizeof posMixMsg, + "positional role-mix arg '%s' (%uc%ua%ub) does not match the active DSWS_NCOMP/" + "DSWS_NAFEED/DSWS_NBFEED config (%uc%ua%ub) -- set the env vars to match or drop " + "the positional arg", g_posMixArg, pc, pa, pb, c.nComp, c.nAfeed, c.nBfeed); + } + } + if (dry) { printf(" [DSWS2_DRYRUN] params only -- NO GPU dispatch.\n"); rc = 0; } + else if (c.nComp < 1) { printf(" *** REFUSE: nComp>=1 required (compute floor); got %u ***\n", c.nComp); rc = 4; } + else if (c.nAfeed < 1) { printf(" *** REFUSE: nAfeed>=1 required (A-feed floor); got %u ***\n", c.nAfeed); rc = 4; } + else if (c.nBfeed < 1) { printf(" *** REFUSE: nBfeed>=1 required (B-feed floor); got %u ***\n", c.nBfeed); rc = 4; } + else if (c.N() != c.nComp + c.nAfeed + c.nBfeed) { + // role-floor/SUM check: N() (the launched wave count, WAVES_LAUNCH downstream) must equal the + // sum of the role counts actually used to size/decode the dispatch -- guards against a future + // refactor desyncing N() from its components (currently tautological by construction). + printf(" *** REFUSE: role-count sum mismatch (N()=%u != nComp+nAfeed+nBfeed=%u) ***\n", + c.N(), c.nComp + c.nAfeed + c.nBfeed); + rc = 4; + } else if (posMixBad && !getenv("DSWS2_FLOW")) { + printf(" *** REFUSE: %s ***\n", posMixMsg); + rc = 4; + } else if (Mo <= 0 || No <= 0 || Ko <= 0 || n_kseg <= 0 || TOTAL_super <= 0) { + printf(" *** REFUSE: degenerate geometry (Mo=%d No=%d Ko=%d n_kseg=%d TOTAL_super=%lld) ***\n", + Mo, No, Ko, n_kseg, TOTAL_super); + rc = 4; + } else if (TOTAL64 > 0xFFFFFFFFull || (uint64_t)TOTAL_super > 0xFFFFFFFFull) { + printf(" *** REFUSE: pool size overflows uint32_t (TOTAL=%llu TOTAL_super=%lld) -- occ[20]'s claim " + "counter and the kernel's sti are both 32-bit ***\n", (unsigned long long)TOTAL64, TOTAL_super); + rc = 4; + } else if (Gv != 6 || (SEGKv != 64 && !(getenv("DSWS2_FLOW") && SEGKv == 32))) { + // G/SEGK are compile-time defsyms baked into the kernel's instruction immediates + // (KSEG_STEPS-unrolled WMMA loop, resident-LDS strides). A host geometry that disagrees + // with the bin's compiled G/SEGK silently corrupts the resident-A/B staging/compute + // addressing (wrong strides, not a bounds violation the gate below would catch). REFUSE + // rather than guess; rebuild a matching bin before changing these envs. + // FIX 1 STAGGER: the flow bin (build_flow.sh) can now be built with SEGK=32 (halves the + // operand footprint so the g=6 write-once accumulator banks fit LDS) -- so SEGK=32 is + // allowed ONLY on the DSWS2_FLOW path, where the run must pass a matching DSWS2_SEGK=32. + printf(" *** REFUSE: DSWS2_G=%d DSWS2_SEGK=%d mismatches the built bin's compile-time geometry " + "(non-flow: G=6 SEGK=64; flow: G=6 SEGK in {32,64}) -- REFUSING geometry/bin mismatch ***\n", Gv, SEGKv); + rc = 4; + } else { + char dswsBin[160]; + // EMERGENT economy (flow): no baked mix. Derive the launch pool from FLOW_WAVES (host-set), + // cap at 30 (coordinator mailbox squat), sanity-check against the lean-fit budget. + uint32_t Wlaunch = getenv("FLOW_WAVES") ? (uint32_t)atoi(getenv("FLOW_WAVES")) : 8u; // 8 = proven-safe + if (Wlaunch < 4) Wlaunch = 4; // floor(wid0/1/2) + >=1 compute + // SAFETY: W>~8 at POOL_N=1 overcommits the SIMD dyn-VGPR pool at launch -> some waves' s_alloc_vgpr 32 + // fails -> the ONLY in-kernel exit (s_endpgm on a failed-alloc wave) corrupts the pool -> OOB page + // fault -> MODE1 brick (2026-07-05). Until that launch-starvation is root-caused, keep W_launch <= 8. + if (Wlaunch > 30) { printf(" [flow] FLOW_WAVES=%u > 30 (coord cap) -> clamping to 30\n", Wlaunch); Wlaunch = 30; } + { + const uint32_t VB = getenv("FLOW_VBUDGET") ? (uint32_t)atoi(getenv("FLOW_VBUDGET")) : 1536u; + const uint32_t leanFit = (VB - (112u - 32u)) / 32u; // (VBUDGET-(NFV-VLEAN))/VLEAN + if (Wlaunch > leanFit) + printf(" [flow] WARNING FLOW_WAVES=%u exceeds lean-fit=%u for VBUDGET=%u (bin's .error would catch a real overflow)\n", Wlaunch, leanFit, VB); + } + // FIX 1: DSWS2_FLOW -> flow bin, DSWS2_RING -> ring bin, else single-slot bin. + if (getenv("DSWS2_FLOW")) + snprintf(dswsBin, sizeof dswsBin, "occ_dsws2_w%u_flow_gd.bin", Wlaunch); // Wlaunch == built WAVES + else if (getenv("DSWS2_RING")) + snprintf(dswsBin, sizeof dswsBin, "occ_dsws2_%uc%ua%ub_ring_gd.bin", c.nComp, c.nAfeed, c.nBfeed); + else + snprintf(dswsBin, sizeof dswsBin, "occ_dsws2_%uc%ua%ub_gd.bin", c.nComp, c.nAfeed, c.nBfeed); + FILE* fb = fopen(dswsBin, "rb"); + if (fb) fclose(fb); + else { + printf(" *** DSWS2 kernel bin '%s' NOT BUILT -- REFUSING to dispatch (build it via " + "./build_dsws.sh, mk2) ***\n", dswsBin); + rc = 4; + } + if (rc == 0) { + const bool isFlow = getenv("DSWS2_FLOW"); // flow: launch Wlaunch (emergent), mix args unused + Dsws2Result o = run_dsws2(node, dswsBin, + isFlow ? Wlaunch : c.nComp, isFlow ? 0u : c.nAfeed, isFlow ? 0u : c.nBfeed, + Gv, SEGKv, FMc, FNc, Mo, No, Ko, orel, oabs, freq_hz); + if (!o.ok && o.okFrags == 0 && o.badFrags == 0) { + printf(" dsws2 INCOMPLETE (hang/timeout/refused before oracle) -> protocol/geometry bug; STOP\n"); + rc = 3; + } else if (o.badFrags > 0) { + printf(" dsws2 oracle *** BAD (race/math) *** -> STOP\n"); + rc = 3; + } else { + printf(" dsws2 oracle CLEAN\n"); + rc = 0; + } + } + } } else { // Default: dyn-VGPR cap probe. Test dyn correctness at increasing s_alloc footprints: // light NACC=8 -> s_alloc 96 (<=128, expected OK) diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_coop.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_coop.s index 3764964fee74..5a31a4e9dd53 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_coop.s +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_coop.s @@ -91,7 +91,49 @@ // (DIAG: computePhase=4, occ[10]=1 then wedge). s_sleep yields issue // cycles to the partner. Steady-state cost ~0 (loops spin ~0x when fed). -.set WAVES, (1 + P) // total waves launched per WG (harness dims must match) +// ---- DSWS (MAD-305 adaptive wave-role controller, SPEC_DSWS_CONTROLLER.md). Phase 1 = STATIC 3-role +// substrate {compute / A-feed / B-feed}, all NEW behavior gated behind `.if DSWS` so DSWS=0 stays +// BYTE-IDENTICAL to the proven 2-role coop d0 (1716B). Design (T1.2 decision, KG 4ce31886): +// * band-partitioned A: A-feed wave a owns strided bands {a, a+NAFEED, ...}; band c has ONE producer +// (prod_a[c]) and ONE consumer (compute c -> cons_a[c]). No min over consumers needed for A. +// * frag-partitioned B: B-feed wave b owns strided frags {b, b+NBFEED, ...} of the FN shared frags; +// each bumps prod_b[b]; compute gates on min(prod_b[*]). The lead B-feed (wid 0) is the SOLE tile +// claimer + ti/epoch broadcaster; A-feed + non-lead B-feed FOLLOW the broadcast like compute. +// * ZERO inter-feed rendezvous (the consumer-side min is the only cross-wave wait) -> the barrier-free +// invariant that keeps dyn-VGPR safe is preserved. ---- +.ifndef DSWS + .set DSWS, 0 +.endif +.if DSWS + .ifndef NCOMP + .set NCOMP, 4 // compute waves (fat, dyn-grow). Compute floor >= 1. + .endif + .ifndef NAFEED + .set NAFEED, 2 // A-feed waves (lean). Feed floor >= 1. + .endif + .ifndef NBFEED + .set NBFEED, 2 // B-feed waves (lean). Feed floor >= 1. + .endif + .ifndef RINGD_A + .set RINGD_A, RINGD // A-ring depth (defaults to the B-ring depth). MUST be power of two. + .endif + // ---- Controller thresholds (Phase 3 actuation; mirror occ_dispatch DSWS_LOW/HIGH/EPOCHSHIFT). ---- + .ifndef EPOCH_SHIFT + .set EPOCH_SHIFT, 3 // decision clock: epoch = segcnt >> EPOCH_SHIFT (small=reactive) + .endif + .ifndef CTRL_LOW + .set CTRL_LOW, 1 // occ_X < CTRL_LOW -> compute starved for X (shrink compute->feedX) + .endif + .ifndef CTRL_HIGH + .set CTRL_HIGH, (RINGD-1) // occ_X > CTRL_HIGH -> feed-X over-serving (grow feedX->compute) + .endif + // The existing compute body is parameterized by P = "#compute waves" (rowblk=trow*P+cid, store strides). + // Under DSWS the compute count is NCOMP -> bind P=NCOMP so all that addressing auto-targets NCOMP bands. + .set P, NCOMP + .set WAVES, (NCOMP + NAFEED + NBFEED) // total waves launched per WG (harness dims must match) +.else + .set WAVES, (1 + P) // total waves launched per WG (harness dims must match) +.endif // ---- LDS layout (bytes) -- matches run_mbcoop sizing: RINGD*FN*256 + 4*(1 + P + 3) ---- .set BRING_OFF, 0 // B ring: RINGD slots, each FN frags x 256B @@ -101,6 +143,29 @@ .set EPOCH_OFF, (TI_OFF + 4) // tile epoch (u32) -- feed bump publishes a new tile .set INITFLAG_OFF, (EPOCH_OFF + 4) // BUSYWAIT init-publish flag (the former spare u32) .set LDS_TOTAL, (EPOCH_OFF + 4 + 4) // + 1 spare u32 (now INITFLAG; sizing unchanged) +.if DSWS + // ---- DSWS extra LDS, all APPENDED after the DSWS=0 region so the proven offsets stay byte-identical ---- + // prod_b[0] reuses PROD_OFF (the frag-partition lead); prod_b[1..NBFEED-1] live just past INITFLAG. + .set PRODB_HI_OFF, LDS_TOTAL // prod_b[1..NBFEED-1] : NBFEED-1 u32 (0 bytes when NBFEED=1) + .set ARING_OFF, (PRODB_HI_OFF + 4*(NBFEED-1)) // A-ring: RINGD_A slots x NCOMP bands x FM frags x 256B + .set PROD_A_OFF, (ARING_OFF + RINGD_A*NCOMP*FM*256) // prod_a[NCOMP] (band-partitioned producers) + .set CONS_A_OFF, (PROD_A_OFF + 4*NCOMP) // cons_a[NCOMP] (compute A release; one consumer per band) + // ---- Controller state (Phase 2 sensing/slots/reservation; Phase 3 actuates). Mirrors dsws_ctrl_model.cpp. ---- + // Role slots: live count of waves currently in each role (leader inits to the launch mix; conversions adjust). + .set NCOMP_SLOT, (CONS_A_OFF + 4*NCOMP) // u32 nComp (live fat-compute wave count) + .set NAFEED_SLOT, (NCOMP_SLOT + 4) // u32 nAfeed (live A-feed wave count) + .set NBFEED_SLOT, (NAFEED_SLOT + 4) // u32 nBfeed (live B-feed wave count) + // Conversion gates: gate[dir] holds the last epoch in which direction `dir` fired (gate_try_win CAS target). + // 4 directions: 0=compute->Afeed, 1=compute->Bfeed, 2=Afeed->compute, 3=Bfeed->compute. + .set GATE_OFF, (NBFEED_SLOT + 4) // u32[4] gate[dir] + .set VRESV_OFF, (GATE_OFF + 4*4) // u32 vgpr_reserved (sum-envelope: reserve_grow target) + .set SEGCNT_OFF, (VRESV_OFF + 4) // u32 segments_processed (per-WG decision clock source) + .set LDS_TOTAL_DSWS, (SEGCNT_OFF + 4) + .set ABAND_STRIDE, (RINGD_A*FM*256) // bytes per compute band's A sub-ring + .if LDS_TOTAL_DSWS > 65536 + .error "DSWS LDS exceeds 64KB/WG (gfx1201): reduce RINGD_A / NCOMP / tile" + .endif +.endif // ---- VGPR layout ---- .set ACC, 32 // accumulators: FM*FN frags x 8 f32 (compute; above the lean block) @@ -109,7 +174,16 @@ // next_free_vgpr ROUNDED to a 16-VGPR dyn-alloc block. 2x4 -> 112 (highest index 107 needs >=108): a clean // 7-block alloc, SAFELY below the 128 cap and OFF the 128 exact-fill edge (the strategic dyn danger zone). .set NFV, ((FB + 2*FN + 15) & ~15) +.if DSWS + // Controller reservation-envelope units (hardware ALLOC footprint, not live-peak): a fat-compute wave + // reserves NFV (its s_alloc_vgpr target); a lean feed wave reserves VLEAN (its `s_alloc_vgpr 32`). + // Phase-3 grow delta = NFV - VLEAN; vgpr_reserved init (Phase 2) = NCOMP*NFV + (NAFEED+NBFEED)*VLEAN. + .set VLEAN, 32 +.endif .set BSTG, 16 // feed B staging: FN x 2 at v16.. (lean block, < 32) +.if DSWS + .set ASTG, 16 // A-feed staging: FM x 2 at v16.. (A-feed is a distinct lean wave, +.endif // so it reuses the same lean block as the B-feed's BSTG) // ---- dyn-VGPR PRE-GROW VGPR CEILING (THE deadlock + dead-marks root cause; Codex + RDNA4 ISA confirmed) ---- // A dyn-armed (RSRC2 bit6) wave LAUNCHES with exactly ONE 16-VGPR block backed (v0..v15); RSRC1.VGPRS is @@ -191,6 +265,69 @@ s_mov_b32 exec_lo, s58 .endif .endm + +.macro mark_set off, val // DIAG: lane0-of-wave PLAIN-store \val -> occ[off] (INSTANTANEOUS, overwrites). +.if DIAG // Unlike `mark` (atomic-MAX = furthest-reached), this streams the CURRENT + s_mov_b32 s58, exec_lo // value so the harness's 200ms poll sees a sensor oscillate (occ_a/occ_b). + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lms\@ + v_mov_b32 v[RM_A], 0 // self-zeroed vaddr (same robustness as mark) + v_mov_b32 v[RM_D], \val + global_store_b32 v[RM_A], v[RM_D], s[0:1] offset:\off scope:SCOPE_DEV +.Lms\@: + s_mov_b32 exec_lo, s58 +.endif +.endm + +.macro mark_inc off // DIAG: lane0-of-wave atomic-ADD 1 -> occ[off] (a free-running counter). +.if DIAG + s_mov_b32 s58, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lmi\@ + v_mov_b32 v[RM_A], 0 + v_mov_b32 v[RM_D], 1 + global_atomic_add_u32 v[RM_A], v[RM_D], s[0:1] offset:\off scope:SCOPE_DEV +.Lmi\@: + s_mov_b32 exec_lo, s58 +.endif +.endm + +// try_gate: the lock-free single-winner conversion ticket (transcribes dsws_ctrl_model.cpp gate_try_win + +// epoch_of EXACTLY). E = segcnt>>EPOCH_SHIFT. gate[dir] holds the last epoch dir fired. Among many waves +// racing the same (g MEM=(MEM==vCMP)?vNEW:MEM, vdst<-old. So vsrc0=E (new), +// vsrc1=g (compare). WIN iff returned-old == g. (T3.1 micro-check CAUGHT the swapped form: it left gate +// stuck at 0 so old==g held for all racers -> would-win ~= NCOMP*epochs instead of ~= epochs.) +.macro try_gate dir, swin + lds_get s62, SEGCNT_OFF // E = epoch_of(segcnt, EPOCH_SHIFT) + s_lshr_b32 s62, s62, EPOCH_SHIFT + lds_get s63, (GATE_OFF + (\dir)*4) // g = gate[dir] + s_mov_b32 \swin, 0 + s_cmp_ge_u32 s63, s62 // g >= E -> dir already fired this/later epoch -> lose + s_cbranch_scc1 .Ltg_done\@ + s_mov_b32 s65, exec_lo // lane0-only CAS (one ticket attempt per WAVE) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltg_restore\@ + v_mov_b32 v5, (GATE_OFF + (\dir)*4) // vaddr = &gate[dir] + v_mov_b32 v6, s63 // v6 = g (vsrc1 = CMP/expected) + v_mov_b32 v7, s62 // v7 = E (vsrc0 = NEW value to store) + ds_cmpstore_rtn_b32 v6, v5, v7, v6 // gate[dir] = (gate[dir]==g) ? E : gate[dir]; v6 <- old + s_wait_dscnt 0x0 +.Ltg_restore\@: + s_mov_b32 exec_lo, s65 + v_readfirstlane_b32 s64, v6 // s64 = old (lane0's CAS result, broadcast) + s_cmp_eq_u32 s64, s63 // WIN iff old == g (we were the swapper) + s_cbranch_scc0 .Ltg_done\@ + s_mov_b32 \swin, 1 +.Ltg_done\@: +.endm + .macro min_cons sdst // sdst = min over c in 0..P-1 of cons_count[c] lds_get \sdst, CONS_OFF .set cc, 1 @@ -200,6 +337,23 @@ .set cc, cc+1 .endr .endm +.if DSWS +.macro lds_get_r sdst, saddr // wave-uniform read LDS[saddr] -> scalar sdst (RUNTIME addr in a sreg) + v_mov_b32 v[RG_A], \saddr + ds_load_b32 v[RG_D], v[RG_A] + s_wait_dscnt 0x0 + v_readfirstlane_b32 \sdst, v[RG_D] +.endm +.macro min_prod sdst // sdst = min over b in 0..NBFEED-1 of prod_b[b] (frag-partitioned B). + lds_get \sdst, PROD_OFF // prod_b[0] is PROD_OFF; prod_b[1..] live in the DSWS hi region. + .set pw, 1 + .rept NBFEED-1 + lds_get s46, (PRODB_HI_OFF + (pw-1)*4) + s_min_u32 \sdst, \sdst, s46 + .set pw, pw+1 + .endr +.endm +.endif .text .globl occ_kernel @@ -274,6 +428,49 @@ occ_kernel: .endr v_mov_b32 v[RI_A], EPOCH_OFF ds_store_b32 v[RI_A], v[RI_D] +.if DSWS + // zero the DSWS monotonic counters: prod_b[1..NBFEED-1], prod_a[NCOMP], cons_a[NCOMP] (start at 0). + // (the A-ring storage needs no zeroing -- it is overwritten before read, gated by prod_a.) + .set zw, 0 + .rept (NBFEED-1) + v_mov_b32 v[RI_A], (PRODB_HI_OFF + zw*4) + ds_store_b32 v[RI_A], v[RI_D] + .set zw, zw+1 + .endr + .set zc, 0 + .rept NCOMP + v_mov_b32 v[RI_A], (PROD_A_OFF + zc*4) + ds_store_b32 v[RI_A], v[RI_D] + v_mov_b32 v[RI_A], (CONS_A_OFF + zc*4) + ds_store_b32 v[RI_A], v[RI_D] + .set zc, zc+1 + .endr + // ---- Controller state init (Phase 2): role slots <- launch mix, gates <- 0, vgpr_reserved <- launch + // envelope footprint, segcnt <- 0. Leader-lane writes constants to WG-shared LDS (idempotent across + // waves: every lane-0 writes identical compile-time values). v[RI_D] still holds 0 from above. ---- + v_mov_b32 v[RI_A], NCOMP_SLOT + v_mov_b32 v[RI_D], NCOMP + ds_store_b32 v[RI_A], v[RI_D] + v_mov_b32 v[RI_A], NAFEED_SLOT + v_mov_b32 v[RI_D], NAFEED + ds_store_b32 v[RI_A], v[RI_D] + v_mov_b32 v[RI_A], NBFEED_SLOT + v_mov_b32 v[RI_D], NBFEED + ds_store_b32 v[RI_A], v[RI_D] + v_mov_b32 v[RI_D], 0 + .set zg, 0 + .rept 4 + v_mov_b32 v[RI_A], (GATE_OFF + zg*4) // gate[dir] = 0 (no conversion has fired) + ds_store_b32 v[RI_A], v[RI_D] + .set zg, zg+1 + .endr + v_mov_b32 v[RI_A], VRESV_OFF // vgpr_reserved = NCOMP*NFV + (NAFEED+NBFEED)*VLEAN + v_mov_b32 v[RI_D], (NCOMP*NFV + (NAFEED+NBFEED)*VLEAN) + ds_store_b32 v[RI_A], v[RI_D] + v_mov_b32 v[RI_A], SEGCNT_OFF // segments_processed = 0 + v_mov_b32 v[RI_D], 0 + ds_store_b32 v[RI_A], v[RI_D] +.endif s_wait_dscnt 0x0 .Linit_done: s_mov_b32 exec_lo, s16 @@ -305,6 +502,19 @@ occ_kernel: mark 108, 1 // DIAGFINE occ[27] = reached post-rendezvous (passed s_barrier) .endif +.if DSWS + // ===== DSWS 3-role branch (wid uniform per wave; scalar-only -> exec stays full for every role). + // wid [0,NBFEED) -> B-feed (wid 0 = lead: sole tile-claimer + ti/epoch broadcaster) + // wid [NBFEED,NBFEED+NAFEED) -> A-feed + // wid [NBFEED+NAFEED, WAVES) -> compute, cid = wid - (NBFEED+NAFEED) ===== + v_readfirstlane_b32 s24, v1 // wid (uniform per wave) + s_cmp_lt_u32 s24, NBFEED + s_cbranch_scc1 .Lbfeed_role + s_cmp_lt_u32 s24, (NBFEED+NAFEED) + s_cbranch_scc1 .Lafeed_role + s_sub_u32 s47, s24, (NBFEED+NAFEED) // cid + s_branch .Ldsws_compute_init +.else // ============ ROLE BRANCH: wave 0 -> feed ; waves 1..P -> compute ============ v_cmp_eq_u32 vcc_lo, 0, v1 s_mov_b32 s25, exec_lo @@ -314,19 +524,44 @@ occ_kernel: s_branch .Lfeed_role .Lcompute_role: s_mov_b32 exec_lo, s25 // only compute waves reach here +.endif // ======================================================================================== - // COMPUTE BODY (waves 1..P). cid = wid-1. Owns M-band [cid*FM*16 ..). Shares the FN N-cols. + // COMPUTE BODY (waves 1..P). cid = wid-1 (DSWS=0) or wid-(NBFEED+NAFEED) (DSWS). Owns M-band + // [cid*FM*16 ..). Shares the FN N-cols. DSWS: also consumes its A-band from the A-ring. // ======================================================================================== +.if DSWS == 0 v_sub_nc_u32 v3, v1, 1 v_readfirstlane_b32 s47, v3 // cid (uniform per wave) +.endif +.Ldsws_compute_init: // DSWS path joins here with s47=cid already set // cons_count[cid] LDS address (runtime): v12 = CONS_OFF + cid*4 s_lshl_b32 s36, s47, 2 s_add_u32 s36, s36, CONS_OFF v_mov_b32 v12, s36 +.if DSWS + // per-band A-consume state: &prod_a[cid], &cons_a[cid], A-band ring base, a_step (cumulative A consume). + s_lshl_b32 s52, s47, 2 + s_add_u32 s52, s52, PROD_A_OFF // &prod_a[cid] + s_lshl_b32 s53, s47, 2 + s_add_u32 s53, s53, CONS_A_OFF // &cons_a[cid] + s_mul_i32 s51, s47, ABAND_STRIDE + s_add_u32 s51, s51, ARING_OFF // A-band ring base (this cid's sub-ring) + s_mov_b32 s54, 0 // a_step = 0 (cumulative A consume; matches producer prod_a[cid]) +.endif s_mov_b32 s35, 0 // local_epoch = 0 (matches the init epoch) s_mov_b32 s57, 0 // tiles processed (POOL=1 count-to-TOTAL terminal, see below) s_mov_b32 s56, 0 // GLOBAL cons step (cumulative consumed; MONOTONIC, matches feed s55) +.if DSWS + // ---- Phase-2 sensing: stream the live role slots (read-only; static in Phase 2) once per compute wave. + // Proves the leader's controller-state init round-trips through WG-shared LDS into the occ snapshot. + lds_get s59, NCOMP_SLOT + mark_set 136, s59 // occ[34] = nComp (live fat-compute wave count) + lds_get s59, NAFEED_SLOT + mark_set 140, s59 // occ[35] = nAfeed (live A-feed wave count) + lds_get s59, NBFEED_SLOT + mark_set 144, s59 // occ[36] = nBfeed (live B-feed wave count) +.endif .Lcompute_loop: mark 28, 1 @@ -413,10 +648,20 @@ occ_kernel: .if DYNVGPR s_sleep SLEEPN // yield to the lean feed so it can publish the next B-step .endif +.if DSWS + min_prod s44 // B ready = min over frag-partitioned producers prod_b[*] > s56 +.else lds_get s44, PROD_OFF +.endif s_cmp_le_u32 s44, s56 s_cbranch_scc1 .Lwait_prod mark 56, s56 // DIAGFINE occ[14] = max step that PASSED the prod-wait (about to ds_load B) +.if DSWS + // Phase-2 sensor: occ_b = prod_b_min - cons, sampled HERE (consume point, pre-release) where it is the + // live ring backlog and -- by the gate above (prod>cons) -- provably in [1, RINGD]. Read-only, NO action. + s_sub_u32 s59, s44, s56 + mark_set 128, s59 // occ[32] = occ_b (instantaneous B-ring backlog, [1,RINGD]) +.endif // ring slot = s56 & (RINGD-1) (GLOBAL step -> matches the feed's continuous ring) s_and_b32 s45, s56, (RINGD-1) s_mul_i32 s45, s45, (FN*256) @@ -433,6 +678,35 @@ occ_kernel: s_add_u32 s56, s56, 1 lds_put_v v12, s56 mark 40, s56 // DIAG: occ[10] = max GLOBAL cons step reached (KT*32=1024 = all consumed) +.if DSWS + // ---- consume FM own A frags from the A-ring band cid (gated on prod_a[cid] > a_step); release cons_a[cid] ---- +.Lwait_proda: + .if DYNVGPR + s_sleep SLEEPN // yield to the lean A-feed so it can publish the next A-step + .endif + lds_get_r s44, s52 // prod_a[cid] + s_cmp_le_u32 s44, s54 + s_cbranch_scc1 .Lwait_proda + // Phase-2 sensor: occ_a = prod_a[cid] - a_step, sampled HERE (A-consume point) where the gate guarantees + // prod_a > a_step -> occ_a in [1, RINGD_A]. Read-only, NO action. + s_sub_u32 s59, s44, s54 + mark_set 132, s59 // occ[33] = occ_a (instantaneous A-band backlog, [1,RINGD_A]) + // A-ring slot addr = A-band base (s51) + (a_step & (RINGD_A-1))*FM*256, per-lane lane*8 (v9) + s_and_b32 s45, s54, (RINGD_A-1) + s_mul_i32 s45, s45, (FM*256) + s_add_u32 s45, s45, s51 + v_add_nc_u32 v13, v9, s45 + .set mi, 0 + .rept FM + ds_load_b64 v[FA+mi*2:FA+mi*2+1], v13 offset:mi*256 + .set mi, mi+1 + .endr + s_wait_dscnt 0x0 + // RELEASE the A slot now (A is in our regs, before WMMA): cons_a[cid] = a_step+1 + s_add_u32 s54, s54, 1 + v_mov_b32 v13, s53 + lds_put_v v13, s54 +.else // direct-load FM own A frags; advance A saddrs +16 (next K16) .set mi, 0 .rept FM @@ -446,6 +720,7 @@ occ_kernel: .set mi, mi+1 .endr s_wait_loadcnt 0x0 +.endif mark 64, s56 // DIAGFINE occ[16] = max step with A global_load'd (about to WMMA) // FM*FN accumulating WMMA .set mi, 0 @@ -483,6 +758,28 @@ occ_kernel: // concurrent WG memory traffic -> it hung forever. Signal done + move on, per the WAR being moot here.) ---- s_add_u32 s57, s57, 1 // counted one more tile (issued its stores) mark 44, s57 // DIAG: occ[11] = tiles the compute COMPLETED (issued+retired) +.if DSWS + // ---- Phase-2 per-WG segments_processed bump (the controller's decision clock): lead compute (cid==0) + // owns it (POOL=1: one tile = one WG segment, all cids lock-step on the broadcast ti). Single writer. + // occ_a/occ_b are sampled at the per-K CONSUME points (where the ring is mid-flight), NOT here -- at + // the segment boundary the ring has fully DRAINED so occ reads ~0, which is both uninformative for + // validation and the wrong signal for the eventual controller (would always read "starved"). ---- + s_cmp_eq_u32 s47, 0 + s_cbranch_scc0 .Lseg_done + lds_get s44, SEGCNT_OFF + s_add_u32 s44, s44, 1 + lds_put SEGCNT_OFF, s44 +.Lseg_done: + // ---- T3.1 UNIT (no actuation): every compute wave races the (compute->Afeed, dir=0) ticket each segment. + // The LDS-CAS in try_gate must yield <=1 winner per (dir,epoch) across all NCOMP racers -> the would-win + // counter occ[39] should land near #epochs (= segcnt>>EPOCH_SHIFT range), NOT NCOMP*#epochs. This is the + // harness-side validation of the gate-CAS before 3.2/3.3 wire real role conversions onto \swin. ---- + try_gate 0, s59 + s_cmp_eq_u32 s59, 1 + s_cbranch_scc0 .Ltg_unit_done + mark_inc 156 // occ[39] = would-win count (compute->Afeed ticket) +.Ltg_unit_done: +.endif s_cmp_lt_u32 s57, s11 // processed < TOTAL ? s_cbranch_scc0 .Lcompute_exit // terminal -> skip store-wait; dispatch fence drains the stores s_wait_storecnt 0x0 // non-terminal: stores MUST drain before the next tile re-zeros ACC @@ -506,10 +803,259 @@ occ_kernel: .endif s_endpgm // (NOFENCE path still completes on occ[0]==0 regardless of the fence) +.if DSWS + // ======================================================================================== + // DSWS B-FEED (frag-partitioned producers). wid 0 = LEAD: sole tile-claimer + ti/epoch broadcaster. + // Every B-feed wave (b_id=wid) owns the strided frag subset {ni : ni%NBFEED==b_id} of the FN shared + // frags, fills them into the shared B-ring slot, and bumps ITS OWN prod_b[b_id]. Compute gates on + // min(prod_b[*]) so no B-feed ever waits on another B-feed (consumer-side min = barrier-free). + // ======================================================================================== +.Lbfeed_role: + mark 24, 1 // DIAG occ[6] feedPhase=1: B-feed role entered + s_mov_b32 s55, 0 // GLOBAL prod step (cumulative; monotonic, never reset) + .if DYNVGPR +.Lbfeed_lean_alloc: + s_alloc_vgpr 32 // commit this lean B-feed at 32 (dyn WG-allocator consistency) + s_cbranch_scc0 .Lbfeed_lean_alloc + .endif + // &prod_b[b_id]: lead (b_id==0) -> PROD_OFF ; else -> PRODB_HI_OFF + (b_id-1)*4 + s_sub_u32 s50, s24, 1 + s_lshl_b32 s50, s50, 2 + s_add_u32 s50, s50, PRODB_HI_OFF + s_mov_b32 s48, PROD_OFF + s_cmp_eq_u32 s24, 0 + s_cselect_b32 s50, s48, s50 + s_mov_b32 s35, 0 // local epoch (non-lead follow clock) +.Lbfeed_loop: + s_cmp_eq_u32 s24, 0 + s_cbranch_scc0 .Lbfeed_follow // non-lead: follow the lead's broadcast + // ---- LEAD: claim BATCH=1 tile (the ONE device-scope atomic), publish ti, bump epoch ---- + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lbfeed_after_grab + v_mov_b32 v3, BATCH + global_atomic_add_u32 v5, v4, v3, s[0:1] offset:20 th:TH_ATOMIC_RETURN scope:SCOPE_DEV + s_wait_loadcnt 0x0 +.Lbfeed_after_grab: + s_mov_b32 exec_lo, s16 + v_readfirstlane_b32 s17, v5 // ti + lds_put TI_OFF, s17 // publish ti FIRST... + lds_get s44, EPOCH_OFF + s_add_u32 s44, s44, 1 + s_mov_b32 s35, s44 + lds_put EPOCH_OFF, s44 // ...then bump epoch (followers see ti before the bump) + s_branch .Lbfeed_have_ti +.Lbfeed_follow: + .if DYNVGPR + s_sleep SLEEPN + .endif + lds_get s44, EPOCH_OFF + s_cmp_eq_u32 s44, s35 + s_cbranch_scc1 .Lbfeed_follow // spin until the lead bumps epoch + s_mov_b32 s35, s44 + lds_get s17, TI_OFF +.Lbfeed_have_ti: + s_cmp_ge_u32 s17, s11 // terminal tile? (broadcast already done -> all roles retire) + s_cbranch_scc1 .Lfeed_exit +.if SAFEPROBE + // SAFEPROBE: clamp the broadcast ti -> [0,TOTAL-1] before B-saddr decode (mirrors compute lines 471-472). + // Pairs with the common-prologue v9 vaddr clamp so EVERY B global address is provably in-buffer even if + // the broadcast read returns a garbage ti (defense-in-depth; terminal test above already retires ti>=TOTAL). + s_sub_u32 s36, s11, 1 // TOTAL-1 (s36 dead here; B-feed recomputes it nowhere) + s_min_u32 s17, s17, s36 +.endif + mark 24, 2 // DIAG occ[6] feedPhase=2: B-feed has a (clamped) ti + mark 32, s17 // DIAG occ[8] feedTi: the broadcast/claimed ti B is producing + // ---- B col-tile saddr (k=0): Bshuf + tcol*(FN*256) ---- + s_mul_hi_u32 s19, s17, s12 + s_mul_i32 s18, s19, s13 + s_sub_u32 s18, s17, s18 // tcol + s_mul_i32 s20, s18, s14 + s_add_u32 s20, s4, s20 + s_addc_u32 s21, s5, 0 + s_mov_b32 s26, 0 // per-tile k +.Lbprod_k: + // slot-free gate (GLOBAL step) on min compute cons: don't overwrite an undrained slot + s_cmp_lt_u32 s55, RINGD + s_cbranch_scc1 .Lbslot_ok + s_sub_u32 s45, s55, RINGD +.Lbwait_slot: + .if DYNVGPR + s_sleep SLEEPN + .endif + min_cons s44 + s_cmp_le_u32 s44, s45 + s_cbranch_scc1 .Lbwait_slot +.Lbslot_ok: + mark 72, s55 // DIAGFINE occ[18] slotok: B step that passed the slot-free gate + s_and_b32 s45, s55, (RINGD-1) + s_mul_i32 s45, s45, (FN*256) + v_add_nc_u32 v13, v9, s45 // ring vaddr base (lane*8 + slot) + // pass 1: global_load_tr owned frags into staging (compile-time owner; runtime guard on b_id) + .set ni, 0 + .rept FN + .set owner, ni % NBFEED + s_cmp_eq_u32 s24, owner + s_cbranch_scc0 1f + global_load_tr_b64 v[BSTG+ni*2:BSTG+ni*2+1], v9, s[20:21] offset:ni*256 +1: + .set ni, ni+1 + .endr + s_wait_loadcnt 0x0 + mark 76, s55 // DIAGFINE occ[19] loadtr: B step global_load_tr'd into staging + // pass 2: ds_store owned frags into the ring slot + .set ni, 0 + .rept FN + .set owner, ni % NBFEED + s_cmp_eq_u32 s24, owner + s_cbranch_scc0 2f + ds_store_b64 v13, v[BSTG+ni*2:BSTG+ni*2+1] offset:ni*256 +2: + .set ni, ni+1 + .endr + s_wait_dscnt 0x0 + mark 80, s55 // DIAGFINE occ[20] dsstore: B step ds_store'd into ring (pre-publish) + // publish prod_b[b_id] = s55+1 (this wave's owned frags for this step are now visible) + s_add_u32 s55, s55, 1 + v_mov_b32 v13, s50 + lds_put_v v13, s55 + mark 84, s55 // DIAGFINE occ[21] publish: max prod_b count PUBLISHED by B-feed + s_add_u32 s20, s20, s10 // advance B saddr += NT*256 + s_addc_u32 s21, s21, 0 + s_add_u32 s26, s26, 1 + s_cmp_lt_u32 s26, s8 + s_cbranch_scc1 .Lbprod_k + // lead drains (waits compute consumed) before next claim; non-lead just loops to wait next epoch + s_cmp_eq_u32 s24, 0 + s_cbranch_scc0 .Lbfeed_loop +.Lbdrain: + .if DYNVGPR + s_sleep SLEEPN + .endif + min_cons s44 + s_cmp_lt_u32 s44, s55 + s_cbranch_scc1 .Lbdrain + s_branch .Lbfeed_loop + + // ======================================================================================== + // DSWS A-FEED (band-partitioned producers). FOLLOWS the lead's ti/epoch broadcast (like compute). + // A-feed wave a_id owns the strided band subset {bnd : bnd%NAFEED==a_id}; band bnd has ONE producer + // (this wave) and ONE consumer (compute bnd). prod_a[bnd] doubles as the band's cumulative step. + // ======================================================================================== +.Lafeed_role: + s_sub_u32 s24, s24, NBFEED // a_id = wid - NBFEED + .if DYNVGPR +.Lafeed_lean_alloc: + s_alloc_vgpr 32 + s_cbranch_scc0 .Lafeed_lean_alloc + .endif + s_mov_b32 s35, 0 // local epoch (follow) + s_mov_b32 s60, 0 // GLOBAL A-step (gk): cumulative across tiles, mirrors compute s54 + // and the B-ring's continuous step. The per-tile k (s26) is for + // A-MATRIX addressing only; the ring protocol (publish/slot/gate) + // MUST be global or tile>=1 re-publishes prod_a<=a_step -> A-starve. +.Lafeed_loop: + .if DYNVGPR + s_sleep SLEEPN + .endif + lds_get s44, EPOCH_OFF + s_cmp_eq_u32 s44, s35 + s_cbranch_scc1 .Lafeed_loop + s_mov_b32 s35, s44 + lds_get s17, TI_OFF + s_cmp_ge_u32 s17, s11 // terminal? -> retire (role-agnostic exit) + s_cbranch_scc1 .Lfeed_exit +.if SAFEPROBE + // SAFEPROBE: clamp the broadcast ti -> [0,TOTAL-1] before A-saddr decode (mirrors compute lines 471-472). + // s36 is dead here (the per-band loop recomputes it from scratch); pairs with the common-prologue v8 + // vaddr clamp so EVERY A global address is provably in-buffer even on a garbage broadcast ti. + s_sub_u32 s36, s11, 1 // TOTAL-1 + s_min_u32 s17, s17, s36 +.endif + mark 112, 1 // DIAGFINE occ[28] Afeed: received broadcast ti, decoding bands + s_mul_hi_u32 s19, s17, s12 // trow = mul_hi(ti, magic) + // ===== K-OUTER / band-INNER (the feed-starvation fix, 2026-06-29): the OLD band-outer/K-inner loop drained + // band b's ENTIRE K-loop before touching band b+NAFEED, so when an A-feed wave owns >1 band the not-yet-fed + // compute bands starved -> min_cons stalled -> the RINGD-deep B-ring jammed -> WG wedged at ~step RINGD + // (confirmed: 2c3a3b NAFEED>=NCOMP greened; 4c2a2b/6c1a1b hung). NOW: every K-step produces ALL owned bands + // one step, so all compute bands advance lock-step with the ring (mirrors B-feed's K-outer/frag-inner). + // With lock-step production astep == k for every owned band -> use s26 (k) directly; no per-band astep state. + s_lshl_b32 s32, s9, 4 // 16*K (constant across the tile; per-frag stride) + s_mov_b32 s26, 0 // K-OUTER counter k (== astep for every owned band) +.LafeedK: + .set bnd, 0 + .rept NCOMP + .set owner, bnd % NAFEED + s_cmp_eq_u32 s24, owner + s_cbranch_scc0 3f // not owned by this A-feed wave -> skip for this k + // slot-free gate (band bnd at GLOBAL step s60): if gk>=RINGD_A wait cons_a[bnd] > gk-RINGD_A + s_cmp_lt_u32 s60, RINGD_A + s_cbranch_scc1 6f + s_sub_u32 s45, s60, RINGD_A +5: + .if DYNVGPR + s_sleep SLEEPN + .endif + lds_get s46, (CONS_A_OFF + bnd*4) + s_cmp_le_u32 s46, s45 + s_cbranch_scc1 5b +6: + // recompute A saddrs for (band bnd, step k): rowblk=trow*NCOMP+bnd ; + // saddr(mi) = A + rowblk*(16*FM)*K + mi*16*K + k*16 + s_mul_i32 s36, s19, NCOMP + s_add_u32 s36, s36, bnd + s_mul_i32 s22, s36, (16*FM) + s_mul_i32 s22, s22, s9 + s_lshl_b32 s44, s26, 4 // k*16 + s_add_u32 s22, s22, s44 + s_add_u32 s40, s2, s22 + s_addc_u32 s41, s3, 0 + .set mi2, 1 + .rept FM-1 + s_add_u32 s[40+2*mi2], s[40+2*(mi2-1)], s32 + s_addc_u32 s[41+2*mi2], s[41+2*(mi2-1)], 0 + .set mi2, mi2+1 + .endr + // global_load FM A-frags + .set mi3, 0 + .rept FM + global_load_b64 v[ASTG+mi3*2:ASTG+mi3*2+1], v8, s[40+2*mi3:41+2*mi3] + .set mi3, mi3+1 + .endr + s_wait_loadcnt 0x0 + mark 116, s26 // DIAGFINE occ[29] Afeed: step global_load'd from A + // ds_store into A-ring band bnd, slot (GLOBAL gk & (RINGD_A-1)); base folds ARING_OFF+bnd*ABAND_STRIDE + s_and_b32 s45, s60, (RINGD_A-1) + s_mul_i32 s45, s45, (FM*256) + s_add_u32 s45, s45, (ARING_OFF + bnd*ABAND_STRIDE) + v_add_nc_u32 v13, v9, s45 + .set mi3, 0 + .rept FM + ds_store_b64 v13, v[ASTG+mi3*2:ASTG+mi3*2+1] offset:mi3*256 + .set mi3, mi3+1 + .endr + s_wait_dscnt 0x0 + mark 120, s26 // DIAGFINE occ[30] Afeed: step ds_store'd into A-ring (pre-publish) + // publish prod_a[bnd] = gk+1 (GLOBAL cumulative count -> matches compute's global a_step consumer) + s_add_u32 s54, s60, 1 + lds_put (PROD_A_OFF + bnd*4), s54 + mark 124, s54 // DIAGFINE occ[31] Afeed: max prod_a count PUBLISHED +3: + .set bnd, bnd+1 + .endr + s_add_u32 s26, s26, 1 // per-tile k (A-matrix addressing) + s_add_u32 s60, s60, 1 // GLOBAL gk (ring protocol; never resets across tiles) + s_cmp_lt_u32 s26, s8 // k < KT ? + s_cbranch_scc1 .LafeedK + s_branch .Lafeed_loop +.endif // DSWS feed bodies + // ======================================================================================== // FEED BODY (wave 0). Claims a tile, publishes (ti, reset counters, epoch++), produces the FN shared // B-frags per K16-step into the ring, then drains (waits all P compute) before claiming the next tile. - // (v1 = tile-synchronous; cross-tile B-ring overlap = FUTURE ENHANCEMENT FE-1.) + // (v1 = tile-synchronous; cross-tile B-ring overlap = FUTURE ENHANCEMENT FE-1.) DSWS=1: DEAD (the DSWS + // role branch never reaches it) but kept assembled so .Lfeed_exit stays the shared retire path. // ======================================================================================== .Lfeed_role: s_mov_b32 s55, 0 // GLOBAL prod step (cumulative B-steps published; MONOTONIC, diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s index 4dc31f40003e..2bd7c5899dfc 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws.s @@ -77,6 +77,43 @@ .ifndef SAFEPROBE .set SAFEPROBE, 0 // 1 = clamp per-lane vector address regs into a provable in-buffer bound .endif +.ifndef TFPROBE + .set TFPROBE, 0 // 1 = emit s_sendmsg_rtn GET_REALTIME wall-span capture (each wave stamps +.endif // occ[2]=min entry tick, occ[3]=max exit tick -> host TF readout). + // Default 0 => ZERO bytes emitted, .text byte-identical to the + // production bin (uses only already-allocated regs s30/s31/s49/v5). +.ifndef PHASEPROBE + .set PHASEPROBE, 0 // 1 = in-kernel PHASE TIMER: each compute wave stamps GET_REALTIME at every +.endif // phase boundary and atomic-adds the delta into per-phase occ accumulators + // (occ[64..69], bytes 256..276, ABOVE the per-chunk memset -> accumulate over + // the whole run). Host prints ticks + % per phase. Default 0 => byte-identical. +.ifndef NOCFLUSH + .set NOCFLUSH, 0 // PERF PROBE ONLY: 1 = skip the global_atomic_add_f32 C-flush loop (keep ALL +.endif // other bookkeeping/handshake). Isolates the device-atomic C-reduction cost + // from the coordination handshake. Result is WRONG (C never written -> oracle + // fails) -- span/TF only. Default 0 => byte-identical. +.ifndef KMAJOR + .set KMAJOR, 0 // PERF PROBE: 1 = K-MAJOR super-tile traversal. Default decode packs ksi in +.endif // the LOW bits (all n_kseg segments of a C cell claimed consecutively -> + // up to n_kseg WGs hammer one C cell at once = max atomic contention). + // KMAJOR decodes ksi = sti / TOTAL (high), t = sti % TOTAL (low) via a + // magic-div (magic_TOTAL from occ[62], loaded to s76 in prologue) -> the + // 32 segments of a cell are spread TOTAL apart in claim order (near-zero + // concurrent contention) + adjacent tiles (shared operand bands) claimed + // close in time (L2 reuse). Correctness-preserving (C add is commutative). +.ifndef CSTORE + .set CSTORE, 0 // PERF PROBE ONLY: 1 = replace the flush's global_atomic_add_f32 with an +.endif // equal-count NON-atomic global_store_b32 (same #mem-ops, same addresses, NO + // RMW/contention). Isolates atomic-contention from raw write-bandwidth: if + // TF jumps vs atomics -> contention; if flat -> bandwidth-bound. Result WRONG + // (last-writer, no accumulation) -- span/TF only. Default 0 => byte-identical. +.ifndef TRACE + .set TRACE, 0 // 1 = per-super-tile CLAIMER trace: append one row/super-tile {tick, segcnt, +.endif // epoch, nComp/nAfeed/nBfeed live role slots, ring occA/occB peak, + // convCount, vresv, sti, quiesce} to a host-provided buffer (VA in + // occ[52:53], cap in occ[54]). Time-series of the adaptive wave-role + // economy. Requires DSWS2_CONV=1 (rows written in the quiesce path). + // Default 0 => ZERO bytes; uses free high SGPRs s70..s74. .ifndef DYNVGPR .set DYNVGPR, 1 // 1 = compute waves s_alloc_vgpr-grow per rowblk; feeds/claimer stay lean 32 .endif @@ -167,7 +204,12 @@ .endif .set SNAP_BASE, (INITFLAG_OFF + 4) // u32[6]: [parity*3 + {0:nC,1:nA,2:nB}] role-mix snapshots .set QUIESCE_CNT_OFF,(SNAP_BASE + 6*4) // u32 role-agnostic bail counter (LDS; DSWS2_GQUIESCE=0) -.set DSWS2_STATE_END,(QUIESCE_CNT_OFF + 4) +// SENSOR FIX: the claimer publishes its MID-DRAIN ring-occupancy PEAK here each super-tile; the conversion +// decisions read THESE instead of sampling occ_sample at their own quiesce (where occ_X reads ~0 post-drain +// -> always "starved" -> the 4/2/2->1/6/1 compute->feed runaway). Mid-drain peak = the true demand signal. +.set OCCA_PUB_OFF, (QUIESCE_CNT_OFF + 4) // claimer-published occ_A peak +.set OCCB_PUB_OFF, (OCCA_PUB_OFF + 4) // claimer-published occ_B peak +.set DSWS2_STATE_END,(OCCB_PUB_OFF + 4) // DSWS2_GQUIESCE (2026-07-02 SUSPECT #2 candidate fix): route the QUIESCE handshake through a DEVICE-SCOPED // GLOBAL atomic in the uncached occ buffer (byte QUIESCE_GOFF), mirroring the GREEN occ[20] claim/occ[0] // live handshake, instead of the barrier-free LDS counter (whose cross-wave visibility is unguaranteed and @@ -178,6 +220,16 @@ .set DSWS2_GQUIESCE, 0 .endif .set QUIESCE_GOFF, 200 // occ[] byte offset for the global QUIESCE counter (occ[50]) +// ---- TRACE (per-super-tile time-series) occ handshake words + row layout ---- +.set TRACE_PTR_OFF, 208 // occ[52:53] = trace buffer VA (host writes lo/hi per chunk) +.set TRACE_CAP_OFF, 216 // occ[54] = MAXROWS (host-provided row capacity) +.set TRACE_IDX_OFF, 220 // occ[55] = GLOBAL row-claim counter (all WGs' claimers share it) +.set TRACE_WGID_OFF, 224 // occ[56] = GLOBAL wg-id dispenser (claim-order 0..pool-1) +.set FATLIVE_OFF, 228 // occ[57] = live count of GROWN (fat NFV-VGPR) compute waves +.set FATMAX_OFF, 232 // occ[58] = PEAK concurrent fat waves -> x NFV = VGPR in flight (== B probe) +.set ALLLIVE_OFF, 240 // occ[60] = live count of ALL resident waves (++entry/--exit) + // occ[1] (byte 4) = PEAK concurrent resident waves (vs 2048 HW ceiling) +.set TRACE_ROW_BYTES, 64 // 16 u32/row // DSWS2_BAILMARK (SUSPECT #2 localization, 2026-07-03): each follower publishes its OWN epoch (s35) to a // PER-WAVE occ slot (BAIL_BASE + wid*4) at its _quiesce bail. One-shot per super-tile per wave -> minimal // timing perturbation (NOT the claimer's per-spin DIAG poll stores, which are the heisenbug source and stay @@ -190,6 +242,7 @@ .set DSWS2_BAILMARK, 0 .endif .set BAIL_BASE, 160 // occ[] byte offset base for per-wave bail marks: occ[40..47] +.set CONVCNT_OFF, 192 // occ[48]: DIAG conversion-commit counter (proves waves switch role) // (host prints occ[40..47] as BAIL[w0..w7]; clear of the // occ[32..36]/occ[39] DSWS sensor+roles slots and occ[50] gq) .set KSEG_STEPS, (SEGK/16) // K16-steps per split-K segment = SEGK K-elements / 16 @@ -325,8 +378,14 @@ // unsigned-division mul_hi (coop GENDIV idiom), since NTL is not generally a power of two. // ============================================================================================ .macro DECODE_STI // in: s17=sti, s67=mask, s68=shift ; out: s19=mblk s30=tcol s31=ksi ; clob: s18,s36 +.if KMAJOR + s_mul_hi_u32 s31, s17, s76 // ksi = sti / TOTAL (magic-div, magic_TOTAL in s76; K-major high bits) + s_mul_i32 s36, s31, s11 // ksi * TOTAL + s_sub_u32 s18, s17, s36 // t = sti - ksi*TOTAL (low) +.else s_and_b32 s31, s17, s67 // ksi = sti & mask (mask-bounded -> ksi in [0,n_kseg-1]) s_lshr_b32 s18, s17, s68 // t = sti >> shift +.endif .if SAFEPROBE // brick-PROOF ti clamp (the "future ti clamp" line 752 promised; COOP_STATUS.md:145 racy-garbage-ti->OOB). // A racy/torn sti read (during the claimer's per-super-tile republish) can decode a garbage t -> garbage @@ -718,6 +777,19 @@ .Lca_commit\@: lds_fetch_add s52, \dst_slot, 1 // (c) inc dest slot (unbounded -> plain atomic add) s_mov_b32 s59, \dst_slot // flip private current-role reg (records new role slot id) +.if DIAG || TRACE + // conversion-commit counter (proves a wave ACTUALLY switched role). Lean-32 pre-grow here -> v3/v4<=v15 + // OOR-safe; s49 exec-save (macro-local). (DIAG||TRACE)-gated -> DSWS2_CONV/DIAG=0/TRACE=0 byte-identical. + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lca_cm_skip\@ + v_mov_b32 v3, 1 + global_atomic_add_u32 v4, v3, s[0:1] offset:CONVCNT_OFF scope:SCOPE_DEV // occ[48] += 1 + s_wait_storecnt 0x0 +.Lca_cm_skip\@: + s_mov_b32 exec_lo, s49 +.endif .if CONV_COOLDOWN > 0 s_mov_b32 s66, CONV_COOLDOWN // Task 4: committed conversion -> arm cooldown .endif @@ -729,6 +801,165 @@ .endm .endif +// ============================================================================================ +// TFPROBE wall-span capture (TF throughput probe). Realtime-tick min/max into occ[2]/occ[3], +// mirroring occ_kernel_coop.s's proven timer idiom. Each wave stamps lane-0 only (exec-masked +// via s49, the DSWS exec-save convention); base addr v4==0 holds kernel-wide (prologue, line +// ~765; invariant per the "v4=0 occ base lane offset" note). s[30:31] free at entry (DECODE_STI +// outputs, computed only inside role bodies) and dead at every terminal. Emits ZERO bytes at +// TFPROBE=0 -> production .text byte-identical. +// ============================================================================================ +.macro tfspan op:req, off:req // op = min (entry, occ[2]/off 8) | max (exit, occ[3]/off 12) +.if TFPROBE + s_sendmsg_rtn_b64 s[30:31], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + v_cmp_eq_u32 vcc_lo, 0, v2 // lane 0 of each wave only (v2 = tid & 31) + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltfspan_skip\@ + v_mov_b32 v5, s30 // low 32 bits of the realtime tick + global_atomic_\op\()_u32 v4, v5, s[0:1] offset:\off scope:SCOPE_DEV +.Ltfspan_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +// ============================================================================================ +// PHASEPROBE: in-kernel per-phase tick timer for the COMPUTE wave (the critical path). s77 holds +// this wave's last-stamp RTC (low 32b). phase_reset seeds it; phase_stamp accumulates (now-s77) +// into occ[\off] and re-seeds. Lane-0-only atomic add; occ slots are ABOVE the per-chunk memset +// so they sum over the whole run. Scratch s62/s63 (RTC), s64 (delta) -- all free in CONV=0 compute. +// Six phases -> six occ accumulators (bytes 256..276): +.set PH_FOLLOW_OFF, 256 // occ[64]: waiting on claimer to publish next super-tile +.set PH_STAGE_OFF, 260 // occ[65]: waiting on A/B feeds to stage this super-tile +.set PH_GROW_OFF, 264 // occ[66]: claim rowblk + s_alloc_vgpr GROW 32->112 +.set PH_WMMA_OFF, 268 // occ[67]: LDS frag loads + v_wmma compute +.set PH_FLUSH_OFF, 272 // occ[68]: global_atomic_add_f32 C flush (split-K reduction) +.set PH_SHRINK_OFF, 276 // occ[69]: s_alloc_vgpr SHRINK 112->32 +// Per-wave phase accumulators live in SGPRs s78..s83 (NO per-stamp store -> zero memory perturbation, no +// s_wait_storecnt pollution). s77 = last-stamp RTC. phase_flush emits them ONCE at compute retire. +.macro phase_reset +.if PHASEPROBE + s_mov_b32 s78, 0 + s_mov_b32 s79, 0 + s_mov_b32 s80, 0 + s_mov_b32 s81, 0 + s_mov_b32 s82, 0 + s_mov_b32 s83, 0 + s_sendmsg_rtn_b64 s[62:63], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + s_mov_b32 s77, s62 +.endif +.endm +.macro phase_stamp acc:req // \acc += (now - s77); s77 = now (pure scalar, no store) +.if PHASEPROBE + s_sendmsg_rtn_b64 s[62:63], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + s_sub_u32 s64, s62, s77 // delta (mod 2^32; phase << 2^32 so wrap-safe) + s_mov_b32 s77, s62 + s_add_u32 \acc, \acc, s64 +.endif +.endm +.macro phase_flush // lane0 atomic-adds s78..s83 -> occ[64..69]; drained here (not the hot loop) +.if PHASEPROBE + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lphf_skip\@ + v_mov_b32 v5, s78 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_FOLLOW_OFF scope:SCOPE_DEV + v_mov_b32 v5, s79 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_STAGE_OFF scope:SCOPE_DEV + v_mov_b32 v5, s80 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_GROW_OFF scope:SCOPE_DEV + v_mov_b32 v5, s81 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_WMMA_OFF scope:SCOPE_DEV + v_mov_b32 v5, s82 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_FLUSH_OFF scope:SCOPE_DEV + v_mov_b32 v5, s83 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_SHRINK_OFF scope:SCOPE_DEV + s_wait_storecnt 0x0 +.Lphf_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +// ============================================================================================ +// TRACE: per-super-tile time-series row (claimer, lane 0). Written once per super-tile at the +// quiesce-satisfied drain-exit (.Lqc_q_ok). Captures the adaptive wave-role economy over time: +// the LIVE role slots (do waves convert?), the per-super-tile ring-occupancy PEAK (s73/s74, +// tracked across the wait_done spins), the cumulative conversion count, and the envelope vresv. +// 16 u32/row -> buffer[segcnt*64]; bounded by MAXROWS (s72). Emits ZERO bytes at TRACE=0. +// Persistent trace regs: s70:s71 = buffer VA, s72 = MAXROWS, s73/s74 = ring occA/occB peak. +// ============================================================================================ +.macro alllive_dec // TRACE: --live on wave exit (pairs with the entry ++ for peak-concurrent) +.if TRACE + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lalld_skip\@ + v_mov_b32 v3, -1 + global_atomic_add_u32 v4, v3, s[0:1] offset:ALLLIVE_OFF scope:SCOPE_DEV +.Lalld_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +.macro trace_row +.if TRACE + // claim a GLOBALLY-unique row index (all WGs' claimers share occ[55]) -> no per-WG SEGCNT collision. + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + v_mov_b32 v14, 1 + global_atomic_add_u32 v15, v4, v14, s[0:1] offset:TRACE_IDX_OFF th:TH_ATOMIC_RETURN scope:SCOPE_DEV // v15=old idx, v4=addr(0), v14=data(1) + s_wait_loadcnt 0x0 + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 s52, v15 // s52 = unique row index (old value returned by the atomic) + s_cmp_ge_u32 s52, s72 // row >= MAXROWS -> skip (buffer bound) + s_cbranch_scc1 .Ltrow_skip\@ + s_lshl_b32 s53, s52, 6 // row * TRACE_ROW_BYTES(64) + s_add_u32 s60, s70, s53 + s_addc_u32 s61, s71, 0 // s[60:61] = row base VA + s_sendmsg_rtn_b64 s[58:59], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + lds_get s55, SEGCNT_OFF // this WG's per-WG super-tile counter (data field, not index) + v_mov_b32 v14, s58 // 0 tick_lo + v_mov_b32 v15, s55 // 1 segcnt (per-WG) + v_mov_b32 v16, s35 // 2 epoch + lds_get s54, NCOMP_SLOT + v_mov_b32 v17, s54 // 3 nComp (live role slot) + lds_get s54, NAFEED_SLOT + v_mov_b32 v18, s54 // 4 nAfeed + lds_get s54, NBFEED_SLOT + v_mov_b32 v19, s54 // 5 nBfeed + v_mov_b32 v20, s73 // 6 occA peak (across wait_done spins) + v_mov_b32 v21, s74 // 7 occB peak + global_load_b32 v22, v4, s[0:1] offset:CONVCNT_OFF scope:SCOPE_DEV // 8 convCount (cumulative) + lds_get s54, VRESV_OFF + v_mov_b32 v23, s54 // 9 vresv (envelope budget) + v_mov_b32 v24, s17 // 10 sti (claimed super-tile id) + lds_get s54, QUIESCE_CNT_OFF + v_mov_b32 v25, s54 // 11 quiesce (final) + v_mov_b32 v26, s59 // 12 tick_hi + v_mov_b32 v27, s69 // 13 chunkHi (context) + v_mov_b32 v28, s75 // 14 wg_id (which workgroup's economy this row belongs to) + v_mov_b32 v29, 0 // 15 reserved + s_wait_loadcnt 0x0 // convCount load drained before the row store + v_cmp_eq_u32 vcc_lo, 0, v2 // lane 0 of the claimer writes the row + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltrow_wskip\@ + global_store_b128 v4, v[14:17], s[60:61] offset:0 scope:SCOPE_DEV + global_store_b128 v4, v[18:21], s[60:61] offset:16 scope:SCOPE_DEV + global_store_b128 v4, v[22:25], s[60:61] offset:32 scope:SCOPE_DEV + global_store_b128 v4, v[26:29], s[60:61] offset:48 scope:SCOPE_DEV +.Ltrow_wskip\@: + s_mov_b32 exec_lo, s49 +.Ltrow_skip\@: +.endif +.endm + // ============================================================================================ // KERNEL // ============================================================================================ @@ -749,6 +980,11 @@ occ_kernel: v_and_b32 v2, 31, v0 // lane = tid & 31 v_and_b32 v6, 15, v0 // lane & 15 (A vaddr) v_mov_b32 v4, 0 +.if KMAJOR + global_load_b32 v3, v4, s[0:1] offset:248 scope:SCOPE_DEV // occ[62] = magic(TOTAL), host-written + s_wait_loadcnt 0x0 + v_readfirstlane_b32 s76, v3 // s76 = magic_TOTAL, persistent for every DECODE_STI (K-major) +.endif // ---- per-lane address constants (mbgemm-identical; dyn-VGPR arming compatible) ---- v_mul_lo_u32 v8, v6, s9 // (lane&15)*K v_bfe_u32 v7, v0, 4, 1 @@ -765,6 +1001,22 @@ occ_kernel: v_min_u32 v10, 0x400, v10 // clamp C vaddr (1024 >= lane*32 max 992) .endif + tfspan min, 8 // TFPROBE: every wave stamps occ[2] = min entry tick (wall-span start) +.if TRACE + // total-occupancy: every wave ++live at entry, atomic-max the peak concurrent resident count (occ[1]). + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lall_enter_skip + v_mov_b32 v3, 1 + global_atomic_add_u32 v5, v4, v3, s[0:1] offset:ALLLIVE_OFF th:TH_ATOMIC_RETURN scope:SCOPE_DEV + s_wait_loadcnt 0x0 + v_add_nc_u32 v5, v5, 1 + global_atomic_max_u32 v4, v5, s[0:1] offset:4 scope:SCOPE_DEV // occ[1] = peak concurrent resident waves +.Lall_enter_skip: + s_mov_b32 exec_lo, s49 +.endif + .if DSWS2 // ===== DSWS v2 role branch (wid uniform per wave; scalar-only -> exec stays full for every role). // wid == 0 -> claimer (pinned super-tile broadcaster; A3) @@ -884,6 +1136,8 @@ occ_kernel: lds_put (SNAP_BASE + 12), NCOMP // parity-1 = launch mix too (init) lds_put (SNAP_BASE + 16), NAFEED lds_put (SNAP_BASE + 20), NBFEED + lds_put OCCA_PUB_OFF, 2 // SENSOR FIX: neutral ring seed (not CTRL_HIGH) until 1st publish + lds_put OCCB_PUB_OFF, 2 .endif lds_put INITFLAG_OFF, 0xACED // LAST: publishes "LDS ready" to all follower waves // FIX 1(e): load this dispatch's chunk terminal bound from occ[24] (host writes occW[6] per chunk; @@ -893,6 +1147,23 @@ occ_kernel: s_wait_loadcnt 0x0 v_readfirstlane_b32 s69, v6 // s69 = chunkHi (this dispatch's terminal sti bound) s_mov_b32 s35, 0 // claimer local epoch +.if TRACE + global_load_b64 v[6:7], v4, s[0:1] offset:TRACE_PTR_OFF scope:SCOPE_DEV // trace buffer VA + global_load_b32 v8, v4, s[0:1] offset:TRACE_CAP_OFF scope:SCOPE_DEV // MAXROWS + s_wait_loadcnt 0x0 + v_readfirstlane_b32 s70, v6 // s[70:71] = trace VA (persistent, claimer-only) + v_readfirstlane_b32 s71, v7 + v_readfirstlane_b32 s72, v8 // s72 = MAXROWS + // wg_id = claim-order dispenser (TGID_X isn't enabled in s15 -> was all-0). One atomic per WG at entry. + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + v_mov_b32 v7, 1 + global_atomic_add_u32 v6, v4, v7, s[0:1] offset:TRACE_WGID_OFF th:TH_ATOMIC_RETURN scope:SCOPE_DEV + s_wait_loadcnt 0x0 + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 s75, v6 // s75 = this WG's claim-order id (persistent, claimer-only) +.endif .Lclaim_loop: // claim next sti: lane0 global_atomic_add occ[20] += 1, return old v_cmp_eq_u32 vcc_lo, 0, v2 @@ -945,10 +1216,19 @@ occ_kernel: .endif .endif lds_put EPOCH_OFF, s44 +.if DSWS2_CONV + s_mov_b32 s73, 0 // reset this super-tile's ring occ_A peak (SENSOR FIX + TRACE) + s_mov_b32 s74, 0 // reset ring occ_B peak (updated across wait_done spins) +.endif BSTAGE // claimer helps stage B for this super-tile (s30,s31) // A7 advance gate: free resident A/B only when ALL G rowblks are computed+flushed .Lclaimer_wait_done: s_sleep SLEEPN +.if DSWS2_CONV + occ_sample s62, s63 // sample ring mid-drain; keep the per-super-tile PEAK (SENSOR FIX + TRACE) + s_max_u32 s73, s73, s62 + s_max_u32 s74, s74, s63 +.endif .if DSWS2_CONV .if DIAG // Phase-B DIAG probe (Task 3): wid 0 samples the LIVE ring occupancy (compute is mid-drain here) @@ -1029,6 +1309,24 @@ occ_kernel: s_cbranch_scc0 .Lqc_q_ok s_mov_b32 s51, 0 .Lqc_q_ok: +.if DSWS2_CONV + // SENSOR FIX: publish this super-tile's mid-drain ring PEAK so the followers' NEXT-epoch conversion + // decisions read a true demand signal instead of sampling occ_X~0 at their own post-drain quiesce. + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lpub_skip + v_mov_b32 v14, OCCA_PUB_OFF + v_mov_b32 v15, s73 + ds_store_b32 v14, v15 + v_mov_b32 v14, OCCB_PUB_OFF + v_mov_b32 v15, s74 + ds_store_b32 v14, v15 + // (no s_wait_dscnt: followers read this a full super-tile later; async LDS store drains long before) +.Lpub_skip: + s_mov_b32 exec_lo, s49 +.endif + trace_row // TRACE: append this super-tile's row (role mix, ring peak, conv, vresv) .if DIAG // DIAG cross-check + Pool-T7 chunk-2 WEDGE FRAME. occ[29]=s50^s51 mismatch. Plus snapshot EVERY advance-gate // counter to host-streamed occ slots so a hung dispatch reads out exactly which sentinel is unmet: @@ -1108,6 +1406,8 @@ occ_kernel: global_atomic_add_u32 v4, v3, s[0:1] scope:SCOPE_DEV .Lclaimer_dead: s_mov_b32 exec_lo, s16 + tfspan max, 12 // TFPROBE: claimer stamps occ[3] = max exit tick (wall-span end) + alllive_dec s_endpgm // ============================================================================================ @@ -1160,7 +1460,8 @@ occ_kernel: .endif .else s_mov_b32 s58, 0 - occ_sample s55, s56 // s55=occ_A, s56=occ_B + lds_get s55, OCCA_PUB_OFF // SENSOR FIX: read claimer-published mid-drain peak (not post-drain ~0) + lds_get s56, OCCB_PUB_OFF s_cmp_gt_u32 s56, CTRL_HIGH_B s_cbranch_scc0 .Lbfeed_cooldn s_mov_b32 s57, 3 // dir 3: B-feed -> compute @@ -1239,7 +1540,8 @@ occ_kernel: .endif .else s_mov_b32 s58, 0 - occ_sample s55, s56 // s55=occ_A, s56=occ_B + lds_get s55, OCCA_PUB_OFF // SENSOR FIX: read claimer-published mid-drain peak (not post-drain ~0) + lds_get s56, OCCB_PUB_OFF s_cmp_gt_u32 s55, CTRL_HIGH_A s_cbranch_scc0 .Lafeed_cooldn s_mov_b32 s57, 2 // dir 2: A-feed -> compute @@ -1284,15 +1586,25 @@ occ_kernel: s_cmp_eq_u32 s44, 0xACED s_cbranch_scc0 .Lcompute_init s_mov_b32 s35, 0 + phase_reset // PHASEPROBE: seed this compute wave's phase clock .Lcompute_follow: s_sleep SLEEPN lds_get s44, EPOCH_OFF s_cmp_eq_u32 s44, s35 s_cbranch_scc1 .Lcompute_follow s_mov_b32 s35, s44 + phase_stamp s78 // close FOLLOW_WAIT (spun waiting for next super-tile) lds_get s17, STI_OFF +.if PHASEPROBE + s_cmp_eq_u32 s17, 0xFFFFFFFF // sentinel -> flush phase accumulators, then retire + s_cbranch_scc0 .Lcompute_go + phase_flush + s_branch .Lretire +.Lcompute_go: +.else s_cmp_eq_u32 s17, 0xFFFFFFFF // FIX 1(f): sentinel (A7) -> retire (was: STI>=TOTAL_super) s_cbranch_scc1 .Lretire +.endif DECODE_STI // s19=mblk s30=tcol s31=ksi // wait until resident A AND B fully STAGED (B: FN frags stored, A: G rowblks stored) .Lcompute_staged: @@ -1303,6 +1615,7 @@ occ_kernel: lds_get s44, AROW_DONE_OFF s_cmp_lt_u32 s44, G s_cbranch_scc1 .Lcompute_staged + phase_stamp s79 // close STAGE_WAIT (spun waiting for A/B feeds) // C tile-term: ti = mblk*NTL + tcol ; ti*(G*FM*FN*1024) (ksi-INDEPENDENT -> split-K accumulates) s_mul_i32 s38, s19, s13 s_add_u32 s38, s38, s30 @@ -1329,7 +1642,22 @@ occ_kernel: .Lcompute_grow: s_alloc_vgpr NFV // grow (SCC-retry guarded, brick-class rule) s_cbranch_scc0 .Lcompute_grow +.if TRACE + // B-probe: this wave just went fat (NFV VGPR). ++live, track the peak concurrent fat count. + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lfatg_skip + v_mov_b32 v3, 1 + global_atomic_add_u32 v5, v4, v3, s[0:1] offset:FATLIVE_OFF th:TH_ATOMIC_RETURN scope:SCOPE_DEV // v5 = old live + s_wait_loadcnt 0x0 + v_add_nc_u32 v5, v5, 1 // new live count + global_atomic_max_u32 v4, v5, s[0:1] offset:FATMAX_OFF scope:SCOPE_DEV // peak = max(peak, new) +.Lfatg_skip: + s_mov_b32 exec_lo, s49 +.endif .endif + phase_stamp s80 // close GROW (rowblk claim + s_alloc_vgpr 32->112) // zero FM*FN fp32 accumulators .set idx, 0 .rept FM*FN @@ -1373,29 +1701,48 @@ occ_kernel: .endr .set ks, ks+1 .endr + phase_stamp s81 // close WMMA (LDS frag loads + v_wmma over the segment) // flush: C base = C + ti-term + r*(FM*FN*1024) ; per (frag,elem) atomic-add one fp32 (vaddr v10=lane*32) s_mul_i32 s39, s33, (FM*FN*1024) s_add_u32 s39, s38, s39 s_add_u32 s28, s6, s39 s_addc_u32 s29, s7, 0 +.if NOCFLUSH == 0 .set frag, 0 .rept FM*FN .set e, 0 .rept 8 + .if CSTORE + global_store_b32 v10, v[ACC+frag*8+e], s[28:29] offset:(frag*1024 + e*4) scope:SCOPE_DEV // probe: non-atomic, same count + .else global_atomic_add_f32 v10, v[ACC+frag*8+e], s[28:29] offset:(frag*1024 + e*4) scope:SCOPE_DEV + .endif .set e, e+1 .endr .set frag, frag+1 .endr +.endif s_wait_storecnt 0x0 // atomic-adds READ ACC -> must drain before shrink frees ACC + phase_stamp s82 // close FLUSH (global_atomic_add_f32 C reduction + drain) .if DYNVGPR .Lcompute_shrink: s_alloc_vgpr 32 // shrink (SCC-retry guarded) s_cbranch_scc0 .Lcompute_shrink +.if TRACE + v_cmp_eq_u32 vcc_lo, 0, v2 // B-probe: this wave went lean again -> --live + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lfats_skip + v_mov_b32 v3, -1 + global_atomic_add_u32 v4, v3, s[0:1] offset:FATLIVE_OFF scope:SCOPE_DEV +.Lfats_skip: + s_mov_b32 exec_lo, s49 +.endif .if DSWS2_ENVELOPE lds_fetch_add s54, VRESV_OFF, -(NFV-VLEAN) // envelope release −Δ (shrink committed; wave lean, v<=v15) .endif .endif + phase_stamp s83 // close SHRINK (s_alloc_vgpr 112->32) lds_inc ROWBLK_DONE_OFF // rowblk r computed + flushed (frees the A7 advance gate) s_branch .Lcompute_claim .Lcompute_drained: @@ -1433,7 +1780,8 @@ occ_kernel: .endif .else s_mov_b32 s58, 0 // s_win = 0 (default: raced no ticket) - occ_sample s55, s56 // s55=occ_A in [0,G], s56=occ_B in [0,FN] (clob s60,s61) + lds_get s55, OCCA_PUB_OFF // SENSOR FIX: read claimer-published mid-drain PEAK, not occ_sample + lds_get s56, OCCB_PUB_OFF // at post-drain quiesce (~0 -> false "starved" -> 4/2/2->1/6/1 runaway) s_cmp_lt_u32 s55, CTRL_LOW s_cbranch_scc0 .Lcmp_try_b s_mov_b32 s57, 0 // dir 0: compute -> A-feed @@ -1495,6 +1843,8 @@ occ_kernel: // ---- A7 role-agnostic terminal (followers): retire. (Claimer retires via .Lclaimer_terminal.) ---- .Lretire: + tfspan max, 12 // TFPROBE: every follower stamps occ[3] = max exit tick (wall-span end) + alllive_dec // TRACE: follower exit -> --live (peak-concurrent occupancy) s_endpgm .else s_endpgm // DSWS2=0 has no v2 body (this file is always built DSWS2=1) diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s new file mode 100644 index 000000000000..d1c4cbe6e9d1 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s @@ -0,0 +1,1789 @@ +// occ_kernel_dsws.s (gfx1201, wave32) -- MAD-305 DSWS v2 SUBSTRATE SCAFFOLD (PLAN_DSWS_SUBSTRATE_V2.md, Task A1). +// +// v2 re-founds the DSWS GEMM substrate so matrix work is *claimed* (not owned by compile-time wave id), with +// split-K folded in. Work = a pool of (mblk, tcol, ksi) super-tiles; a pinned claimer (wid 0) broadcasts the +// current super-tile; live waves of each role drain shared LDS atomic counters against resident-in-LDS A/B. +// +// Task A1 is SCAFFOLD-ONLY: lift the proven prologue/arming setup from occ_kernel_coop.s (NEVER modified), +// assert the v2 LDS layout fits the 32 KB group segment, and leave each role as a distinct s_endpgm stub. +// The real claimer / feed / compute loops land in A3..A7. The only gates here: (1) assembles clean, +// (2) RGA 0-spill, (3) harness --dsws2 dry-prints the computed params. +// +// ============================================================================================ +// v2 KERNARG CONTRACT (USER_SGPR=15, s0..s14; hardware-preloaded user SGPRs). The host A8 launch +// MUST set COMPUTE_PGM_RSRC2.USER_SGPR=15 and load COMPUTE_USER_DATA_0..14 to match EXACTLY: +// s0:s1 = occ buffer base (>=0x1000B; host zero-inits; see CLAIM-COUNTER / completion offsets below) +// s2:s3 = A base (fp8 e4m3, row-major, 1 byte/elem) +// s4:s5 = Bshuf base (the shuffled-B layout the coop B-feed consumes; same global_load_tr_b64 idiom) +// s6:s7 = C base (fp32; HOST MUST MEMSET C=0 before dispatch -- compute uses global_atomic_add_f32) +// s8 = KT (total K16-steps for the whole matrix = K/16) +// s9 = K(bytes/A-row) (= K, fp8 1 byte/elem) +// s10 = NT*256 (B-saddr advance per K16-step) +// s11 = TOTAL (coop-style total *tiles* = MTL*NTL; carried for addressing compat, NOT the pool size) +// s12 = magic(ceil(2^32/NTL)) (unsigned-div magic for /NTL ; tcol/mblk decode) +// s13 = NTL (number of N tile-columns) +// s14 = FN*256 (B-saddr stride per N-frag) +// (TGID_X now lands in s15 -- UNUSED; this kernel is pool-claim, not workgroup-id based.) +// NOTE: G and SEGK are COMPILE-TIME defsyms (baked into instruction immediates); they are NOT kernargs. +// FIX 1 (round-table Opus+Codex pass): v1 of this contract passed n_kseg/TOTAL_super/magic_kseg as +// s15/s16/s17, but the PM4 host only preloads COMPUTE_USER_DATA_0..15 (USER_SGPR<=16; every proven +// launch path in this tree uses 15) -- s16/s17 could NEVER actually arrive in hardware SGPRs, AND +// s16 was independently being reused per-chunk on the host as the compositor-safe chunk terminal +// (a second, unrelated collision on the same slot). This file now drops s15/s16/s17 entirely: +// n_kseg is DERIVED in-kernel from KT (s8) and the compile-time KSEG_STEPS=SEGK/16: +// n_kseg = KT >> NKSEG_SHIFT, where NKSEG_SHIFT=log2(KSEG_STEPS) is a compile-time `.set` +// (small .if ladder over KSEG_STEPS in {1,2,4,8,16}; SEGK is always a power-of-two +// multiple of 16, so KSEG_STEPS is always a power of two in that set). +// shift/mask (the sti -> (t,ksi) split) are derived ONCE in the prologue from n_kseg: +// shift = s_ff1_i32_b32(n_kseg) (bit index of n_kseg's single set bit; n_kseg=1 -> 0) +// mask = n_kseg - 1 +// DECODE_STI then does ksi = sti & mask ; t = sti >> shift -- this handles n_kseg=1 for +// free (shift=0, mask=0 -> ksi=0, t=sti), so the old magic-div n_kseg==1 special-case is +// GONE (it's no longer needed, not just hidden). +// the chunk terminal (old TOTAL_super/"chunkHi") is now MEMORY-CARRIED instead of a kernarg: the +// host writes the current chunk's terminal sti bound to occ[24] (occW[6]) once per chunk; +// the claimer reads occ[24] ONCE per dispatch (stable for the whole chunk) instead of +// receiving it as a broadcast kernarg. On sti >= occ[24] the claimer publishes a SENTINEL +// (0xFFFFFFFF) into STI_OFF instead of the raw over-claimed sti; followers (b-feed/a-feed/ +// compute) retire when STI_OFF == 0xFFFFFFFF instead of comparing against the (now +// nonexistent) TOTAL_super kernarg. +// See "CLAIM-COUNTER & completion occ offsets" by .Lclaimer for the full occ-buffer layout +// (occ[24]/occW[6] = chunk terminal bound, added by FIX 1). +// +// SCALAR REGS (derived in the prologue, before any clobber; none collide with DECODE_STI's own clobber +// list s18/s36, lds_*'s s49, the claimer's s16/s17/s35/s44, or any role body's transients, all <= s65): +// s66 = n_kseg (derived; dead after shift/mask below are computed -- kept only for that derivation) +// s67 = mask (n_kseg - 1) -- LIVE for the whole kernel; read by every DECODE_STI call, every role. +// s68 = shift (log2 n_kseg) -- LIVE for the whole kernel; read by every DECODE_STI call, every role. +// s69 = chunkHi (claimer-only; loaded once per dispatch from occ[24] right before .Lclaim_loop). +// +// Everything new is gated behind the fresh `DSWS2` build symbol (analogous to coop's `DSWS`). + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1201" + +// ---- tile defsyms (lifted from occ_kernel_coop.s) ---- +.ifndef FM + .set FM, 2 // per-compute-wave M-frags (M-band = FM*16 rows) +.endif +.ifndef FN + .set FN, 4 // shared N-frags (the reuse operand) +.endif +.ifndef RGADESC + .set RGADESC, 0 // 1 = emit analysis-only AMDHSA descriptor for RGA livereg +.endif +.ifndef DIAG + .set DIAG, 0 // 1 = phase-marker instrumentation (unused in the A1 scaffold) +.endif +.ifndef SAFEPROBE + .set SAFEPROBE, 0 // 1 = clamp per-lane vector address regs into a provable in-buffer bound +.endif +.ifndef TFPROBE + .set TFPROBE, 0 // 1 = emit s_sendmsg_rtn GET_REALTIME wall-span capture (each wave stamps +.endif // occ[2]=min entry tick, occ[3]=max exit tick -> host TF readout). + // Default 0 => ZERO bytes emitted, .text byte-identical to the + // production bin (uses only already-allocated regs s30/s31/s49/v5). +.ifndef PHASEPROBE + .set PHASEPROBE, 0 // 1 = in-kernel PHASE TIMER: each compute wave stamps GET_REALTIME at every +.endif // phase boundary and atomic-adds the delta into per-phase occ accumulators + // (occ[64..69], bytes 256..276, ABOVE the per-chunk memset -> accumulate over + // the whole run). Host prints ticks + % per phase. Default 0 => byte-identical. +.ifndef NOCFLUSH + .set NOCFLUSH, 0 // PERF PROBE ONLY: 1 = skip the global_atomic_add_f32 C-flush loop (keep ALL +.endif // other bookkeeping/handshake). Isolates the device-atomic C-reduction cost + // from the coordination handshake. Result is WRONG (C never written -> oracle + // fails) -- span/TF only. Default 0 => byte-identical. +.ifndef KMAJOR + .set KMAJOR, 0 // PERF PROBE: 1 = K-MAJOR super-tile traversal. Default decode packs ksi in +.endif // the LOW bits (all n_kseg segments of a C cell claimed consecutively -> + // up to n_kseg WGs hammer one C cell at once = max atomic contention). + // KMAJOR decodes ksi = sti / TOTAL (high), t = sti % TOTAL (low) via a + // magic-div (magic_TOTAL from occ[62], loaded to s76 in prologue) -> the + // 32 segments of a cell are spread TOTAL apart in claim order (near-zero + // concurrent contention) + adjacent tiles (shared operand bands) claimed + // close in time (L2 reuse). Correctness-preserving (C add is commutative). +.ifndef CSTORE + .set CSTORE, 0 // PERF PROBE ONLY: 1 = replace the flush's global_atomic_add_f32 with an +.endif // equal-count NON-atomic global_store_b32 (same #mem-ops, same addresses, NO + // RMW/contention). Isolates atomic-contention from raw write-bandwidth: if + // TF jumps vs atomics -> contention; if flat -> bandwidth-bound. Result WRONG + // (last-writer, no accumulation) -- span/TF only. Default 0 => byte-identical. +.ifndef TRACE + .set TRACE, 0 // 1 = per-super-tile CLAIMER trace: append one row/super-tile {tick, segcnt, +.endif // epoch, nComp/nAfeed/nBfeed live role slots, ring occA/occB peak, + // convCount, vresv, sti, quiesce} to a host-provided buffer (VA in + // occ[52:53], cap in occ[54]). Time-series of the adaptive wave-role + // economy. Requires DSWS2_CONV=1 (rows written in the quiesce path). + // Default 0 => ZERO bytes; uses free high SGPRs s70..s74. +.ifndef DYNVGPR + .set DYNVGPR, 1 // 1 = compute waves s_alloc_vgpr-grow per rowblk; feeds/claimer stay lean 32 +.endif +.ifndef SLEEPN + .set SLEEPN, 2 // s_sleep arg in the busy-waits (yield issue cycles to partner waves) +.endif + +// ============================================================================================ +// DSWS v2 LDS layout (bytes from group-segment base; words u32 unless noted). Mirrors the placement +// of the coop file's LDS `.set` block. Defined unconditionally (uses only G/SEGK/FM/FN, always set). +// ============================================================================================ +.ifndef DSWS2 + .set DSWS2, 0 +.endif +.ifndef G + .set G, 6 // cooperative M-extent (rowblks per super-tile) = LDS accumulator-bank count (ACC_N) +.endif +.ifndef SEGK + .set SEGK, 64 // split-K segment size in K-elements (multiple of 16) +.endif +// ---- v2 control/claim words ---- +.set STI_OFF, 0 // broadcast super-tile id +.set EPOCH_OFF, 4 +.set ROWBLK_NEXT_OFF, 8 // per-super-tile rowblk claim counter +.set ROWBLK_DONE_OFF, 12 // per-super-tile completion counter +.set BFRAG_NEXT_OFF, 16 // B-frag claim counter +.set AROW_NEXT_OFF, 20 // A-rowblk claim counter +.set NCOMP_SLOT, 24 +.set NAFEED_SLOT, 28 +.set NBFEED_SLOT, 32 +.set GATE_OFF, 36 // u32[4] -> 36,40,44,48 (conversion gates) +.set VRESV_OFF, 52 // vgpr_reserved +.set SEGCNT_OFF, 56 // controller clock +// ---- A3..A7 additions (still inside the 0..256 control region; A1 offsets 0..56 unchanged) ---- +.set BFRAG_DONE_OFF, 60 // B-frag STORE-completion counter (compute gates on this, NOT the claim ctr) +.set AROW_DONE_OFF, 64 // A-rowblk STORE-completion counter (compute gates on this) +.set INITFLAG_OFF, 68 // barrier-free LDS-init publish flag (claimer writes 0xACED LAST) +// ---- Phase-B (DSWS2_CONV) control state: role-mix snapshot slots + quiesce counter ---- +// Based at INITFLAG_OFF+4 (NOT the brief's SEGCNT_OFF+4): the brief predates the A3..A7 control +// words (BFRAG_DONE/AROW_DONE/INITFLAG at 60/64/68), so SEGCNT_OFF+4=60 would collide with them. +// Basing after the LAST control word keeps the new state inside the 0..255 control gap BELOW the +// fixed resident region (BRES_OFF=256), so NO resident-region repoint is needed -- the resident +// BRES_OFF/ARES_OFF immediates (emitted unconditionally in the kernel body) stay untouched, which +// is what keeps the DSWS2_CONV=0 binary byte-identical to the Phase-A green bin. All `.set`s here +// are inert (emit no bytes); the only new code (claimer init) is gated under `.if DSWS2_CONV`. +.ifndef DSWS2_CONV + .set DSWS2_CONV, 0 // 0 = pre-conversion static substrate (Phase A green); 1 = Phase B +.endif +.ifndef DSWS2_TICKET_SELFTEST + .set DSWS2_TICKET_SELFTEST, 0 // DIAG-only try_gate single-winner smoke (Task 4 Step 3); default 0 = no bytes +.endif +.ifndef CONV_COOLDOWN + .set CONV_COOLDOWN, 0 // Task 4: per-wave post-conversion cooldown epochs. 0 = spec-faithful (no + // cooldown, byte-identical to pre-Task-4); >0 damps thrash (skip N epochs + // of watermark decision after a wave converts role). +.endif +// Task 5: deterministic bring-up hook. DSWS2_FORCE=1 makes exactly wave DSWS2_FORCE_WID convert +// direction DSWS2_FORCE_DIR at epoch DSWS2_FORCE_EPOCH, watermarks bypassed -- a reproducible, +// single-wave/single-epoch GPU proof of role conversion. Default DSWS2_FORCE=0 emits ZERO bytes +// (byte-identical to pre-Task-5). +.ifndef DSWS2_FORCE + .set DSWS2_FORCE, 0 +.endif +.ifndef DSWS2_FORCE_WID + .set DSWS2_FORCE_WID, 0 +.endif +.ifndef DSWS2_FORCE_DIR + .set DSWS2_FORCE_DIR, 0 // 0/1 = compute->A/B ; 2/3 = A/B->compute +.endif +.ifndef DSWS2_FORCE_EPOCH + .set DSWS2_FORCE_EPOCH, 1 +.endif +// Rolling dyn-VGPR sum-envelope (2026-07-02 spec). ENVELOPE routes the per-rowblk compute burst grow +// through the shared vgpr_reserved counter so at most PEAK_CONC waves hold peak at once (the +// multi-grower collision, ISA 3.3.3.2, becomes unreachable). All default to the byte-identical value: +// ENVELOPE=0/STAGGER=0 emit ZERO new bytes and PEAK_CONC/STAGGER_PERIOD are inert unless their gate is on. +.ifndef DSWS2_ENVELOPE + .set DSWS2_ENVELOPE, 0 // 1 = route the per-rowblk compute burst grow through the vgpr_reserved +.endif // sum-envelope. 0 = HEAD (bare .Lcompute_grow) -> .text byte-identical. +.ifndef PEAK_CONC + .set PEAK_CONC, 2 // concurrent compute peaks the budget admits (R3 sweep). Used iff ENVELOPE=1. +.endif +.ifndef DSWS2_STAGGER + .set DSWS2_STAGGER, 0 // 1 = lock-free phase-token stagger (Task 9). 0 -> emergent envelope stagger. +.endif +.ifndef STAGGER_PERIOD + .set STAGGER_PERIOD, 4 // phase slots in the stagger ring (R3 sweep). Used iff STAGGER=1 (inert here). +.endif +.set SNAP_BASE, (INITFLAG_OFF + 4) // u32[6]: [parity*3 + {0:nC,1:nA,2:nB}] role-mix snapshots +.set QUIESCE_CNT_OFF,(SNAP_BASE + 6*4) // u32 role-agnostic bail counter (LDS; DSWS2_GQUIESCE=0) +// SENSOR FIX: the claimer publishes its MID-DRAIN ring-occupancy PEAK here each super-tile; the conversion +// decisions read THESE instead of sampling occ_sample at their own quiesce (where occ_X reads ~0 post-drain +// -> always "starved" -> the 4/2/2->1/6/1 compute->feed runaway). Mid-drain peak = the true demand signal. +.set OCCA_PUB_OFF, (QUIESCE_CNT_OFF + 4) // claimer-published occ_A peak +.set OCCB_PUB_OFF, (OCCA_PUB_OFF + 4) // claimer-published occ_B peak +.set DSWS2_STATE_END,(OCCB_PUB_OFF + 4) +// DSWS2_GQUIESCE (2026-07-02 SUSPECT #2 candidate fix): route the QUIESCE handshake through a DEVICE-SCOPED +// GLOBAL atomic in the uncached occ buffer (byte QUIESCE_GOFF), mirroring the GREEN occ[20] claim/occ[0] +// live handshake, instead of the barrier-free LDS counter (whose cross-wave visibility is unguaranteed and +// is the leading SUSPECT #2 hang mechanism). occ buffer = AllocGpu 0x1000 (1024 u32, uncached); host uses +// occ[0..6] + DIAG scratch (<= byte 116); byte 200 (occ[50]) is provably free. Default 0 => LDS path, +// .text byte-identical. Requires DSWS2_CONV (QUIESCE only exists there). +.ifndef DSWS2_GQUIESCE + .set DSWS2_GQUIESCE, 0 +.endif +.set QUIESCE_GOFF, 200 // occ[] byte offset for the global QUIESCE counter (occ[50]) +// ---- TRACE (per-super-tile time-series) occ handshake words + row layout ---- +.set TRACE_PTR_OFF, 208 // occ[52:53] = trace buffer VA (host writes lo/hi per chunk) +.set TRACE_CAP_OFF, 216 // occ[54] = MAXROWS (host-provided row capacity) +.set TRACE_IDX_OFF, 220 // occ[55] = GLOBAL row-claim counter (all WGs' claimers share it) +.set TRACE_WGID_OFF, 224 // occ[56] = GLOBAL wg-id dispenser (claim-order 0..pool-1) +.set FATLIVE_OFF, 228 // occ[57] = live count of GROWN (fat NFV-VGPR) compute waves +.set FATMAX_OFF, 232 // occ[58] = PEAK concurrent fat waves -> x NFV = VGPR in flight (== B probe) +.set ALLLIVE_OFF, 240 // occ[60] = live count of ALL resident waves (++entry/--exit) + // occ[1] (byte 4) = PEAK concurrent resident waves (vs 2048 HW ceiling) +.set TRACE_ROW_BYTES, 64 // 16 u32/row +// DSWS2_BAILMARK (SUSPECT #2 localization, 2026-07-03): each follower publishes its OWN epoch (s35) to a +// PER-WAVE occ slot (BAIL_BASE + wid*4) at its _quiesce bail. One-shot per super-tile per wave -> minimal +// timing perturbation (NOT the claimer's per-spin DIAG poll stores, which are the heisenbug source and stay +// DIAG-only). After a watchdog abort the host reads occ[BAIL_BASE/4 + wid]: every follower's slot == the +// hung epoch => all reached their bail (=> a QUIESCE visibility/lost-update, gq relevant); ONE slot stale +// at the prior epoch => that exact wave is the STRAGGLER (stuck in _alloc/_init/_follow; gq irrelevant). +// Per-WAVE (not per-role): 4 compute share one role, so a role mark's last-writer-wins would hide a single +// straggler. Default 0 => no bytes, .text byte-identical. Requires DSWS2_CONV. +.ifndef DSWS2_BAILMARK + .set DSWS2_BAILMARK, 0 +.endif +.set BAIL_BASE, 160 // occ[] byte offset base for per-wave bail marks: occ[40..47] +.set CONVCNT_OFF, 192 // occ[48]: DIAG conversion-commit counter (proves waves switch role) + // (host prints occ[40..47] as BAIL[w0..w7]; clear of the + // occ[32..36]/occ[39] DSWS sensor+roles slots and occ[50] gq) +.set KSEG_STEPS, (SEGK/16) // K16-steps per split-K segment = SEGK K-elements / 16 +// FIX 1(b): NKSEG_SHIFT = log2(KSEG_STEPS), so the prologue can derive n_kseg = KT >> NKSEG_SHIFT instead +// of receiving it as a (now-dropped) kernarg. SEGK is always a power-of-two multiple of 16 in every +// config this file is built with, so KSEG_STEPS is always a power of two in {1,2,4,8,16}; a static +// ladder over that small set is simpler/safer than a general-purpose compile-time log2. +.if KSEG_STEPS == 1 + .set NKSEG_SHIFT, 0 +.elseif KSEG_STEPS == 2 + .set NKSEG_SHIFT, 1 +.elseif KSEG_STEPS == 4 + .set NKSEG_SHIFT, 2 +.elseif KSEG_STEPS == 8 + .set NKSEG_SHIFT, 3 +.elseif KSEG_STEPS == 16 + .set NKSEG_SHIFT, 4 +.else + .error "KSEG_STEPS (SEGK/16) must be a power of two in {1,2,4,8,16}" +.endif +// resident regions aligned to 256B +.set BRES_OFF, 256 // resident B for current super-tile +.set BRES_BYTES, (FN*16*SEGK) // = 4*16*64 = 4096 at the default config +.set ARES_OFF, (BRES_OFF + BRES_BYTES) // resident A for current super-tile +.set ARES_BYTES, (G*16*FM*SEGK) // = 6*16*2*64 = 12288 at the default config +.set LDS_TOTAL_DSWS2, (ARES_OFF + ARES_BYTES) +// (old single-slot cap check retained; 16640 < 32768 -> always passes. The RING layout below is what +// the ring role loops actually use; its own cap check follows.) +.if LDS_TOTAL_DSWS2 > 32768 + .error "DSWS2 LDS layout exceeds 32768B group segment" +.endif +// ============================================================================================ +// FIX 1 (flow) -- FLOW ECONOMY LDS layout (N-deep pool + per-wave ROLE mailbox). NEW symbols; the +// single-slot control words above stay DEFINED (no bytes; only used under .if DSWS2_CONV/DIAG/TRACE, +// all 0). The flow role loops reference ONLY the symbols below. There is NO publish flag to poll: +// compute streams rowblks from the READY pool (DRAIN_HEAD), feeds stage the next super-tile into the +// next FREE slot (STAGE_HEAD); both frontiers are super-tile indices, slot = (index mod POOL_N). Each +// wave reads its ROLE[wid] mailbox each cycle and simply IS that role (stale mailbox = last role = +// coast). Single writer (coordinator wid0) -> no CAS. See FLOW_ECONOMY_DESIGN.md. +.ifndef POOL_N + .set POOL_N, 3 // pool depth: 3 slots -> 48KB operands + ctrl < 64KB (4 = 64KB, too tight) +.endif +.ifndef ACC_N + .set ACC_N, 1 // per-rowblk fp32 reduction accumulator banks (rowblk-lifetime). CO-BUDGET with +.endif // POOL_N: OP_BASE + POOL_N*OPSTRIDE + ACC_N*ACC_STRIDE <= 65536. Defaults keep + // the POOL_N=3 bare build legal (57600); stagger build uses POOL_N=2 ACC_N=2 (49408). +.ifndef COORD_PERIOD + .set COORD_PERIOD, 64 // coordinator sense/nudge cadence (loop cycles); lazy is fine (waves coast) +.endif +// DEADMAN: per-wave wall-clock watchdog. Every wave stamps its start RTC and, at each loop head, force- +// retires if it has been alive > DEADMAN_TICKS. Converts a COORDINATION hang (a frontier that never +// advances -> waves spin the loop forever) into a CLEAN drain: all waves retire, occ[0]->0, the queue +// goes idle and the EOP fence fires -> NO wedge, NO desktop brick (the result is just incomplete, which +// the oracle flags). This is what makes scale-stress SAFE to run. Fires at 0.5s < host chunkMaxS(0.75s) +// so the host observes the clean drain before its own bail. It does NOT cover the s_alloc_vgpr grow-spin +// (grow-stagger, ISA 3.3.3.2) -- that spin never reaches the loop head; it's a separate gate (M=576 only). +// DEADMAN=0 -> zero bytes (for clean perf/byte-identity bins). +.ifndef DEADMAN + .set DEADMAN, 1 +.endif +.ifndef RETBARRIER + .set RETBARRIER, 1 // count-to-WAVES collective exit: each wave checks in at .Lflow_dead and all +.endif // s_endpgm TOGETHER once the WG's count hits WAVES -> the EOP registers a +.ifndef RETBAR_MAX // clean coordinated dispatch completion -> the fence FIRES (the staggered + .set RETBAR_MAX, 1000000 // coordinator-broadcast retire fires at 8 waves but not 16). Bounded-wait +.endif // (no message bus / no s_alloc) so it can NEVER hang the wave. +.ifndef DEADMAN_TICKS + .set DEADMAN_TICKS, 50000000 // 0.5s @ 100MHz RTC; a normal chunk is ~ms, so this fires ONLY on a wedge +.endif +// ---- shared frontiers + mailbox (single copy, at the front). 3-frontier pipeline: +// DRAIN_HEAD <= STAGE_HEAD <= ASSIGN_HEAD <= DRAIN_HEAD + POOL_N (all monotone u32) ---- +.set ASSIGN_HEAD_OFF, 0 // next local index to assign a global super-tile (SINGLE writer = coordinator wid0) +.set STAGE_HEAD_OFF, 4 // oldest assigned-but-not-fully-staged index (feeds; ds_cmpstore advance) +.set DRAIN_HEAD_OFF, 8 // oldest not-fully-drained index (compute; ds_cmpstore advance) +.set RINGINIT_OFF, 12 // barrier-free LDS-init publish flag (coordinator writes 0xACED LAST) +.set FLOWTERM_OFF, 16 // terminal flag: coordinator sets 0xDEAD once all super-tiles claimed +.set ROLE_BASE, 20 // per-wave mailbox: ROLE[wid] at ROLE_BASE + wid*4 (coordinator-written) +// role codes stored in ROLE[wid]: +.set ROLE_COMPUTE, 0 +.set ROLE_AFEED, 1 +.set ROLE_BFEED, 2 +.set ROLE_RETIRE, 3 +// ---- per-slot control block: SLOTC_BASE + slot*SLOTC_STRIDE + field (same fields as the ring) ---- +// coordinator-local TILE-CLAIM state (single writer = wid0), tucked in the reserved 32-wave mailbox +// tail (safe while WAVES<=30). occ[20] claims whole TILES; the coordinator emits a tile's n_kseg +// super-tiles (sti=(t<= G +.set SL_BFNEXT, 16 // B-frag claim counter (B-feeds) +.set SL_BFDONE, 20 // B-frags stored; slot B-ready when == FN +.set SL_ARNEXT, 24 // A-rowblk claim counter (A-feeds) +.set SL_ARDONE, 28 // A-rowblks staged; slot A-ready when == G (slot READY = B-ready && A-ready) +// ---- per-slot operand buffers: OP_BASE + slot*OPSTRIDE ; BRES at +BRES_ROFF, ARES at +ARES_ROFF ---- +.set OP_BASE, 256 // 256B-aligned; below it: frontier + mailbox + POOL_N ctrl blocks +.set OPSTRIDE, (BRES_BYTES + ARES_BYTES) // 4096 + 12288 = 16384 per slot +.set BRES_ROFF, 0 // resident B within a slot +.set ARES_ROFF, BRES_BYTES // resident A within a slot (after B) +// ---- per-rowblk reduction accumulator pool: ACC_BASE + bank*ACC_STRIDE (bank in [0,ACC_N)) ---- +// fp32, rowblk-lifetime (persists across all n_kseg K-segments of a rowblk); DISTINCT from the +// segment-lifetime operand pool above. One bank = one C-rowblk = FM*FN frags x 1024B. +.set ACC_BASE, (OP_BASE + POOL_N*OPSTRIDE) // after the operand pool +.set ACC_STRIDE, (FM*FN*1024) // = 8192 @ FM=2 FN=4 (one C-rowblk) +.set LDS_TOTAL_FLOW,(ACC_BASE + ACC_N*ACC_STRIDE) // POOL3/ACC1: 57600 ; POOL2/ACC2: 49408 +.if LDS_TOTAL_FLOW > 65536 + .error "FLOW LDS layout exceeds 65536B group segment (hardware WGP limit) -- lower POOL_N or ACC_N" +.endif +.if (SLOTC_BASE + POOL_N*SLOTC_STRIDE) > OP_BASE + .error "FLOW per-slot control blocks overlap the operand region (raise OP_BASE)" +.endif +// Phase-B state must fit in the control gap below the resident region (inert compile check, no bytes). +.if DSWS2_STATE_END > BRES_OFF + .error "DSWS2 Phase-B state (SNAP_BASE/QUIESCE_CNT) overlaps resident B region (BRES_OFF)" +.endif + +.if DSWS2 + // ---- launch wave count (EMERGENT economy: NO baked compute/feed mix; roles emerge at runtime) ---- + .ifndef WAVES + .set WAVES, 16 // waves/WG launched; host launches the SAME count. + .endif + .if WAVES > 30 + .error "WAVES>30 collides with COORD_KSI/T at ROLE[30]/ROLE[31] -- relocate coord state first" + .endif +.endif + +// ============================================================================================ +// VGPR layout (lifted from occ_kernel_coop.s) -- compute frags live ABOVE the lean-32 block and are +// only touched AFTER s_alloc_vgpr NFV. Feeds/claimer stay in the lean block (v0..v31). +// ============================================================================================ +.set ACC, 32 // accumulators: FM*FN frags x 8 f32 (v32..) +.set FA, (ACC + 8*FM*FN) // compute A frags (from resident LDS): FM x 2 +.set FB, (FA + 2*FM) // compute B frags (from resident LDS): FN x 2 +.set NFV, ((FB + 2*FN + 15) & ~15) // grown footprint, rounded to a 16-VGPR dyn block (=112 @ 2x4) +.set VLEAN, 32 // lean footprint (feeds, claimer, compute pre/post rowblk) +.set BSTG, 16 // staging regs (lean block, < 32): B-feed FN-frag / A-feed FM-frag + +// ---- dyn-VGPR PRE-GROW temp-reg ceiling (coop death-cert: a >v15 src pre-grow is poison under dyn). +// Gate every PRE-grow-reachable LDS/atomic temp to v11/v14 (INTERIOR to the launched 16-VGPR block). ---- +.if DYNVGPR + .set RG_A, 11 // lds_get / fetch_add address + .set RG_D, 14 // lds_get / fetch_add data+return + .set RP_A, 11 // lds_put address + .set RP_D, 14 // lds_put data +.else + .set RG_A, 27 + .set RG_D, 28 + .set RP_A, 28 + .set RP_D, 29 +.endif + +// ============================================================================================ +// LDS helper macros (s49 = exec save; v2 = lane = tid&31, set in prologue). +// ============================================================================================ +.macro lds_get sdst, off // wave-uniform read LDS[off] -> scalar sdst + v_mov_b32 v[RG_A], \off + ds_load_b32 v[RG_D], v[RG_A] + s_wait_dscnt 0x0 + v_readfirstlane_b32 \sdst, v[RG_D] +.endm +.macro lds_get_r sdst, saddr // wave-uniform read LDS[saddr] (RUNTIME addr in a sreg) -> sdst + v_mov_b32 v[RG_A], \saddr + ds_load_b32 v[RG_D], v[RG_A] + s_wait_dscnt 0x0 + v_readfirstlane_b32 \sdst, v[RG_D] +.endm +.macro lds_put off, ssrc // lane-0-of-wave writes scalar ssrc -> LDS[off] + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lput_skip\@ + v_mov_b32 v[RP_A], \off + v_mov_b32 v[RP_D], \ssrc + ds_store_b32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lput_skip\@: + s_mov_b32 exec_lo, s49 +.endm +.macro lds_fetch_add sdst, off, val // sdst <- old LDS[off]; LDS[off]+=val (lane-0 atomic, broadcast) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lfa_skip\@ + v_mov_b32 v[RP_A], \off + v_mov_b32 v[RP_D], \val + ds_add_rtn_u32 v[RP_D], v[RP_A], v[RP_D] // v[RP_D] <- old; LDS[off] += val + s_wait_dscnt 0x0 +.Lfa_skip\@: + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 \sdst, v[RP_D] // broadcast lane-0's old value +.endm +.macro lds_inc off // lane-0-of-wave LDS[off] += 1 (no return) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Linc_skip\@ + v_mov_b32 v[RP_A], \off + v_mov_b32 v[RP_D], 1 + ds_add_u32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Linc_skip\@: + s_mov_b32 exec_lo, s49 +.endm +// ---- FIX 1a ring: RUNTIME-address variants (slot-indexed counters live at SLOTC_BASE+slot*32+field, +// a runtime scalar). Mirror lds_fetch_add / lds_inc but take the address in a sreg. ---- +.macro lds_fetch_add_r sdst, saddr, val // sdst <- old LDS[saddr]; LDS[saddr]+=val (lane-0 atomic, bcast) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lfar_skip\@ + v_mov_b32 v[RP_A], \saddr + v_mov_b32 v[RP_D], \val + ds_add_rtn_u32 v[RP_D], v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lfar_skip\@: + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 \sdst, v[RP_D] +.endm +.macro lds_inc_r saddr // lane-0-of-wave LDS[saddr] += 1 (RUNTIME addr, no return) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lincr_skip\@ + v_mov_b32 v[RP_A], \saddr + v_mov_b32 v[RP_D], 1 + ds_add_u32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lincr_skip\@: + s_mov_b32 exec_lo, s49 +.endm +// ---- FIX 1 flow: slot-of-head (index mod POOL_N) + monotone CAS frontier advance ---- +.macro slot_of dst, head, scr // \dst = \head mod POOL_N (\scr = scratch, only used for N=3) +.if POOL_N == 1 + s_mov_b32 \dst, 0 // single tile in flight (stagger model: one tile's g banks fill LDS) +.elseif POOL_N == 2 + s_and_b32 \dst, \head, 1 +.elseif POOL_N == 4 + s_and_b32 \dst, \head, 3 +.elseif POOL_N == 3 + s_mul_hi_u32 \dst, \head, 0xAAAAAAAB // q ~ head/3 (magic-div; q = mulhi>>1) + s_lshr_b32 \dst, \dst, 1 + s_mul_i32 \scr, \dst, 3 + s_sub_u32 \dst, \head, \scr // slot = head - 3*q +.else + .error "slot_of: POOL_N must be in {2,3,4}" +.endif +.endm + +// acc_base_of: \dst = LDS byte address of accumulator bank \bank (bank in [0,ACC_N)) = ACC_BASE + bank*ACC_STRIDE. +// ACC_STRIDE is a compile-time constant, so s_mul_i32 is exact for any FM*FN (no pow2 assumption). +.macro acc_base_of dst, bank + s_mul_i32 \dst, \bank, ACC_STRIDE + s_add_u32 \dst, \dst, ACC_BASE +.endm +.macro lds_cmpstore_adv off, sexp // lane0 monotone bump: if LDS[off]==\sexp -> LDS[off]=\sexp+1 + s_mov_b32 s49, exec_lo + s_add_u32 s60, \sexp, 1 // scalar ALU ignores exec -> safe under the mask + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lcasadv_skip\@ + v_mov_b32 v11, \off // vaddr + v_mov_b32 v14, s60 // vNEW = exp+1 + v_mov_b32 v13, \sexp // vCMP -> vdst old (return unused) + ds_cmpstore_rtn_b32 v13, v11, v14, v13 // MEM=(MEM==exp)?exp+1:MEM (idempotent; losers no-op) + s_wait_dscnt 0x0 +.Lcasadv_skip\@: + s_mov_b32 exec_lo, s49 +.endm +// ---- DEADMAN watchdog: s[70:71] = this wave's start RTC; deadman_check force-retires past the deadline ---- +.macro deadman_stamp // stamp start RTC (low 32b in s70) once at entry +.if DEADMAN + s_sendmsg_rtn_b64 s[70:71], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 +.endif +.endm +.macro deadman_check // if alive > DEADMAN_TICKS -> clean force-retire (no wedge) +.if DEADMAN + s_sendmsg_rtn_b64 s[62:63], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + s_sub_u32 s62, s62, s70 // elapsed = now_lo - start_lo (u32 wrap-safe; deadline << 42s) + s_cmp_ge_u32 s62, DEADMAN_TICKS + s_cbranch_scc1 .Lflow_retire +.endif +.endm +// lds_put_r (RUNTIME-addr write) is also defined inside the .if DSWS2_CONV||DSWS2_ENVELOPE block below; +// the ring needs it at CONV=0/ENV=0, so define an identical copy here, guarded to avoid a double-def +// when either gate is on (the ring is always built CONV=0 ENV=0). +.if !(DSWS2_CONV || DSWS2_ENVELOPE) +.macro lds_put_r saddr, ssrc // lane-0 write ssrc -> LDS[saddr] (RUNTIME addr) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lputr_skip\@ + v_mov_b32 v[RP_A], \saddr + v_mov_b32 v[RP_D], \ssrc + ds_store_b32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lputr_skip\@: + s_mov_b32 exec_lo, s49 +.endm +.endif + +// ============================================================================================ +// Super-tile decode + resident A/B staging macros (A3..A6). Decode (Naming/symbols): +// ksi = sti & mask ; t = sti >> shift ; mblk = t / NTL ; tcol = t % NTL. +// FIX 1(d): n_kseg is ALWAYS a power of two (it's KT >> NKSEG_SHIFT, both compile-time-shift-derived), +// so the sti->(t,ksi) split is an exact shift/mask -- no magic-div, no n_kseg==1 special-case (shift=0, +// mask=0 falls out of the general path for free: ksi=0, t=sti). /NTL still goes via magic (s12), +// unsigned-division mul_hi (coop GENDIV idiom), since NTL is not generally a power of two. +// ============================================================================================ +.macro DECODE_STI // in: s17=sti, s67=mask, s68=shift ; out: s19=mblk s30=tcol s31=ksi ; clob: s18,s36 +.if KMAJOR + s_mul_hi_u32 s31, s17, s76 // ksi = sti / TOTAL (magic-div, magic_TOTAL in s76; K-major high bits) + s_mul_i32 s36, s31, s11 // ksi * TOTAL + s_sub_u32 s18, s17, s36 // t = sti - ksi*TOTAL (low) +.else + s_and_b32 s31, s17, s67 // ksi = sti & mask (mask-bounded -> ksi in [0,n_kseg-1]) + s_lshr_b32 s18, s17, s68 // t = sti >> shift +.endif +.if SAFEPROBE + // brick-PROOF ti clamp (the "future ti clamp" line 752 promised; COOP_STATUS.md:145 racy-garbage-ti->OOB). + // A racy/torn sti read (during the claimer's per-super-tile republish) can decode a garbage t -> garbage + // mblk/tcol -> the A/B/C SCALAR base goes out of buffer -> gfxhub page fault -> MODE1 brick. SAFEPROBE + // already pins the per-lane vaddr (v8/v9/v10); this pins the tile index too, so EVERY global address is + // provably in-buffer. s11=TOTAL is userdata, never clobbered. s36 is DECODE_STI scratch (rewritten below). + s_sub_u32 s36, s11, 1 // TOTAL-1 + s_min_u32 s18, s18, s36 // t clamped to [0,TOTAL-1] -> mblk in-bounds) +.endif + s_mul_hi_u32 s19, s18, s12 // mblk = t / NTL + s_mul_i32 s36, s19, s13 // mblk * NTL + s_sub_u32 s30, s18, s36 // tcol = t - mblk*NTL +.endm + +// RESIDENT B FRAG LAYOUT: B frag (kstep ks, frag f) at BRES_OFF + (ks*FN + f)*256 +// (each frag = the SAME 256B block coop stores per B-ring slot; lane*8 vaddr base = v9). +// Built here as: dst vbase = v9 + BRES_OFF + f*256 , ds_store offset:(ks*FN*256). +// B global addr (lift coop B-feed): Bshuf + tcol*(FN*256=s14) + (seg k0)* [ksi*KSEG_STEPS*(NT*256=s10)] +// + f*256 (frag, folded into saddr) + ks*(NT*256=s10) (k-step, folded into saddr). +.macro BSTAGE // in: s30=tcol s31=ksi ; clob: s20,s21,s23,s25,s26,s27,v13,v[BSTG..] + s_mul_i32 s20, s30, s14 // tcol * FN*256 + s_mul_i32 s21, s31, KSEG_STEPS // ksi * KSEG_STEPS + s_mul_i32 s21, s21, s10 // * NT*256 -> segment k-start byte offset + s_add_u32 s20, s20, s21 + s_add_u32 s20, s4, s20 + s_addc_u32 s21, s5, 0 // s[20:21] = B base (tcol,ksi, seg k-step 0) +.Lbcl\@: + lds_fetch_add s23, BFRAG_NEXT_OFF, 1 // claim frag f + s_cmp_ge_u32 s23, FN + s_cbranch_scc1 .Lbsd\@ // f>=FN -> all frags claimed + s_lshl_b32 s25, s23, 8 // f*256 + s_add_u32 s26, s20, s25 + s_addc_u32 s27, s21, 0 // s[26:27] = frag f base (seg k0) + v_add_nc_u32 v13, v9, BRES_OFF + v_add_nc_u32 v13, v13, s25 // resident B dst vbase for frag f + .set ks, 0 + .rept KSEG_STEPS + global_load_tr_b64 v[BSTG+ks*2:BSTG+ks*2+1], v9, s[26:27] + s_add_u32 s26, s26, s10 // next k-step (last iter over-advances; unused) + s_addc_u32 s27, s27, 0 + .set ks, ks+1 + .endr + s_wait_loadcnt 0x0 + .set ks, 0 + .rept KSEG_STEPS + ds_store_b64 v13, v[BSTG+ks*2:BSTG+ks*2+1] offset:(ks*FN*256) + .set ks, ks+1 + .endr + s_wait_dscnt 0x0 + lds_inc BFRAG_DONE_OFF // frag f STORED -> publish completion (compute gates on this) + s_branch .Lbcl\@ +.Lbsd\@: +.endm + +// RESIDENT A FRAG LAYOUT: A frag (kstep ks, rowblk r, mi) at ARES_OFF + ((ks*G + r)*FM + mi)*256 +// Built as: dst vbase = v9 + ARES_OFF + r*(FM*256) , ds_store offset:((ks*G*FM + mi)*256). +// A global addr (lift coop compute/A-feed): A + rowblk_abs*(16*FM)*K + mi*16*K + koff, rowblk_abs=mblk*G+r, +// koff = ksi*SEGK (segment K byte offset, fp8 1B/elem), k-step within segment via global offset:ks*16. +.macro ASTAGE // in: s19=mblk s31=ksi ; clob: s22,s23,s25,s32,s36,s40,s41,s44,s45,v13,v[BSTG..] + s_lshl_b32 s32, s9, 4 // rowstride16 = 16*K +.Lacl\@: + lds_fetch_add s23, AROW_NEXT_OFF, 1 // claim rowblk r + s_cmp_ge_u32 s23, G + s_cbranch_scc1 .Lasd\@ + s_mul_i32 s36, s19, G + s_add_u32 s36, s36, s23 // rowblk_abs = mblk*G + r + s_mul_i32 s22, s36, (16*FM) + s_mul_i32 s22, s22, s9 // rowblk_abs*(16*FM)*K + s_mul_i32 s25, s31, SEGK // ksi*SEGK (segment K byte offset) + s_add_u32 s22, s22, s25 + s_add_u32 s40, s2, s22 + s_addc_u32 s41, s3, 0 // s[40:41] = A base (rowblk_abs, mi0, seg k0) + s_mul_i32 s25, s23, (FM*256) // r*FM*256 + v_add_nc_u32 v13, v9, ARES_OFF + v_add_nc_u32 v13, v13, s25 // resident A dst vbase for rowblk r + .set mi, 0 + .rept FM + .if mi == 0 + s_mov_b32 s44, s40 + s_mov_b32 s45, s41 + .else + s_add_u32 s44, s44, s32 // += 16*K (next M-frag) + s_addc_u32 s45, s45, 0 + .endif + .set ks, 0 + .rept KSEG_STEPS + global_load_b64 v[BSTG:BSTG+1], v8, s[44:45] offset:(ks*16) + s_wait_loadcnt 0x0 + ds_store_b64 v13, v[BSTG:BSTG+1] offset:((ks*G*FM + mi)*256) + s_wait_dscnt 0x0 + .set ks, ks+1 + .endr + .set mi, mi+1 + .endr + lds_inc AROW_DONE_OFF // rowblk r fully STAGED -> publish completion + s_branch .Lacl\@ +.Lasd\@: +.endm + +// ============================================================================================ +// FIX 1a -- RING staging macros: slot-indexed BSTAGE_R / ASTAGE_R. Identical math to BSTAGE/ASTAGE +// but claim/done counters live in the per-slot control block (\scb = SLOTC_BASE + slot*32, runtime) +// and operands land in the per-slot buffer (\sob = OP_BASE + slot*OPSTRIDE, runtime; B at +// +BRES_ROFF=0, A at +ARES_ROFF). \scb and \sob are READ-only (never clobbered). Internal address +// scratch: s46/s47 (free in the feed context). ds offset immediates are vbase-relative -> unchanged. +// ============================================================================================ +.macro BSTAGE_R scb, sob // in: s30=tcol s31=ksi ; clob: s20,s21,s23,s25,s26,s27,s46,s47,v13,v[BSTG..] + s_mul_i32 s20, s30, s14 // tcol * FN*256 + s_mul_i32 s21, s31, KSEG_STEPS // ksi * KSEG_STEPS + s_mul_i32 s21, s21, s10 // * NT*256 -> segment k-start byte offset + s_add_u32 s20, s20, s21 + s_add_u32 s20, s4, s20 + s_addc_u32 s21, s5, 0 // s[20:21] = B base (tcol,ksi, seg k-step 0) + s_add_u32 s46, \scb, SL_BFNEXT // &SL_BFNEXT[slot] +.Lbclr\@: + lds_fetch_add_r s23, s46, 1 // claim frag f + s_cmp_ge_u32 s23, FN + s_cbranch_scc1 .Lbsdr\@ // f>=FN -> all frags claimed + s_lshl_b32 s25, s23, 8 // f*256 + s_add_u32 s26, s20, s25 + s_addc_u32 s27, s21, 0 // s[26:27] = frag f base (seg k0) + v_add_nc_u32 v13, v9, \sob // + slot operand base + v_add_nc_u32 v13, v13, s25 // + f*256 (BRES_ROFF = 0) + .set ks, 0 + .rept KSEG_STEPS + global_load_tr_b64 v[BSTG+ks*2:BSTG+ks*2+1], v9, s[26:27] + s_add_u32 s26, s26, s10 + s_addc_u32 s27, s27, 0 + .set ks, ks+1 + .endr + s_wait_loadcnt 0x0 + .set ks, 0 + .rept KSEG_STEPS + ds_store_b64 v13, v[BSTG+ks*2:BSTG+ks*2+1] offset:(ks*FN*256) + .set ks, ks+1 + .endr + s_wait_dscnt 0x0 + s_add_u32 s47, \scb, SL_BFDONE + lds_inc_r s47 // frag f STORED -> compute gates on SL_BFDONE==FN + s_branch .Lbclr\@ +.Lbsdr\@: +.endm + +.macro ASTAGE_R scb, sob // in: s19=mblk s31=ksi ; clob: s22,s23,s25,s32,s36,s40,s41,s44,s45,s46,s47,v13,v[BSTG..] + s_lshl_b32 s32, s9, 4 // rowstride16 = 16*K + s_add_u32 s46, \scb, SL_ARNEXT // &SL_ARNEXT[slot] +.Laclr\@: + lds_fetch_add_r s23, s46, 1 // claim rowblk r + s_cmp_ge_u32 s23, G + s_cbranch_scc1 .Lasdr\@ + s_mul_i32 s36, s19, G + s_add_u32 s36, s36, s23 // rowblk_abs = mblk*G + r + s_mul_i32 s22, s36, (16*FM) + s_mul_i32 s22, s22, s9 // rowblk_abs*(16*FM)*K + s_mul_i32 s25, s31, SEGK // ksi*SEGK + s_add_u32 s22, s22, s25 + s_add_u32 s40, s2, s22 + s_addc_u32 s41, s3, 0 // s[40:41] = A base + s_mul_i32 s25, s23, (FM*256) // r*FM*256 + v_add_nc_u32 v13, v9, \sob // + slot operand base + v_add_nc_u32 v13, v13, ARES_ROFF // + A-within-slot offset (BRES_BYTES) + v_add_nc_u32 v13, v13, s25 // + r*FM*256 + .set mi, 0 + .rept FM + .if mi == 0 + s_mov_b32 s44, s40 + s_mov_b32 s45, s41 + .else + s_add_u32 s44, s44, s32 // += 16*K (next M-frag) + s_addc_u32 s45, s45, 0 + .endif + .set ks, 0 + .rept KSEG_STEPS + global_load_b64 v[BSTG:BSTG+1], v8, s[44:45] offset:(ks*16) + s_wait_loadcnt 0x0 + ds_store_b64 v13, v[BSTG:BSTG+1] offset:((ks*G*FM + mi)*256) + s_wait_dscnt 0x0 + .set ks, ks+1 + .endr + .set mi, mi+1 + .endr + s_add_u32 s47, \scb, SL_ARDONE + lds_inc_r s47 // rowblk r STAGED -> compute gates on SL_ARDONE==G + s_branch .Laclr\@ +.Lasdr\@: +.endm + +// ============================================================================================ +// Phase-B (DSWS2_CONV) consume-point ring-occupancy sensor -- Task 3, READ-ONLY (actuation is Task 5). +// Mirrors the coop occ_a/occ_b sensor (occ = producer - consumer, sampled where the value is +// CONSUMED, not at the segment boundary). The claimer's A7 wait-done spin runs CONCURRENTLY with the +// compute drain, so it observes the ring mid-flight; at the segment boundary the resident region has +// fully drained and occ would read a stuck ~0 (permanent false-starvation) -- exactly what SPEC warns. +// +// COUNTER IDENTITIES (confirmed against the live claim/consume sites -- see report): +// producer = the STORE-completion counters the compute wave actually gates on: +// A-ring: AROW_DONE_OFF (A rowblks resident, monotonic in [0,G]; lds_inc @ ASTAGE) +// B-ring: BFRAG_DONE_OFF (B frags resident, monotonic in [0,FN]; lds_inc @ BSTAGE) +// NOT the *_NEXT claim counters: AROW_NEXT/BFRAG_NEXT overshoot the ring depth by the role +// terminal-bails (G+NAFEED / FN+NBFEED), which would break the occ <= depth bound. +// consumer = ROWBLK_NEXT_OFF, the compute rowblk-claim clock (consume progress through the super-tile: +// each claimed rowblk r consumes A(r) and re-reads all FN shared B frags). +// min-clamp: cons is clamped to prod before the subtract so the u32 result cannot underflow when the +// consume clock outruns a shallower ring (G=6 > FN=4 -> ROWBLK_NEXT can exceed BFRAG_DONE). +// INVARIANT preserved: occ_A in [0,G], occ_B in [0,FN] (nonnegative, bounded by ring depth). +// +// REGISTER DISCIPLINE (brick-critical; this path is reachable pre-grow -- a >v15 vector temp is +// OOR-poison under dyn-VGPR, SPEC S4): scalars <= s65 only (s60/s61 scratch; callers pass dst in +// [s62,s65]); the only vector temps are inside lds_get, which uses v11/v14 (INTERIOR to the launch +// 16-VGPR block) -- NO >v15 temp is introduced here. +// GATE: DSWS2_CONV || DSWS2_ENVELOPE. reserve_try + the BUDGET default are the pool-economy primitives the +// rolling envelope needs INDEPENDENTLY of role conversion (they touch only VRESV_OFF/lds_fetch_add), so the +// envelope must be able to run at CONV=0 (the isolation config). Everything in this block is macro/.set +// definition (emits ZERO bytes), so widening the gate is byte-identical at CONV=0/ENV=0 and CONV=1. +.if DSWS2_CONV || DSWS2_ENVELOPE +.macro occ_sample dst_a, dst_b // out: \dst_a=occ_A in [0,G], \dst_b=occ_B in [0,FN]; clob s60,s61 + lds_get \dst_a, AROW_DONE_OFF // prod_a: A rowblks resident (store-completion) + lds_get \dst_b, BFRAG_DONE_OFF // prod_b: B frags resident (store-completion) + lds_get s60, ROWBLK_NEXT_OFF // cons : compute rowblk-claim consume clock + s_min_u32 s61, s60, \dst_a // cons_a = min(clock, prod_a) (clamp -> no u32 underflow) + s_sub_u32 \dst_a, \dst_a, s61 // occ_A = prod_a - cons_a in [0,G] + s_min_u32 s61, s60, \dst_b // cons_b = min(clock, prod_b) + s_sub_u32 \dst_b, \dst_b, s61 // occ_B = prod_b - cons_b in [0,FN] +.endm + +// ---- DSWS2_GQUIESCE: device-scoped GLOBAL QUIESCE handshake (mirrors the green occ[20]/occ[0] pattern). +// All three ops are lane-0-masked (v2==0), exec saved/restored via s49 (the LDS-macro convention -- s49 is +// never live across a macro boundary, so it is provably free at every site these replace an lds_* op). +// vaddr = v4 (the stable occ-base per-lane offset, =0, prologue-set), data/dst = v3/v5 (occ scratch vregs, +// same as the claim/live ops). scope:SCOPE_DEV + uncached occ buffer => device-coherent visibility (the +// fix). s_wait_storecnt/loadcnt drain before proceeding so the poll observes committed bumps. +.macro gq_reset // claimer: occ[QUIESCE_GOFF] = 0 (committed before EPOCH publish) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lgqr_skip\@ + v_mov_b32 v3, 0 + global_store_b32 v4, v3, s[0:1] offset:QUIESCE_GOFF scope:SCOPE_DEV + s_wait_storecnt 0x0 +.Lgqr_skip\@: + s_mov_b32 exec_lo, s49 +.endm +.macro gq_bump // follower: occ[QUIESCE_GOFF] += 1 (one bump/wave/super-tile) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lgqb_skip\@ + v_mov_b32 v3, 1 + global_atomic_add_u32 v4, v3, s[0:1] offset:QUIESCE_GOFF scope:SCOPE_DEV + s_wait_storecnt 0x0 +.Lgqb_skip\@: + s_mov_b32 exec_lo, s49 +.endm +.macro gq_read dst // claimer: \dst = occ[QUIESCE_GOFF] (lane0 load + broadcast) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lgqrd_skip\@ + global_load_b32 v5, v4, s[0:1] offset:QUIESCE_GOFF scope:SCOPE_DEV + s_wait_loadcnt 0x0 +.Lgqrd_skip\@: + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 \dst, v5 +.endm + +// ---- Pool-T7 chunk-2 wedge localization (DIAG-only; DSWS2_CONV=0 emits nothing -> .text byte-identical). +// epoch_mark: lane-0 publishes this role's live epoch (s35) to a host-streamed occ slot so a hung dispatch +// shows how far each role advanced (stream field roles[C/A/B]). v14<=v15 (feeds/compute are lean-32 at the +// _quiesce call sites), v4=0 (occ base lane offset, prologue), s49 exec-save (LDS-macro convention). ---- +.macro epoch_mark off +.if DSWS2_CONV && DIAG + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lem_skip\@ + v_mov_b32 v14, s35 + global_store_b32 v4, v14, s[0:1] offset:\off scope:SCOPE_DEV +.Lem_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +// bail_mark: PER-WAVE localization mark. Lane-0 writes this wave's epoch (s35) to occ[BAIL_BASE + wid*4] +// (runtime vaddr since the offset depends on wid=s24). s48 scratch, s49 exec-save (macro-local; free at the +// _quiesce bail sites), v13 vaddr, v14 data (both <=v15; the wave is lean-32 at every bail site). One-shot +// per super-tile -> negligible perturbation vs the DIAG per-spin claimer stores. Enabled by DIAG OR BAILMARK. +.macro bail_mark +.if DSWS2_CONV && (DIAG || DSWS2_BAILMARK) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lbmk_skip\@ + s_lshl_b32 s48, s24, 2 // wid*4 + s_add_u32 s48, s48, BAIL_BASE // occ byte offset for THIS wave + v_mov_b32 v13, s48 // vaddr = per-wave byte offset (lane0) + v_mov_b32 v14, s35 // data = this wave's current epoch + global_store_b32 v13, v14, s[0:1] scope:SCOPE_DEV +.Lbmk_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +// -------------------------------------------------------------------------------------------- +// Phase-B controller thresholds + sum-envelope budget (Task 4). EPOCH_SHIFT mirrors coop / +// occ_dispatch (epoch = segcnt >> EPOCH_SHIFT). BUDGET is the per-WG VGPR sum-envelope ceiling +// the reservation counter must never exceed; default = the launch reservation, which makes the +// envelope a strict conservation law (a feed->compute grow can only fit if a compute->feed shrink +// already freed the delta). Task 5 may re-tune via `-defsym BUDGET=` if per-SIMD headroom exists. +// -------------------------------------------------------------------------------------------- +.ifndef EPOCH_SHIFT + .set EPOCH_SHIFT, 3 // decision clock: epoch = segcnt >> EPOCH_SHIFT (small = reactive) +.endif +.ifndef VBUDGET + .set VBUDGET, 1536 // physical VGPR-file credit ceiling (R9700 wave32, per SIMD). Calibrate. +.endif // Sanity ceiling only: the hardware s_alloc_vgpr is the real concurrent-fat cap. +.ifndef BUDGET +.if DSWS2_ENVELOPE + .set BUDGET, (WAVES*VLEAN + PEAK_CONC*(NFV-VLEAN)) // rolling: lean floor + concurrent-peak headroom +.else + .set BUDGET, VBUDGET // EMERGENT: budget is PHYSICAL, not mix-derived (ledger is dormant; conv-only). +.endif +.endif + +// emergent-economy PHYSICAL sanity (always on): all waves fit lean, and >=1 can grow. +.if (WAVES * VLEAN) > BUDGET + .error "WAVES*VLEAN exceeds VBUDGET -- pool cannot stay all-lean" +.endif +.if (WAVES*VLEAN + (NFV-VLEAN)) > BUDGET + .error "VBUDGET admits < 1 concurrent grow -- compute can never make progress" +.endif + +// try_gate: the lock-free single-winner conversion ticket (transcribed VERBATIM from occ_kernel_coop.s, +// which transcribes dsws_ctrl_model.cpp gate_try_win + epoch_of EXACTLY). E = segcnt>>EPOCH_SHIFT. +// gate[dir] holds the last epoch dir fired. Among many waves racing the same (gtry_gate->reserve_try point -- +// occ_sample's s62/s63 result is consumed into `dir` BEFORE this runs), v5/v6/v7 (<=v15: pre-grow / +// lean-safe). CAS operand order (gfx1201, GCN order -- NOT flipped, KG 9ed04f3c): +// ds_cmpstore_rtn_b32 vdst,vaddr,vNEW,vCMP -> MEM=(MEM==vCMP)?vNEW:MEM, vdst<-old. So vsrc0=E (new), +// vsrc1=g (compare). WIN iff returned-old == g. (Swapping them leaves gate stuck so old==g for ALL +// racers -> every racer "wins" -> would-win ~= NCOMP*epochs instead of ~= epochs.) +.macro try_gate dir, swin + lds_get s62, SEGCNT_OFF // E = epoch_of(segcnt, EPOCH_SHIFT) + s_lshr_b32 s62, s62, EPOCH_SHIFT + lds_get s63, (GATE_OFF + (\dir)*4) // g = gate[dir] + s_mov_b32 \swin, 0 + s_cmp_ge_u32 s63, s62 // g >= E -> dir already fired this/later epoch -> lose + s_cbranch_scc1 .Ltg_done\@ + s_mov_b32 s65, exec_lo // lane0-only CAS (one ticket attempt per WAVE) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltg_restore\@ + v_mov_b32 v5, (GATE_OFF + (\dir)*4) // vaddr = &gate[dir] + v_mov_b32 v6, s63 // v6 = g (vsrc1 = CMP/expected) + v_mov_b32 v7, s62 // v7 = E (vsrc0 = NEW value to store) + ds_cmpstore_rtn_b32 v6, v5, v7, v6 // gate[dir] = (gate[dir]==g) ? E : gate[dir]; v6 <- old + s_wait_dscnt 0x0 +.Ltg_restore\@: + s_mov_b32 exec_lo, s65 + v_readfirstlane_b32 s64, v6 // s64 = old (lane0's CAS result, broadcast) + s_cmp_eq_u32 s64, s63 // WIN iff old == g (we were the swapper) + s_cbranch_scc0 .Ltg_done\@ + s_mov_b32 \swin, 1 +.Ltg_done\@: +.endm + +// reserve_try: the VGPR sum-envelope reservation (transcribes reserve_grow, dsws_ctrl_model.cpp:47). +// Reserve first (atomic add of SIGNED \delta on vgpr_reserved), then validate prev+delta <= BUDGET; +// on over-budget cleanly UNDO (atomic add of -\delta) and reject. The LDS atomic serializes the <=2 +// concurrent grows an epoch permits: the second to validate sees the first's reservation and backs off. +// GROW (feed->compute): pass \delta = +(NFV-VLEAN). Over-budget -> undo, \won=0 (stay in role). +// SHRINK(compute->feed): pass \delta = -(NFV-VLEAN). new = prev+delta < prev <= BUDGET, so the +// validate branch is a proven no-op -> \won=1 ALWAYS (shrink never fails). +// One macro, one call site (Task 5 `reserve_try delta, s_ok`); direction is the sign of \delta. +// Scratch: s62/s63 (free at the bail-commit point -- try_gate's s62..s65 are long dead by then). +.macro reserve_try delta, won + lds_fetch_add s62, VRESV_OFF, (\delta) // s62 = prev reserved; vgpr_reserved += delta + s_add_u32 s63, s62, (\delta) // s63 = new reservation = prev + delta + s_mov_b32 \won, 1 + s_cmp_le_u32 s63, BUDGET // new <= BUDGET -> commit (win); shrink always passes + s_cbranch_scc1 .Lrt_done\@ + lds_fetch_add s62, VRESV_OFF, -(\delta) // over-budget: undo the reservation, reject + s_mov_b32 \won, 0 +.Lrt_done\@: +.endm + +// -------------------------------------------------------------------------------------------- +// Phase-B (Task 5) watermark thresholds + LDS put-runtime helper + bail-time commit macros. +// Watermark decision (SPEC; mirrors coop CTRL_LOW/CTRL_HIGH, occ_dispatch DSWS_LOW/HIGH): +// occ_X < CTRL_LOW -> compute STARVED for X -> shrink a compute wave into feed-X. +// occ_X > CTRL_HIGH_X -> feed-X OVER-SERVING -> grow a feed-X wave into compute. +// occ_A in [0,G], occ_B in [0,FN] (occ_sample bounds), so the HIGH marks are per-ring-depth. +// -------------------------------------------------------------------------------------------- +.ifndef CTRL_LOW + .set CTRL_LOW, 1 // occ_X < 1 (== 0, ring empty at consume) -> starved +.endif +.ifndef CTRL_HIGH_A + .set CTRL_HIGH_A, (G-1) // occ_A > G-1 -> A-ring saturated -> A-feed over-serving +.endif +.ifndef CTRL_HIGH_B + .set CTRL_HIGH_B, (FN-1) // occ_B > FN-1 -> B-ring saturated -> B-feed over-serving +.endif + +// lds_put_r: lane-0-of-wave write scalar \ssrc -> LDS[\saddr] (RUNTIME byte offset in a sreg). Mirrors +// the coop lds_put_v idiom but takes a SCALAR address (symmetry with lds_get_r). Used by the claimer's +// Step-4 snapshot write into the runtime parity half of SNAP_BASE. Temps RP_A/RP_D are v11/v14 (<=v15, +// pre-grow safe); s49 is the exec save (matches lds_put). +.macro lds_put_r saddr, ssrc + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lputr_skip\@ + v_mov_b32 v[RP_A], \saddr + v_mov_b32 v[RP_D], \ssrc + ds_store_b32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lputr_skip\@: + s_mov_b32 exec_lo, s49 +.endm + +// conv_dec_floor: floor-guarded ATOMIC decrement of a role slot -- \ok <- 1 iff it decremented \slot_off +// (only when the current value was > 1), else 0 (floor hit; source role must keep >= 1 wave). A +// ds_cmpstore_rtn_b32 CAS loop (re-reads on a lost race), so two same-source converters in one epoch +// (e.g. compute->Afeed and compute->Bfeed both dec NCOMP_SLOT) can never drive the slot below 1. +// Clob: s52 (read value), s53 (new/CAS-return), s65 (exec save); v5/v6/v7 (<=v15, pre-grow safe). +.macro conv_dec_floor slot_off, ok + s_mov_b32 \ok, 0 +.Lcdf_retry\@: + lds_get s52, \slot_off // s52 = current source-slot count + s_cmp_le_u32 s52, 1 + s_cbranch_scc1 .Lcdf_done\@ // <=1 -> at floor, cannot convert away (ok stays 0) + s_sub_u32 s53, s52, 1 // new = old - 1 + s_mov_b32 s65, exec_lo // lane0-only CAS (one attempt per WAVE) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lcdf_restore\@ + v_mov_b32 v5, \slot_off // vaddr = &slot + v_mov_b32 v6, s52 // v6 = expected old (CMP) + v_mov_b32 v7, s53 // v7 = new value (NEW) + ds_cmpstore_rtn_b32 v6, v5, v7, v6 // slot = (slot==old)? new : slot ; v6 <- prior + s_wait_dscnt 0x0 +.Lcdf_restore\@: + s_mov_b32 exec_lo, s65 + v_readfirstlane_b32 s53, v6 // s53 = prior (lane0 CAS result, broadcast) + s_cmp_eq_u32 s53, s52 // success iff prior == expected (we were the swapper) + s_cbranch_scc0 .Lcdf_retry\@ // lost the race -> re-read and retry + s_mov_b32 \ok, 1 +.Lcdf_done\@: +.endm + +// conv_apply: the bail-time role-conversion COMMIT (SPEC 3.4 Approach A). Precondition: s58 = s_win +// (1 iff this wave won the (dir,epoch) ticket). Ordered strictly BEFORE the QUIESCE_CNT bump the +// CALLER emits after this macro (the quiesce counter is the snapshot handshake). +// ORDER: (a) floor-guarded dec of \src_slot -> (b) reserve the VGPR sum-envelope \delta (shrink +// always ok; grow may abort over BUDGET) -> (c) on ok: inc \dst_slot, flip private role reg (s59), +// s_alloc_vgpr \alloc_sz (GROW=NFV feed->compute / SHRINK=32 compute->feed) with SCC-retry -> +// (d) on floor-fail or reserve-abort: cancel, remain current role (undo the source dec if a +// reservation abort happened after the dec). +// PRE-GROW OOR WINDOW (SPEC 4, #1 brick risk): the wave is lean-32 on entry; every LDS/atomic temp +// read before the s_alloc_vgpr GROW is <=v15 (occ_sample/try_gate v5/v6/v7 + v11/v14; conv_dec_floor +// v5/v6/v7; lds_fetch_add v11/v14) and every carried scalar is <=s65. NO >v15 source before GROW. +// Clob: s52,s53,s54 (+ conv_dec_floor / reserve_try scratch); s59 = new role slot id (record). +.macro conv_apply src_slot, dst_slot, delta, alloc_sz + s_cmp_eq_u32 s58, 0 + s_cbranch_scc1 .Lca_skip\@ // lost the ticket -> no conversion this bail + conv_dec_floor \src_slot, s54 // (a) floor-guarded atomic dec of source slot + s_cmp_eq_u32 s54, 0 + s_cbranch_scc1 .Lca_skip\@ // floor-fail (source at 1) -> cancel, remain current role + reserve_try (\delta), s53 // (b) reserve VGPR envelope (grow may abort; shrink ok) + s_cmp_eq_u32 s53, 0 + s_cbranch_scc0 .Lca_commit\@ + lds_fetch_add s52, \src_slot, 1 // (d) reserve aborted: UNDO the source dec, cancel + s_branch .Lca_skip\@ +.Lca_commit\@: + lds_fetch_add s52, \dst_slot, 1 // (c) inc dest slot (unbounded -> plain atomic add) + s_mov_b32 s59, \dst_slot // flip private current-role reg (records new role slot id) +.if DIAG || TRACE + // conversion-commit counter (proves a wave ACTUALLY switched role). Lean-32 pre-grow here -> v3/v4<=v15 + // OOR-safe; s49 exec-save (macro-local). (DIAG||TRACE)-gated -> DSWS2_CONV/DIAG=0/TRACE=0 byte-identical. + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lca_cm_skip\@ + v_mov_b32 v3, 1 + global_atomic_add_u32 v4, v3, s[0:1] offset:CONVCNT_OFF scope:SCOPE_DEV // occ[48] += 1 + s_wait_storecnt 0x0 +.Lca_cm_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.if CONV_COOLDOWN > 0 + s_mov_b32 s66, CONV_COOLDOWN // Task 4: committed conversion -> arm cooldown +.endif + // ---- s_alloc_vgpr resize: THE pre-grow OOR window closes here; all reads above were <=v15 ---- +.Lca_alloc\@: + s_alloc_vgpr \alloc_sz // GROW(NFV) / SHRINK(32); SCC-retry (brick-class rule) + s_cbranch_scc0 .Lca_alloc\@ +.Lca_skip\@: +.endm +.endif + +// ============================================================================================ +// TFPROBE wall-span capture (TF throughput probe). Realtime-tick min/max into occ[2]/occ[3], +// mirroring occ_kernel_coop.s's proven timer idiom. Each wave stamps lane-0 only (exec-masked +// via s49, the DSWS exec-save convention); base addr v4==0 holds kernel-wide (prologue, line +// ~765; invariant per the "v4=0 occ base lane offset" note). s[30:31] free at entry (DECODE_STI +// outputs, computed only inside role bodies) and dead at every terminal. Emits ZERO bytes at +// TFPROBE=0 -> production .text byte-identical. +// ============================================================================================ +.macro tfspan op:req, off:req // op = min (entry, occ[2]/off 8) | max (exit, occ[3]/off 12) +.if TFPROBE + s_sendmsg_rtn_b64 s[30:31], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + v_cmp_eq_u32 vcc_lo, 0, v2 // lane 0 of each wave only (v2 = tid & 31) + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltfspan_skip\@ + v_mov_b32 v5, s30 // low 32 bits of the realtime tick + global_atomic_\op\()_u32 v4, v5, s[0:1] offset:\off scope:SCOPE_DEV +.Ltfspan_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +// ============================================================================================ +// PHASEPROBE: in-kernel per-phase tick timer for the COMPUTE wave (the critical path). s77 holds +// this wave's last-stamp RTC (low 32b). phase_reset seeds it; phase_stamp accumulates (now-s77) +// into occ[\off] and re-seeds. Lane-0-only atomic add; occ slots are ABOVE the per-chunk memset +// so they sum over the whole run. Scratch s62/s63 (RTC), s64 (delta) -- all free in CONV=0 compute. +// Six phases -> six occ accumulators (bytes 256..276): +.set PH_FOLLOW_OFF, 256 // occ[64]: waiting on claimer to publish next super-tile +.set PH_STAGE_OFF, 260 // occ[65]: waiting on A/B feeds to stage this super-tile +.set PH_GROW_OFF, 264 // occ[66]: claim rowblk + s_alloc_vgpr GROW 32->112 +.set PH_WMMA_OFF, 268 // occ[67]: LDS frag loads + v_wmma compute +.set PH_FLUSH_OFF, 272 // occ[68]: global_atomic_add_f32 C flush (split-K reduction) +.set PH_SHRINK_OFF, 276 // occ[69]: s_alloc_vgpr SHRINK 112->32 +// ---- STAGINSTR: lightweight write-once diagnostic counters (is the wall feed or compute?). Lane0 +// atomic at branch points ONLY (never inside the WMMA timing region -> no s_wait_storecnt pollution). +// STAGINSTR=0 => emits nothing => byte-identical. Ratio COAST/(COAST+COMP) = compute-wave feed-starve. +.ifndef STAGINSTR + .set STAGINSTR, 0 +.endif +.set STINSTR_COAST, 280 // occ[70]: compute-wave loop iters spent coasting (feed-starved) +.set STINSTR_COMP, 284 // occ[71]: rowblk-segments actually computed + reduced +.set STINSTR_FEED, 288 // occ[72]: feed stage completions (staging throughput) +.set STINSTR_GROWFAIL, 292 // occ[73]: per-burst grow SCC0 (budget full -> coast) = stagger repulsion events +.macro instr_inc off +.if STAGINSTR + s_mov_b32 s57, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 // lane0 of the wave only + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lii_skip\@ + v_mov_b32 v3, 1 + global_atomic_add_u32 v4, v3, s[0:1] offset:\off scope:SCOPE_DEV // v4=occ base vaddr(0), v3=1 +.Lii_skip\@: + s_mov_b32 exec_lo, s57 +.endif +.endm +// Per-wave phase accumulators live in SGPRs s78..s83 (NO per-stamp store -> zero memory perturbation, no +// s_wait_storecnt pollution). s77 = last-stamp RTC. phase_flush emits them ONCE at compute retire. +.macro phase_reset +.if PHASEPROBE + s_mov_b32 s78, 0 + s_mov_b32 s79, 0 + s_mov_b32 s80, 0 + s_mov_b32 s81, 0 + s_mov_b32 s82, 0 + s_mov_b32 s83, 0 + s_sendmsg_rtn_b64 s[62:63], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + s_mov_b32 s77, s62 +.endif +.endm +.macro phase_stamp acc:req // \acc += (now - s77); s77 = now (pure scalar, no store) +.if PHASEPROBE + s_sendmsg_rtn_b64 s[62:63], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + s_sub_u32 s64, s62, s77 // delta (mod 2^32; phase << 2^32 so wrap-safe) + s_mov_b32 s77, s62 + s_add_u32 \acc, \acc, s64 +.endif +.endm +.macro phase_flush // lane0 atomic-adds s78..s83 -> occ[64..69]; drained here (not the hot loop) +.if PHASEPROBE + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lphf_skip\@ + v_mov_b32 v5, s78 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_FOLLOW_OFF scope:SCOPE_DEV + v_mov_b32 v5, s79 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_STAGE_OFF scope:SCOPE_DEV + v_mov_b32 v5, s80 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_GROW_OFF scope:SCOPE_DEV + v_mov_b32 v5, s81 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_WMMA_OFF scope:SCOPE_DEV + v_mov_b32 v5, s82 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_FLUSH_OFF scope:SCOPE_DEV + v_mov_b32 v5, s83 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_SHRINK_OFF scope:SCOPE_DEV + s_wait_storecnt 0x0 +.Lphf_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +// ============================================================================================ +// TRACE: per-super-tile time-series row (claimer, lane 0). Written once per super-tile at the +// quiesce-satisfied drain-exit (.Lqc_q_ok). Captures the adaptive wave-role economy over time: +// the LIVE role slots (do waves convert?), the per-super-tile ring-occupancy PEAK (s73/s74, +// tracked across the wait_done spins), the cumulative conversion count, and the envelope vresv. +// 16 u32/row -> buffer[segcnt*64]; bounded by MAXROWS (s72). Emits ZERO bytes at TRACE=0. +// Persistent trace regs: s70:s71 = buffer VA, s72 = MAXROWS, s73/s74 = ring occA/occB peak. +// ============================================================================================ +.macro alllive_dec // TRACE: --live on wave exit (pairs with the entry ++ for peak-concurrent) +.if TRACE + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lalld_skip\@ + v_mov_b32 v3, -1 + global_atomic_add_u32 v4, v3, s[0:1] offset:ALLLIVE_OFF scope:SCOPE_DEV +.Lalld_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +.macro trace_row +.if TRACE + // claim a GLOBALLY-unique row index (all WGs' claimers share occ[55]) -> no per-WG SEGCNT collision. + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + v_mov_b32 v14, 1 + global_atomic_add_u32 v15, v4, v14, s[0:1] offset:TRACE_IDX_OFF th:TH_ATOMIC_RETURN scope:SCOPE_DEV // v15=old idx, v4=addr(0), v14=data(1) + s_wait_loadcnt 0x0 + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 s52, v15 // s52 = unique row index (old value returned by the atomic) + s_cmp_ge_u32 s52, s72 // row >= MAXROWS -> skip (buffer bound) + s_cbranch_scc1 .Ltrow_skip\@ + s_lshl_b32 s53, s52, 6 // row * TRACE_ROW_BYTES(64) + s_add_u32 s60, s70, s53 + s_addc_u32 s61, s71, 0 // s[60:61] = row base VA + s_sendmsg_rtn_b64 s[58:59], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + lds_get s55, SEGCNT_OFF // this WG's per-WG super-tile counter (data field, not index) + v_mov_b32 v14, s58 // 0 tick_lo + v_mov_b32 v15, s55 // 1 segcnt (per-WG) + v_mov_b32 v16, s35 // 2 epoch + lds_get s54, NCOMP_SLOT + v_mov_b32 v17, s54 // 3 nComp (live role slot) + lds_get s54, NAFEED_SLOT + v_mov_b32 v18, s54 // 4 nAfeed + lds_get s54, NBFEED_SLOT + v_mov_b32 v19, s54 // 5 nBfeed + v_mov_b32 v20, s73 // 6 occA peak (across wait_done spins) + v_mov_b32 v21, s74 // 7 occB peak + global_load_b32 v22, v4, s[0:1] offset:CONVCNT_OFF scope:SCOPE_DEV // 8 convCount (cumulative) + lds_get s54, VRESV_OFF + v_mov_b32 v23, s54 // 9 vresv (envelope budget) + v_mov_b32 v24, s17 // 10 sti (claimed super-tile id) + lds_get s54, QUIESCE_CNT_OFF + v_mov_b32 v25, s54 // 11 quiesce (final) + v_mov_b32 v26, s59 // 12 tick_hi + v_mov_b32 v27, s69 // 13 chunkHi (context) + v_mov_b32 v28, s75 // 14 wg_id (which workgroup's economy this row belongs to) + v_mov_b32 v29, 0 // 15 reserved + s_wait_loadcnt 0x0 // convCount load drained before the row store + v_cmp_eq_u32 vcc_lo, 0, v2 // lane 0 of the claimer writes the row + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltrow_wskip\@ + global_store_b128 v4, v[14:17], s[60:61] offset:0 scope:SCOPE_DEV + global_store_b128 v4, v[18:21], s[60:61] offset:16 scope:SCOPE_DEV + global_store_b128 v4, v[22:25], s[60:61] offset:32 scope:SCOPE_DEV + global_store_b128 v4, v[26:29], s[60:61] offset:48 scope:SCOPE_DEV +.Ltrow_wskip\@: + s_mov_b32 exec_lo, s49 +.Ltrow_skip\@: +.endif +.endm + +// ============================================================================================ +// KERNEL +// ============================================================================================ + .text + .globl occ_kernel + .p2align 8 + .type occ_kernel,@function +occ_kernel: + // ---- FIX 1(b,c): derive n_kseg from KT (s8) + the compile-time NKSEG_SHIFT, then the shift/mask + // decode pair, into the reserved high SGPRs s66/s67/s68 BEFORE any clobber (SAFEPROBE below reuses + // s16 purely as scratch; the role bodies keep all transients <= s65). No v2 kernargs are read here + // anymore -- s15/s16/s17 are NOT hardware-preloaded under USER_SGPR=15 (see KERNARG CONTRACT above). ---- + s_lshr_b32 s66, s8, NKSEG_SHIFT // n_kseg = KT >> NKSEG_SHIFT (KT=s8) + s_ff1_i32_b32 s68, s66 // shift = log2(n_kseg) (bit index of the single set bit; n_kseg=1 -> 0) + s_sub_u32 s67, s66, 1 // mask = n_kseg - 1 + // ---- identity (lifted from coop prologue; v0=tid hardware-preloaded) ---- + v_lshrrev_b32 v1, 5, v0 // wid = tid >> 5 + v_and_b32 v2, 31, v0 // lane = tid & 31 + v_and_b32 v6, 15, v0 // lane & 15 (A vaddr) + v_mov_b32 v4, 0 +.if KMAJOR + global_load_b32 v3, v4, s[0:1] offset:248 scope:SCOPE_DEV // occ[62] = magic(TOTAL), host-written + s_wait_loadcnt 0x0 + v_readfirstlane_b32 s76, v3 // s76 = magic_TOTAL, persistent for every DECODE_STI (K-major) +.endif + // ---- per-lane address constants (mbgemm-identical; dyn-VGPR arming compatible) ---- + v_mul_lo_u32 v8, v6, s9 // (lane&15)*K + v_bfe_u32 v7, v0, 4, 1 + v_lshlrev_b32 v7, 3, v7 + v_add_nc_u32 v8, v8, v7 // v8 = A vaddr = (lane&15)*K + colhi*8 + v_lshlrev_b32 v9, 3, v2 // v9 = B/ds vaddr = lane*8 + v_lshlrev_b32 v10, 5, v2 // v10 = C store vaddr = lane*32 +.if SAFEPROBE + // brick-PROOF: clamp the per-lane VECTOR address regs to a loose upper bound (>= true max) so even a + // grow-corrupted vaddr cannot push a global access past the data+guard (pairs with the future ti clamp). + s_lshl_b32 s16, s9, 4 // 16*K (>= v8 max = (lane&15)*K + colhi*8 = 15*K+8) + v_min_u32 v8, s16, v8 // clamp A vaddr + v_min_u32 v9, 0x100, v9 // clamp B/ds vaddr (256 >= lane*8 max 248) + v_min_u32 v10, 0x400, v10 // clamp C vaddr (1024 >= lane*32 max 992) +.endif + + tfspan min, 8 // TFPROBE: every wave stamps occ[2] = min entry tick (wall-span start) +.if TRACE + // total-occupancy: every wave ++live at entry, atomic-max the peak concurrent resident count (occ[1]). + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lall_enter_skip + v_mov_b32 v3, 1 + global_atomic_add_u32 v5, v4, v3, s[0:1] offset:ALLLIVE_OFF th:TH_ATOMIC_RETURN scope:SCOPE_DEV + s_wait_loadcnt 0x0 + v_add_nc_u32 v5, v5, 1 + global_atomic_max_u32 v4, v5, s[0:1] offset:4 scope:SCOPE_DEV // occ[1] = peak concurrent resident waves +.Lall_enter_skip: + s_mov_b32 exec_lo, s49 +.endif + +.if DSWS2 +// ============================================================================================ +// FIX 1 -- FLOW ECONOMY unified role section (replaces the ring's dispatcher/feed/compute). +// Every wave runs ONE loop: read ROLE[wid] mailbox -> be that role (resize on change) -> +// try_grab one atomic (work-or-empty) -> do work, or COAST (code-path flip, no resize). NO +// publish poll anywhere. wid0 = coordinator (single writer): assigns super-tiles to free slots, +// seeds/nudges mailboxes, and also does lean B-feed work. See FLOW_ECONOMY_DESIGN.md. +// Persistent regs: s24=wid s34=cur_role s50=coord period ctr s69=chunkHi +// 3-frontier pipeline (LDS): DRAIN_HEAD <= STAGE_HEAD <= ASSIGN_HEAD <= DRAIN_HEAD+POOL_N +// ============================================================================================ + v_readfirstlane_b32 s24, v1 // wid (uniform) +.if DYNVGPR +.Lflow_alloc: + s_alloc_vgpr 32 // all start lean; compute grows on adopting ROLE_COMPUTE + s_cbranch_scc0 .Lflow_alloc // (NOTE: a bounded-exit + s_endpgm here BRICKS -- s_endpgm from a +.endif // wave that failed s_alloc_vgpr corrupts the SIMD dyn-VGPR pool + // -> OOB page fault. Do NOT exit a starved wave; cap W_launch.) + // live++ : lane0 occ[0] += 1 + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lflow_live + v_mov_b32 v3, 1 + global_atomic_add_u32 v4, v3, s[0:1] scope:SCOPE_DEV +.Lflow_live: + s_mov_b32 exec_lo, s16 + s_mov_b32 s34, 0xFFFFFFFF // cur_role = none -> first .Lflow_body forces a resize + s_mov_b32 s50, 0 // coordinator period counter + deadman_stamp // s[70:71] = start RTC (watchdog baseline) + s_cmp_eq_u32 s24, 0 + s_cbranch_scc0 .Lflow_wait_init // non-coordinator waits for LDS init + // ---- coordinator (wid0) barrier-free LDS init ---- + lds_put ASSIGN_HEAD_OFF, 0 + lds_put STAGE_HEAD_OFF, 0 + lds_put DRAIN_HEAD_OFF, 0 + lds_put FLOWTERM_OFF, 0 +.if RETBARRIER + lds_put QUIESCE_CNT_OFF, 0 // count-to-WAVES collective-exit counter (reset per dispatch) +.endif + // EMERGENT economy seed: minimal liveness FLOOR + everything else COMPUTE. wid0=coordinator (runs + // lean B-feed between ASSIGN duties), wid1=dedicated A-feed, wid2=dedicated B-feed; wid>=3=COMPUTE. + // Excess compute waves self-distribute to feed via .Lflow_coast; concurrent-fat emerges from the + // hardware s_alloc_vgpr grow-fail. NO baked NCOMP/NAFEED/NBFEED. + .set w, 0 + .rept WAVES + .if w == 0 + lds_put (ROLE_BASE + w*4), ROLE_BFEED + .elseif w == 1 + lds_put (ROLE_BASE + w*4), ROLE_AFEED + .elseif w == 2 + lds_put (ROLE_BASE + w*4), ROLE_BFEED + .else + lds_put (ROLE_BASE + w*4), ROLE_COMPUTE + .endif + .set w, w+1 + .endr + // init POOL_N slot control blocks: STAMP = sentinel, all counters 0 + .set sl, 0 + .rept POOL_N + lds_put (SLOTC_BASE + sl*SLOTC_STRIDE + SL_STI), 0xFFFFFFFF + lds_put (SLOTC_BASE + sl*SLOTC_STRIDE + SL_GEN), 0 + lds_put (SLOTC_BASE + sl*SLOTC_STRIDE + SL_RBNEXT), 0 + lds_put (SLOTC_BASE + sl*SLOTC_STRIDE + SL_RBDONE), 0 + lds_put (SLOTC_BASE + sl*SLOTC_STRIDE + SL_BFNEXT), 0 + lds_put (SLOTC_BASE + sl*SLOTC_STRIDE + SL_BFDONE), 0 + lds_put (SLOTC_BASE + sl*SLOTC_STRIDE + SL_ARNEXT), 0 + lds_put (SLOTC_BASE + sl*SLOTC_STRIDE + SL_ARDONE), 0 + .set sl, sl+1 + .endr + lds_put COORD_KSI_OFF, 0xFFFFFFFF // tile-claim sentinel: first ASSIGN claims a fresh tile + lds_put RINGINIT_OFF, 0xACED // LAST: publishes "LDS ready" + global_load_b32 v6, v4, s[0:1] offset:24 scope:SCOPE_DEV + s_wait_loadcnt 0x0 + v_readfirstlane_b32 s69, v6 // chunkHi + s_branch .Lflow_loop +.Lflow_wait_init: + s_sleep 1 + deadman_check // watchdog: wid0 never published init -> clean retire, no wedge + lds_get s44, RINGINIT_OFF + s_cmp_eq_u32 s44, 0xACED + s_cbranch_scc0 .Lflow_wait_init + global_load_b32 v6, v4, s[0:1] offset:24 scope:SCOPE_DEV + s_wait_loadcnt 0x0 + v_readfirstlane_b32 s69, v6 // chunkHi + +// ======================= the unified flow loop ======================= +.Lflow_loop: + deadman_check // watchdog at every loop head: a stalled frontier -> clean drain + s_cmp_eq_u32 s24, 0 + s_cbranch_scc0 .Lflow_body // non-coordinator -> straight to role work + // ---- coordinator duty (wid0): ASSIGN + (later) sense/nudge ---- + lds_get s44, FLOWTERM_OFF + s_cmp_eq_u32 s44, 0xDEAD + s_cbranch_scc1 .Lflow_drainwait // already terminal -> wait for drain + lds_get s44, ASSIGN_HEAD_OFF // ah + lds_get s45, DRAIN_HEAD_OFF // dh + s_sub_u32 s46, s44, s45 + s_cmp_ge_u32 s46, POOL_N + s_cbranch_scc1 .Lflow_coord_period // pool full -> no assign this cycle + // TILE-CLAIM: write-once needs a WG to own a whole tile's n_kseg segments so its LDS banks sum a + // full tile. occ[20] now counts TILES; emit n_kseg super-tiles sti=(t< continue current tile + s_cbranch_scc1 .Lflow_same_tile + // ksi exhausted (or sentinel) -> claim a NEW tile: lane0 occ[20]++ (counts tiles) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lflow_claim_done + v_mov_b32 v3, 1 + global_atomic_add_u32 v5, v4, v3, s[0:1] offset:20 th:TH_ATOMIC_RETURN scope:SCOPE_DEV + s_wait_loadcnt 0x0 +.Lflow_claim_done: + s_mov_b32 exec_lo, s16 + v_readfirstlane_b32 s56, v5 // t = claimed tile index + s_cmp_ge_u32 s56, s69 // t >= chunkHi (=TOTAL tiles) -> terminal + s_cbranch_scc1 .Lflow_set_term + lds_put COORD_T_OFF, s56 // remember current tile + s_mov_b32 s55, 0 // ksi cursor = 0 + s_branch .Lflow_form_sti +.Lflow_same_tile: + lds_get s56, COORD_T_OFF // reuse current tile +.Lflow_form_sti: + s_lshl_b32 s17, s56, s68 // t << shift + s_or_b32 s17, s17, s55 // sti = (t< keep helping (coordinator feeds/coasts) + // all assigned super-tiles drained -> tell everyone to retire, then retire self + .set w, 0 + .rept WAVES + lds_put (ROLE_BASE + w*4), ROLE_RETIRE + .set w, w+1 + .endr + s_branch .Lflow_retire + +// ---- role adopt + dispatch (every wave) ---- +.Lflow_body: + s_lshl_b32 s45, s24, 2 + s_add_u32 s45, s45, ROLE_BASE + lds_get_r s35, s45 // role = ROLE[wid] (stale == last role == coast, free) + s_cmp_eq_u32 s35, ROLE_RETIRE + s_cbranch_scc1 .Lflow_retire + s_cmp_eq_u32 s35, s34 // role unchanged? + s_cbranch_scc1 .Lflow_dispatch + // role changed -> resize at this lean boundary. + // STAGGER: compute waves grow PER-BURST inside .Lflow_compute (the trapezoid), NOT once at role-adopt. + // So a role change never GROWS here; every wave sits LEAN at the loop head (a compute wave shrank + // after its last burst). A defensive shrink-to-lean preserves that invariant if ever fat crossing here. +.if DYNVGPR +.Lflow_shrink: + s_alloc_vgpr 32 + s_cbranch_scc0 .Lflow_shrink +.Lflow_resized: +.endif + s_mov_b32 s34, s35 // cur_role = role +.Lflow_dispatch: + s_cmp_eq_u32 s34, ROLE_COMPUTE + s_cbranch_scc1 .Lflow_compute + s_branch .Lflow_feed + +// ---- COMPUTE work (wave is fat): pull one rowblk from the DRAIN_HEAD slot, WMMA, flush ---- +.Lflow_compute: + lds_get s46, DRAIN_HEAD_OFF // dh + lds_get s44, STAGE_HEAD_OFF // sh + s_cmp_ge_u32 s46, s44 // DRAIN >= STAGE -> nothing fully staged -> coast to feed + s_cbranch_scc1 .Lflow_coast + slot_of s45, s46, s47 // slot = dh mod N + s_lshl_b32 s48, s45, 5 + s_add_u32 s48, s48, SLOTC_BASE // scb + s_lshl_b32 s52, s45, 14 + s_add_u32 s52, s52, OP_BASE // sob + // PER-BURST GROW: trapezoid peak starts here (fat through WMMA+ds_add, lean otherwise). COAST-ON-FAIL + // is the floodgate: if the SIMD VGPR budget is full, grow SCC0 -> coast lean, committing NO claim. +.if DYNVGPR + s_alloc_vgpr NFV + s_cbranch_scc0 .Lflow_growfail +.endif + s_add_u32 s45, s48, SL_RBNEXT + lds_fetch_add_r s33, s45, 1 // claim rowblk r (committed only AFTER grow succeeds) + s_cmp_ge_u32 s33, G + s_cbranch_scc1 .Lflow_cmp_tryadv // rowblks exhausted (we are fat) -> shrink + try advance + // read STAMP (gsti) for C addressing + s_add_u32 s45, s48, SL_STI + lds_get_r s17, s45 + DECODE_STI // s19=mblk s30=tcol s31=ksi + // zero FM*FN accumulators + .set idx, 0 + .rept FM*FN + v_mov_b32 v[ACC+idx*8+0], 0 + v_mov_b32 v[ACC+idx*8+1], 0 + v_mov_b32 v[ACC+idx*8+2], 0 + v_mov_b32 v[ACC+idx*8+3], 0 + v_mov_b32 v[ACC+idx*8+4], 0 + v_mov_b32 v[ACC+idx*8+5], 0 + v_mov_b32 v[ACC+idx*8+6], 0 + v_mov_b32 v[ACC+idx*8+7], 0 + .set idx, idx+1 + .endr + v_add_nc_u32 v12, v9, s52 // B resident base (BRES_ROFF=0) + s_mul_i32 s37, s33, (FM*256) + v_add_nc_u32 v13, v9, s52 + v_add_nc_u32 v13, v13, ARES_ROFF + v_add_nc_u32 v13, v13, s37 // A resident base for rowblk r + .set ks, 0 + .rept KSEG_STEPS + .set ni, 0 + .rept FN + ds_load_b64 v[FB+ni*2:FB+ni*2+1], v12 offset:((ks*FN+ni)*256) + .set ni, ni+1 + .endr + .set mi, 0 + .rept FM + ds_load_b64 v[FA+mi*2:FA+mi*2+1], v13 offset:((ks*G*FM+mi)*256) + .set mi, mi+1 + .endr + s_wait_dscnt 0x0 + .set mi, 0 + .rept FM + .set ni, 0 + .rept FN + v_wmma_f32_16x16x16_fp8_fp8 v[ACC+(mi*FN+ni)*8:ACC+(mi*FN+ni)*8+7], v[FA+mi*2:FA+mi*2+1], v[FB+ni*2:FB+ni*2+1], v[ACC+(mi*FN+ni)*8:ACC+(mi*FN+ni)*8+7] + .set ni, ni+1 + .endr + .set mi, mi+1 + .endr + .set ks, ks+1 + .endr + // WRITE-ONCE REDUCE: accumulate this segment's partial into LDS bank[r] (mirrors C frag layout; + // vaddr = v10=lane*32, base = ACC_BASE + r*ACC_STRIDE). ksi==0 (tile's first segment, POOL_N=1 + // guarantees it drains before any later ksi) WRITES; ksi>0 ADDS. C is stored ONCE at ksi==mask + // (last segment) in .Lflow_cmp_tryadv. s31=ksi (survives WMMA), s33=rowblk r. + acc_base_of s39, s33 // s39 = ACC_BASE + r*ACC_STRIDE + v_add_nc_u32 v12, v10, s39 // v12 = bank r ds vaddr (lane*32 + bankbase) + s_cmp_eq_u32 s31, 0 // first segment of this tile's rowblk? + s_cbranch_scc1 .Lflow_bankwr + .set frag, 0 + .rept FM*FN + .set e, 0 + .rept 8 + ds_add_f32 v12, v[ACC+frag*8+e] offset:(frag*1024 + e*4) + .set e, e+1 + .endr + .set frag, frag+1 + .endr + s_branch .Lflow_bankdn +.Lflow_bankwr: + .set frag, 0 + .rept FM*FN + .set e, 0 + .rept 8 + ds_store_b32 v12, v[ACC+frag*8+e] offset:(frag*1024 + e*4) + .set e, e+1 + .endr + .set frag, frag+1 + .endr +.Lflow_bankdn: + s_wait_dscnt 0x0 + instr_inc STINSTR_COMP // diag: a rowblk-segment was actually computed+reduced + s_add_u32 s45, s48, SL_RBDONE + lds_fetch_add_r s47, s45, 1 // s47 = old RBDONE; old==G-1 -> I am the UNIQUE completer +.if DYNVGPR +.Lflow_bshrink: + s_alloc_vgpr 32 // SHRINK -> lean (close the trapezoid burst) BEFORE any store + s_cbranch_scc0 .Lflow_bshrink +.endif + s_add_u32 s47, s47, 1 + s_cmp_ge_u32 s47, G // (old+1) >= G -> I completed this super-tile + s_cbranch_scc0 .Lflow_loop // not the completer -> done, loop + // COMPLETER (single wave -> NO race, NO redundant store, NO spinning losers): the super-tile is fully + // reduced. If it's the tile's LAST ksi (ksi==mask), store the G banks to C ONCE, s_wait_storecnt, + // THEN advance DRAIN -> the next tile's ksi=0 cannot overwrite the banks until this store is drained. + // s19/s30/s31 still hold mblk/tcol/ksi from this wave's own DECODE_STI (untouched by the reduce). + s_cmp_eq_u32 s31, s67 // ksi == mask (n_kseg-1) -> tile complete? + s_cbranch_scc0 .Lflow_drain_adv // not last ksi -> just advance DRAIN (no store) + s_mul_i32 s38, s19, s13 // mblk*NTL + s_add_u32 s38, s38, s30 // + tcol + s_mul_i32 s38, s38, (G*FM*FN*1024) // * per-tile C bytes + s_add_u32 s28, s6, s38 + s_addc_u32 s29, s7, 0 // s[28:29] = C tile base (rowblk 0) + .set r, 0 + .rept G + s_mov_b32 s39, (ACC_BASE + r*(FM*FN*1024)) // bank r LDS base (compile-time) + v_add_nc_u32 v12, v10, s39 // v12 = bank r ds vaddr (lane*32) + .set frag, 0 + .rept FM*FN + .set e, 0 + .rept 8 + ds_load_b32 v13, v12 offset:(frag*1024 + e*4) + s_wait_dscnt 0x0 + global_store_b32 v10, v13, s[28:29] offset:(r*(FM*FN*1024) + frag*1024 + e*4) scope:SCOPE_DEV + .set e, e+1 + .endr + .set frag, frag+1 + .endr + .set r, r+1 + .endr + s_wait_storecnt 0x0 // store COMPLETE before DRAIN++ -> banks safe to reuse +.Lflow_drain_adv: + lds_get s44, DRAIN_HEAD_OFF + lds_cmpstore_adv DRAIN_HEAD_OFF, s44 // completer advances DRAIN (unique wave; store already done) + s_branch .Lflow_loop +.Lflow_cmp_tryadv: +.if DYNVGPR +.Lflow_tashrink: + s_alloc_vgpr 32 // grew but rowblks exhausted (no claim) -> shrink back lean + s_cbranch_scc0 .Lflow_tashrink +.endif + s_branch .Lflow_loop // the bank store + DRAIN advance are done by the COMPLETER + // (the unique wave whose SL_RBDONE inc hit G, in .Lflow_bankdn) + +// ---- FEED work: stage the STAGE_HEAD slot (A if ROLE_AFEED, B if ROLE_BFEED), then try-advance STAGE ---- +.Lflow_feed: + lds_get s44, STAGE_HEAD_OFF // sh + lds_get s45, ASSIGN_HEAD_OFF // ah + s_cmp_ge_u32 s44, s45 // STAGE >= ASSIGN -> nothing assigned to stage -> yield + s_cbranch_scc1 .Lflow_feed_empty + slot_of s46, s44, s47 // slot = sh mod N + s_lshl_b32 s48, s46, 5 + s_add_u32 s48, s48, SLOTC_BASE // scb + s_lshl_b32 s52, s46, 14 + s_add_u32 s52, s52, OP_BASE // sob + s_add_u32 s45, s48, SL_STI + lds_get_r s17, s45 // gsti = STAMP (assigned -> set) + DECODE_STI + s_cmp_eq_u32 s34, ROLE_BFEED + s_cbranch_scc1 .Lflow_stageB + ASTAGE_R s48, s52 + s_branch .Lflow_stage_adv +.Lflow_stageB: + BSTAGE_R s48, s52 +.Lflow_stage_adv: + // try-advance STAGE_HEAD: if the CURRENT STAGE_HEAD slot is fully staged, CAS-bump it + lds_get s44, STAGE_HEAD_OFF + lds_get s45, ASSIGN_HEAD_OFF + s_cmp_ge_u32 s44, s45 + s_cbranch_scc1 .Lflow_loop + slot_of s46, s44, s47 + s_lshl_b32 s48, s46, 5 + s_add_u32 s48, s48, SLOTC_BASE + s_add_u32 s45, s48, SL_BFDONE + lds_get_r s47, s45 + s_cmp_lt_u32 s47, FN + s_cbranch_scc1 .Lflow_loop + s_add_u32 s45, s48, SL_ARDONE + lds_get_r s47, s45 + s_cmp_lt_u32 s47, G + s_cbranch_scc1 .Lflow_loop + lds_cmpstore_adv STAGE_HEAD_OFF, s44 + s_branch .Lflow_loop +.Lflow_feed_empty: + s_sleep SLEEPN // lean feed can't compute without a grow -> yield; coordinator rebalances + s_branch .Lflow_loop + +// ---- COAST: a fat COMPUTE wave with no staged work runs feed code (FREE, no resize) to help staging ---- +.Lflow_growfail: + instr_inc STINSTR_GROWFAIL // diag: per-burst grow failed (budget full) -> coast = stagger repulsion + // fall through: the failed grow allocated nothing, so we are still lean and safe to coast +.Lflow_coast: + instr_inc STINSTR_COAST // diag: a wave coasted (no staged work, or a grow-fail) + lds_get s44, STAGE_HEAD_OFF + lds_get s45, ASSIGN_HEAD_OFF + s_cmp_ge_u32 s44, s45 + s_cbranch_scc1 .Lflow_feed_empty // nothing assigned to stage -> yield + slot_of s46, s44, s47 + s_lshl_b32 s48, s46, 5 + s_add_u32 s48, s48, SLOTC_BASE + s_lshl_b32 s52, s46, 14 + s_add_u32 s52, s52, OP_BASE + s_add_u32 s45, s48, SL_BFDONE + lds_get_r s47, s45 + s_cmp_lt_u32 s47, FN + s_cbranch_scc1 .Lflow_coastB // B behind -> help B + s_add_u32 s45, s48, SL_STI + lds_get_r s17, s45 + DECODE_STI + ASTAGE_R s48, s52 + s_branch .Lflow_stage_adv +.Lflow_coastB: + s_add_u32 s45, s48, SL_STI + lds_get_r s17, s45 + DECODE_STI + BSTAGE_R s48, s52 + s_branch .Lflow_stage_adv + +.Lflow_retire: + // live-- : lane0 occ[0] -= 1 (harness completion gate) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lflow_dead + v_mov_b32 v3, -1 + global_atomic_add_u32 v4, v3, s[0:1] scope:SCOPE_DEV +.Lflow_dead: + s_mov_b32 exec_lo, s16 + tfspan max, 12 // TFPROBE: wall-span end + alllive_dec +.if RETBARRIER + // COUNT-TO-WAVES collective exit: check in, then all s_endpgm TOGETHER once the WG's count hits WAVES. + // This coordinated exit is what the EOP needs to register a clean dispatch completion -> fence FIRES + // (the coordinator's staggered RETIRE broadcast fires the fence at 8 waves but not 16). Bounded wait + // (plain counter + s_sleep; NO RTC/message bus, NO s_alloc) -> can never hang the wave. + lds_inc QUIESCE_CNT_OFF // this wave checked in at the exit + s_mov_b32 s52, 0 +.Lflow_retbar: + lds_get s53, QUIESCE_CNT_OFF + s_cmp_ge_u32 s53, WAVES // all WAVES waves in -> exit together + s_cbranch_scc1 .Lflow_endpgm + s_add_u32 s52, s52, 1 + s_cmp_ge_u32 s52, RETBAR_MAX // safety bound -> exit anyway (never hang) + s_cbranch_scc1 .Lflow_endpgm + s_sleep SLEEPN + s_branch .Lflow_retbar +.Lflow_endpgm: +.endif + s_endpgm +.else + s_endpgm // DSWS2=0 has no v2 body (this file is always built DSWS2=1) +.endif + .size occ_kernel, .-occ_kernel + +// ---- RGADESC: analysis-only descriptor so `rga -s bin --co` can enumerate + livereg this kernel. +// vgpr 256 ceiling so livereg reports the true s_alloc-grown peak-live. NOT emitted for the PM4 .bin. ---- +.if RGADESC +.amdhsa_kernel occ_kernel + .amdhsa_next_free_vgpr 256 + .amdhsa_next_free_sgpr 72 // body uses up to s69 (s66=n_kseg s67=mask s68=shift s69=chunkHi, FIX 1) + .amdhsa_group_segment_fixed_size 65536 // FIX 1a ring: D=2 needs 33024B (RGA-analysis descriptor only) + .amdhsa_user_sgpr_count 15 // FIX 1(g): v2 contract now s0..s14 only (n_kseg/TOTAL_super/ + // magic_kseg dropped -- derived in-kernel / memory-carried) + .amdhsa_wavefront_size32 1 +.end_amdhsa_kernel +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: occ_kernel + .symbol: occ_kernel.kd + .kernarg_segment_size: 60 + .kernarg_segment_align: 8 + .group_segment_fixed_size: 65536 + .private_segment_fixed_size: 0 + .wavefront_size: 32 + .sgpr_count: 72 + .vgpr_count: 256 + .max_flat_workgroup_size: 256 + .args: [] +.end_amdgpu_metadata +.endif diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_ring.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_ring.s new file mode 100644 index 000000000000..7d6659048038 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_ring.s @@ -0,0 +1,1696 @@ +// occ_kernel_dsws.s (gfx1201, wave32) -- MAD-305 DSWS v2 SUBSTRATE SCAFFOLD (PLAN_DSWS_SUBSTRATE_V2.md, Task A1). +// +// v2 re-founds the DSWS GEMM substrate so matrix work is *claimed* (not owned by compile-time wave id), with +// split-K folded in. Work = a pool of (mblk, tcol, ksi) super-tiles; a pinned claimer (wid 0) broadcasts the +// current super-tile; live waves of each role drain shared LDS atomic counters against resident-in-LDS A/B. +// +// Task A1 is SCAFFOLD-ONLY: lift the proven prologue/arming setup from occ_kernel_coop.s (NEVER modified), +// assert the v2 LDS layout fits the 32 KB group segment, and leave each role as a distinct s_endpgm stub. +// The real claimer / feed / compute loops land in A3..A7. The only gates here: (1) assembles clean, +// (2) RGA 0-spill, (3) harness --dsws2 dry-prints the computed params. +// +// ============================================================================================ +// v2 KERNARG CONTRACT (USER_SGPR=15, s0..s14; hardware-preloaded user SGPRs). The host A8 launch +// MUST set COMPUTE_PGM_RSRC2.USER_SGPR=15 and load COMPUTE_USER_DATA_0..14 to match EXACTLY: +// s0:s1 = occ buffer base (>=0x1000B; host zero-inits; see CLAIM-COUNTER / completion offsets below) +// s2:s3 = A base (fp8 e4m3, row-major, 1 byte/elem) +// s4:s5 = Bshuf base (the shuffled-B layout the coop B-feed consumes; same global_load_tr_b64 idiom) +// s6:s7 = C base (fp32; HOST MUST MEMSET C=0 before dispatch -- compute uses global_atomic_add_f32) +// s8 = KT (total K16-steps for the whole matrix = K/16) +// s9 = K(bytes/A-row) (= K, fp8 1 byte/elem) +// s10 = NT*256 (B-saddr advance per K16-step) +// s11 = TOTAL (coop-style total *tiles* = MTL*NTL; carried for addressing compat, NOT the pool size) +// s12 = magic(ceil(2^32/NTL)) (unsigned-div magic for /NTL ; tcol/mblk decode) +// s13 = NTL (number of N tile-columns) +// s14 = FN*256 (B-saddr stride per N-frag) +// (TGID_X now lands in s15 -- UNUSED; this kernel is pool-claim, not workgroup-id based.) +// NOTE: G and SEGK are COMPILE-TIME defsyms (baked into instruction immediates); they are NOT kernargs. +// FIX 1 (round-table Opus+Codex pass): v1 of this contract passed n_kseg/TOTAL_super/magic_kseg as +// s15/s16/s17, but the PM4 host only preloads COMPUTE_USER_DATA_0..15 (USER_SGPR<=16; every proven +// launch path in this tree uses 15) -- s16/s17 could NEVER actually arrive in hardware SGPRs, AND +// s16 was independently being reused per-chunk on the host as the compositor-safe chunk terminal +// (a second, unrelated collision on the same slot). This file now drops s15/s16/s17 entirely: +// n_kseg is DERIVED in-kernel from KT (s8) and the compile-time KSEG_STEPS=SEGK/16: +// n_kseg = KT >> NKSEG_SHIFT, where NKSEG_SHIFT=log2(KSEG_STEPS) is a compile-time `.set` +// (small .if ladder over KSEG_STEPS in {1,2,4,8,16}; SEGK is always a power-of-two +// multiple of 16, so KSEG_STEPS is always a power of two in that set). +// shift/mask (the sti -> (t,ksi) split) are derived ONCE in the prologue from n_kseg: +// shift = s_ff1_i32_b32(n_kseg) (bit index of n_kseg's single set bit; n_kseg=1 -> 0) +// mask = n_kseg - 1 +// DECODE_STI then does ksi = sti & mask ; t = sti >> shift -- this handles n_kseg=1 for +// free (shift=0, mask=0 -> ksi=0, t=sti), so the old magic-div n_kseg==1 special-case is +// GONE (it's no longer needed, not just hidden). +// the chunk terminal (old TOTAL_super/"chunkHi") is now MEMORY-CARRIED instead of a kernarg: the +// host writes the current chunk's terminal sti bound to occ[24] (occW[6]) once per chunk; +// the claimer reads occ[24] ONCE per dispatch (stable for the whole chunk) instead of +// receiving it as a broadcast kernarg. On sti >= occ[24] the claimer publishes a SENTINEL +// (0xFFFFFFFF) into STI_OFF instead of the raw over-claimed sti; followers (b-feed/a-feed/ +// compute) retire when STI_OFF == 0xFFFFFFFF instead of comparing against the (now +// nonexistent) TOTAL_super kernarg. +// See "CLAIM-COUNTER & completion occ offsets" by .Lclaimer for the full occ-buffer layout +// (occ[24]/occW[6] = chunk terminal bound, added by FIX 1). +// +// SCALAR REGS (derived in the prologue, before any clobber; none collide with DECODE_STI's own clobber +// list s18/s36, lds_*'s s49, the claimer's s16/s17/s35/s44, or any role body's transients, all <= s65): +// s66 = n_kseg (derived; dead after shift/mask below are computed -- kept only for that derivation) +// s67 = mask (n_kseg - 1) -- LIVE for the whole kernel; read by every DECODE_STI call, every role. +// s68 = shift (log2 n_kseg) -- LIVE for the whole kernel; read by every DECODE_STI call, every role. +// s69 = chunkHi (claimer-only; loaded once per dispatch from occ[24] right before .Lclaim_loop). +// +// Everything new is gated behind the fresh `DSWS2` build symbol (analogous to coop's `DSWS`). + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1201" + +// ---- tile defsyms (lifted from occ_kernel_coop.s) ---- +.ifndef FM + .set FM, 2 // per-compute-wave M-frags (M-band = FM*16 rows) +.endif +.ifndef FN + .set FN, 4 // shared N-frags (the reuse operand) +.endif +.ifndef RGADESC + .set RGADESC, 0 // 1 = emit analysis-only AMDHSA descriptor for RGA livereg +.endif +.ifndef DIAG + .set DIAG, 0 // 1 = phase-marker instrumentation (unused in the A1 scaffold) +.endif +.ifndef SAFEPROBE + .set SAFEPROBE, 0 // 1 = clamp per-lane vector address regs into a provable in-buffer bound +.endif +.ifndef TFPROBE + .set TFPROBE, 0 // 1 = emit s_sendmsg_rtn GET_REALTIME wall-span capture (each wave stamps +.endif // occ[2]=min entry tick, occ[3]=max exit tick -> host TF readout). + // Default 0 => ZERO bytes emitted, .text byte-identical to the + // production bin (uses only already-allocated regs s30/s31/s49/v5). +.ifndef PHASEPROBE + .set PHASEPROBE, 0 // 1 = in-kernel PHASE TIMER: each compute wave stamps GET_REALTIME at every +.endif // phase boundary and atomic-adds the delta into per-phase occ accumulators + // (occ[64..69], bytes 256..276, ABOVE the per-chunk memset -> accumulate over + // the whole run). Host prints ticks + % per phase. Default 0 => byte-identical. +.ifndef NOCFLUSH + .set NOCFLUSH, 0 // PERF PROBE ONLY: 1 = skip the global_atomic_add_f32 C-flush loop (keep ALL +.endif // other bookkeeping/handshake). Isolates the device-atomic C-reduction cost + // from the coordination handshake. Result is WRONG (C never written -> oracle + // fails) -- span/TF only. Default 0 => byte-identical. +.ifndef KMAJOR + .set KMAJOR, 0 // PERF PROBE: 1 = K-MAJOR super-tile traversal. Default decode packs ksi in +.endif // the LOW bits (all n_kseg segments of a C cell claimed consecutively -> + // up to n_kseg WGs hammer one C cell at once = max atomic contention). + // KMAJOR decodes ksi = sti / TOTAL (high), t = sti % TOTAL (low) via a + // magic-div (magic_TOTAL from occ[62], loaded to s76 in prologue) -> the + // 32 segments of a cell are spread TOTAL apart in claim order (near-zero + // concurrent contention) + adjacent tiles (shared operand bands) claimed + // close in time (L2 reuse). Correctness-preserving (C add is commutative). +.ifndef CSTORE + .set CSTORE, 0 // PERF PROBE ONLY: 1 = replace the flush's global_atomic_add_f32 with an +.endif // equal-count NON-atomic global_store_b32 (same #mem-ops, same addresses, NO + // RMW/contention). Isolates atomic-contention from raw write-bandwidth: if + // TF jumps vs atomics -> contention; if flat -> bandwidth-bound. Result WRONG + // (last-writer, no accumulation) -- span/TF only. Default 0 => byte-identical. +.ifndef TRACE + .set TRACE, 0 // 1 = per-super-tile CLAIMER trace: append one row/super-tile {tick, segcnt, +.endif // epoch, nComp/nAfeed/nBfeed live role slots, ring occA/occB peak, + // convCount, vresv, sti, quiesce} to a host-provided buffer (VA in + // occ[52:53], cap in occ[54]). Time-series of the adaptive wave-role + // economy. Requires DSWS2_CONV=1 (rows written in the quiesce path). + // Default 0 => ZERO bytes; uses free high SGPRs s70..s74. +.ifndef DYNVGPR + .set DYNVGPR, 1 // 1 = compute waves s_alloc_vgpr-grow per rowblk; feeds/claimer stay lean 32 +.endif +.ifndef SLEEPN + .set SLEEPN, 2 // s_sleep arg in the busy-waits (yield issue cycles to partner waves) +.endif + +// ============================================================================================ +// DSWS v2 LDS layout (bytes from group-segment base; words u32 unless noted). Mirrors the placement +// of the coop file's LDS `.set` block. Defined unconditionally (uses only G/SEGK/FM/FN, always set). +// ============================================================================================ +.ifndef DSWS2 + .set DSWS2, 0 +.endif +.ifndef G + .set G, 6 // cooperative M-extent (rowblks per super-tile) = NCOMP_MAX +.endif +.ifndef SEGK + .set SEGK, 64 // split-K segment size in K-elements (multiple of 16) +.endif +// ---- v2 control/claim words ---- +.set STI_OFF, 0 // broadcast super-tile id +.set EPOCH_OFF, 4 +.set ROWBLK_NEXT_OFF, 8 // per-super-tile rowblk claim counter +.set ROWBLK_DONE_OFF, 12 // per-super-tile completion counter +.set BFRAG_NEXT_OFF, 16 // B-frag claim counter +.set AROW_NEXT_OFF, 20 // A-rowblk claim counter +.set NCOMP_SLOT, 24 +.set NAFEED_SLOT, 28 +.set NBFEED_SLOT, 32 +.set GATE_OFF, 36 // u32[4] -> 36,40,44,48 (conversion gates) +.set VRESV_OFF, 52 // vgpr_reserved +.set SEGCNT_OFF, 56 // controller clock +// ---- A3..A7 additions (still inside the 0..256 control region; A1 offsets 0..56 unchanged) ---- +.set BFRAG_DONE_OFF, 60 // B-frag STORE-completion counter (compute gates on this, NOT the claim ctr) +.set AROW_DONE_OFF, 64 // A-rowblk STORE-completion counter (compute gates on this) +.set INITFLAG_OFF, 68 // barrier-free LDS-init publish flag (claimer writes 0xACED LAST) +// ---- Phase-B (DSWS2_CONV) control state: role-mix snapshot slots + quiesce counter ---- +// Based at INITFLAG_OFF+4 (NOT the brief's SEGCNT_OFF+4): the brief predates the A3..A7 control +// words (BFRAG_DONE/AROW_DONE/INITFLAG at 60/64/68), so SEGCNT_OFF+4=60 would collide with them. +// Basing after the LAST control word keeps the new state inside the 0..255 control gap BELOW the +// fixed resident region (BRES_OFF=256), so NO resident-region repoint is needed -- the resident +// BRES_OFF/ARES_OFF immediates (emitted unconditionally in the kernel body) stay untouched, which +// is what keeps the DSWS2_CONV=0 binary byte-identical to the Phase-A green bin. All `.set`s here +// are inert (emit no bytes); the only new code (claimer init) is gated under `.if DSWS2_CONV`. +.ifndef DSWS2_CONV + .set DSWS2_CONV, 0 // 0 = pre-conversion static substrate (Phase A green); 1 = Phase B +.endif +.ifndef DSWS2_TICKET_SELFTEST + .set DSWS2_TICKET_SELFTEST, 0 // DIAG-only try_gate single-winner smoke (Task 4 Step 3); default 0 = no bytes +.endif +.ifndef CONV_COOLDOWN + .set CONV_COOLDOWN, 0 // Task 4: per-wave post-conversion cooldown epochs. 0 = spec-faithful (no + // cooldown, byte-identical to pre-Task-4); >0 damps thrash (skip N epochs + // of watermark decision after a wave converts role). +.endif +// Task 5: deterministic bring-up hook. DSWS2_FORCE=1 makes exactly wave DSWS2_FORCE_WID convert +// direction DSWS2_FORCE_DIR at epoch DSWS2_FORCE_EPOCH, watermarks bypassed -- a reproducible, +// single-wave/single-epoch GPU proof of role conversion. Default DSWS2_FORCE=0 emits ZERO bytes +// (byte-identical to pre-Task-5). +.ifndef DSWS2_FORCE + .set DSWS2_FORCE, 0 +.endif +.ifndef DSWS2_FORCE_WID + .set DSWS2_FORCE_WID, 0 +.endif +.ifndef DSWS2_FORCE_DIR + .set DSWS2_FORCE_DIR, 0 // 0/1 = compute->A/B ; 2/3 = A/B->compute +.endif +.ifndef DSWS2_FORCE_EPOCH + .set DSWS2_FORCE_EPOCH, 1 +.endif +// Rolling dyn-VGPR sum-envelope (2026-07-02 spec). ENVELOPE routes the per-rowblk compute burst grow +// through the shared vgpr_reserved counter so at most PEAK_CONC waves hold peak at once (the +// multi-grower collision, ISA 3.3.3.2, becomes unreachable). All default to the byte-identical value: +// ENVELOPE=0/STAGGER=0 emit ZERO new bytes and PEAK_CONC/STAGGER_PERIOD are inert unless their gate is on. +.ifndef DSWS2_ENVELOPE + .set DSWS2_ENVELOPE, 0 // 1 = route the per-rowblk compute burst grow through the vgpr_reserved +.endif // sum-envelope. 0 = HEAD (bare .Lcompute_grow) -> .text byte-identical. +.ifndef PEAK_CONC + .set PEAK_CONC, 2 // concurrent compute peaks the budget admits (R3 sweep). Used iff ENVELOPE=1. +.endif +.ifndef DSWS2_STAGGER + .set DSWS2_STAGGER, 0 // 1 = lock-free phase-token stagger (Task 9). 0 -> emergent envelope stagger. +.endif +.ifndef STAGGER_PERIOD + .set STAGGER_PERIOD, NCOMP // phase slots in the stagger ring (R3 sweep). Used iff STAGGER=1. +.endif +.set SNAP_BASE, (INITFLAG_OFF + 4) // u32[6]: [parity*3 + {0:nC,1:nA,2:nB}] role-mix snapshots +.set QUIESCE_CNT_OFF,(SNAP_BASE + 6*4) // u32 role-agnostic bail counter (LDS; DSWS2_GQUIESCE=0) +// SENSOR FIX: the claimer publishes its MID-DRAIN ring-occupancy PEAK here each super-tile; the conversion +// decisions read THESE instead of sampling occ_sample at their own quiesce (where occ_X reads ~0 post-drain +// -> always "starved" -> the 4/2/2->1/6/1 compute->feed runaway). Mid-drain peak = the true demand signal. +.set OCCA_PUB_OFF, (QUIESCE_CNT_OFF + 4) // claimer-published occ_A peak +.set OCCB_PUB_OFF, (OCCA_PUB_OFF + 4) // claimer-published occ_B peak +.set DSWS2_STATE_END,(OCCB_PUB_OFF + 4) +// DSWS2_GQUIESCE (2026-07-02 SUSPECT #2 candidate fix): route the QUIESCE handshake through a DEVICE-SCOPED +// GLOBAL atomic in the uncached occ buffer (byte QUIESCE_GOFF), mirroring the GREEN occ[20] claim/occ[0] +// live handshake, instead of the barrier-free LDS counter (whose cross-wave visibility is unguaranteed and +// is the leading SUSPECT #2 hang mechanism). occ buffer = AllocGpu 0x1000 (1024 u32, uncached); host uses +// occ[0..6] + DIAG scratch (<= byte 116); byte 200 (occ[50]) is provably free. Default 0 => LDS path, +// .text byte-identical. Requires DSWS2_CONV (QUIESCE only exists there). +.ifndef DSWS2_GQUIESCE + .set DSWS2_GQUIESCE, 0 +.endif +.set QUIESCE_GOFF, 200 // occ[] byte offset for the global QUIESCE counter (occ[50]) +// ---- TRACE (per-super-tile time-series) occ handshake words + row layout ---- +.set TRACE_PTR_OFF, 208 // occ[52:53] = trace buffer VA (host writes lo/hi per chunk) +.set TRACE_CAP_OFF, 216 // occ[54] = MAXROWS (host-provided row capacity) +.set TRACE_IDX_OFF, 220 // occ[55] = GLOBAL row-claim counter (all WGs' claimers share it) +.set TRACE_WGID_OFF, 224 // occ[56] = GLOBAL wg-id dispenser (claim-order 0..pool-1) +.set FATLIVE_OFF, 228 // occ[57] = live count of GROWN (fat NFV-VGPR) compute waves +.set FATMAX_OFF, 232 // occ[58] = PEAK concurrent fat waves -> x NFV = VGPR in flight (== B probe) +.set ALLLIVE_OFF, 240 // occ[60] = live count of ALL resident waves (++entry/--exit) + // occ[1] (byte 4) = PEAK concurrent resident waves (vs 2048 HW ceiling) +.set TRACE_ROW_BYTES, 64 // 16 u32/row +// DSWS2_BAILMARK (SUSPECT #2 localization, 2026-07-03): each follower publishes its OWN epoch (s35) to a +// PER-WAVE occ slot (BAIL_BASE + wid*4) at its _quiesce bail. One-shot per super-tile per wave -> minimal +// timing perturbation (NOT the claimer's per-spin DIAG poll stores, which are the heisenbug source and stay +// DIAG-only). After a watchdog abort the host reads occ[BAIL_BASE/4 + wid]: every follower's slot == the +// hung epoch => all reached their bail (=> a QUIESCE visibility/lost-update, gq relevant); ONE slot stale +// at the prior epoch => that exact wave is the STRAGGLER (stuck in _alloc/_init/_follow; gq irrelevant). +// Per-WAVE (not per-role): 4 compute share one role, so a role mark's last-writer-wins would hide a single +// straggler. Default 0 => no bytes, .text byte-identical. Requires DSWS2_CONV. +.ifndef DSWS2_BAILMARK + .set DSWS2_BAILMARK, 0 +.endif +.set BAIL_BASE, 160 // occ[] byte offset base for per-wave bail marks: occ[40..47] +.set CONVCNT_OFF, 192 // occ[48]: DIAG conversion-commit counter (proves waves switch role) + // (host prints occ[40..47] as BAIL[w0..w7]; clear of the + // occ[32..36]/occ[39] DSWS sensor+roles slots and occ[50] gq) +.set KSEG_STEPS, (SEGK/16) // K16-steps per split-K segment = SEGK K-elements / 16 +// FIX 1(b): NKSEG_SHIFT = log2(KSEG_STEPS), so the prologue can derive n_kseg = KT >> NKSEG_SHIFT instead +// of receiving it as a (now-dropped) kernarg. SEGK is always a power-of-two multiple of 16 in every +// config this file is built with, so KSEG_STEPS is always a power of two in {1,2,4,8,16}; a static +// ladder over that small set is simpler/safer than a general-purpose compile-time log2. +.if KSEG_STEPS == 1 + .set NKSEG_SHIFT, 0 +.elseif KSEG_STEPS == 2 + .set NKSEG_SHIFT, 1 +.elseif KSEG_STEPS == 4 + .set NKSEG_SHIFT, 2 +.elseif KSEG_STEPS == 8 + .set NKSEG_SHIFT, 3 +.elseif KSEG_STEPS == 16 + .set NKSEG_SHIFT, 4 +.else + .error "KSEG_STEPS (SEGK/16) must be a power of two in {1,2,4,8,16}" +.endif +// resident regions aligned to 256B +.set BRES_OFF, 256 // resident B for current super-tile +.set BRES_BYTES, (FN*16*SEGK) // = 4*16*64 = 4096 at the default config +.set ARES_OFF, (BRES_OFF + BRES_BYTES) // resident A for current super-tile +.set ARES_BYTES, (G*16*FM*SEGK) // = 6*16*2*64 = 12288 at the default config +.set LDS_TOTAL_DSWS2, (ARES_OFF + ARES_BYTES) +// (old single-slot cap check retained; 16640 < 32768 -> always passes. The RING layout below is what +// the ring role loops actually use; its own cap check follows.) +.if LDS_TOTAL_DSWS2 > 32768 + .error "DSWS2 LDS layout exceeds 32768B group segment" +.endif +// ============================================================================================ +// FIX 1a -- RING-OF-SLOTS LDS layout (D=2 double-buffered pipeline). NEW symbols; the single-slot +// control words above (STI_OFF..INITFLAG_OFF, SNAP/QUIESCE/TRACE state) stay DEFINED (they emit no +// bytes and their only *code* uses are under .if DSWS2_CONV/DIAG/TRACE, all 0 here -> not assembled). +// The ring role loops reference ONLY the symbols below. GEN is a GLOBAL-monotonic publish counter: +// the dispatcher bumps it once per publish and writes it to the published slot's SL_GEN LAST (the +// release fence); followers gate "new occupant" on SL_GEN[idx] > last-seen-gen. Slot counters reset +// only at FREE->FILLED (SL_RBDONE>=G), only by the dispatcher -> the old QUIESCE handshake is +// unnecessary (compute can't bump SL_RBDONE until SL_BFDONE==FN && SL_ARDONE==G, so no straggler is +// mid-claim at reset). See RING_SLOTS_DESIGN.md "Protocol refinements". +.set RING_D, 2 +// ---- shared frontier (single copy, at the front) ---- +.set FILL_IDX_OFF, 0 // slot the feeds are currently staging (dispatcher publishes into it) +.set DRAIN_IDX_OFF, 4 // slot compute is currently draining +.set RINGINIT_OFF, 8 // barrier-free LDS-init publish flag (dispatcher writes 0xACED LAST) +// ---- per-slot control block: SLOTC_BASE + slot*SLOTC_STRIDE + field ---- +.set SLOTC_BASE, 32 +.set SLOTC_STRIDE, 32 +.set SL_STI, 0 // super-tile id resident in this slot (0xFFFFFFFF = sentinel/retire) +.set SL_GEN, 4 // global-monotonic publish gen (release fence; consumers gate on > last-seen) +.set SL_RBNEXT, 8 // rowblk claim counter (compute) +.set SL_RBDONE, 12 // rowblks computed+flushed; dispatcher recycles this slot when >= G +.set SL_BFNEXT, 16 // B-frag claim counter (B-feeds) +.set SL_BFDONE, 20 // B-frags stored; compute gates on == FN +.set SL_ARNEXT, 24 // A-rowblk claim counter (A-feeds) +.set SL_ARDONE, 28 // A-rowblks staged; compute gates on == G +// ---- per-slot operand buffers: OP_BASE + slot*OPSTRIDE ; BRES at +BRES_ROFF, ARES at +ARES_ROFF ---- +.set OP_BASE, 256 // 256B-aligned; below it: frontier + RING_D control blocks +.set OPSTRIDE, (BRES_BYTES + ARES_BYTES) // 4096 + 12288 = 16384 per slot +.set BRES_ROFF, 0 // resident B within a slot +.set ARES_ROFF, BRES_BYTES // resident A within a slot (after B) +.set LDS_TOTAL_RING,(OP_BASE + RING_D*OPSTRIDE) // 256 + 2*16384 = 33024 +.if LDS_TOTAL_RING > 65536 + .error "RING LDS layout exceeds 65536B group segment (hardware WGP limit)" +.endif +.if (SLOTC_BASE + RING_D*SLOTC_STRIDE) > OP_BASE + .error "RING per-slot control blocks overlap the operand region (raise OP_BASE)" +.endif +// Phase-B state must fit in the control gap below the resident region (inert compile check, no bytes). +.if DSWS2_STATE_END > BRES_OFF + .error "DSWS2 Phase-B state (SNAP_BASE/QUIESCE_CNT) overlaps resident B region (BRES_OFF)" +.endif + +.if DSWS2 + // ---- role counts (lifted from coop's `.ifndef NCOMP` etc., gated under DSWS2) ---- + .ifndef NCOMP + .set NCOMP, 4 // compute waves (fat, dyn-grow). Compute floor >= 1. + .endif + .ifndef NAFEED + .set NAFEED, 2 // A-feed waves (lean). Feed floor >= 1. + .endif + .ifndef NBFEED + .set NBFEED, 2 // B-feed waves (lean). Feed floor >= 1. + .endif + .set WAVES, (NCOMP + NAFEED + NBFEED) // total waves launched per WG (harness dims must match) +.endif + +// ============================================================================================ +// VGPR layout (lifted from occ_kernel_coop.s) -- compute frags live ABOVE the lean-32 block and are +// only touched AFTER s_alloc_vgpr NFV. Feeds/claimer stay in the lean block (v0..v31). +// ============================================================================================ +.set ACC, 32 // accumulators: FM*FN frags x 8 f32 (v32..) +.set FA, (ACC + 8*FM*FN) // compute A frags (from resident LDS): FM x 2 +.set FB, (FA + 2*FM) // compute B frags (from resident LDS): FN x 2 +.set NFV, ((FB + 2*FN + 15) & ~15) // grown footprint, rounded to a 16-VGPR dyn block (=112 @ 2x4) +.set VLEAN, 32 // lean footprint (feeds, claimer, compute pre/post rowblk) +.set BSTG, 16 // staging regs (lean block, < 32): B-feed FN-frag / A-feed FM-frag + +// ---- dyn-VGPR PRE-GROW temp-reg ceiling (coop death-cert: a >v15 src pre-grow is poison under dyn). +// Gate every PRE-grow-reachable LDS/atomic temp to v11/v14 (INTERIOR to the launched 16-VGPR block). ---- +.if DYNVGPR + .set RG_A, 11 // lds_get / fetch_add address + .set RG_D, 14 // lds_get / fetch_add data+return + .set RP_A, 11 // lds_put address + .set RP_D, 14 // lds_put data +.else + .set RG_A, 27 + .set RG_D, 28 + .set RP_A, 28 + .set RP_D, 29 +.endif + +// ============================================================================================ +// LDS helper macros (s49 = exec save; v2 = lane = tid&31, set in prologue). +// ============================================================================================ +.macro lds_get sdst, off // wave-uniform read LDS[off] -> scalar sdst + v_mov_b32 v[RG_A], \off + ds_load_b32 v[RG_D], v[RG_A] + s_wait_dscnt 0x0 + v_readfirstlane_b32 \sdst, v[RG_D] +.endm +.macro lds_get_r sdst, saddr // wave-uniform read LDS[saddr] (RUNTIME addr in a sreg) -> sdst + v_mov_b32 v[RG_A], \saddr + ds_load_b32 v[RG_D], v[RG_A] + s_wait_dscnt 0x0 + v_readfirstlane_b32 \sdst, v[RG_D] +.endm +.macro lds_put off, ssrc // lane-0-of-wave writes scalar ssrc -> LDS[off] + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lput_skip\@ + v_mov_b32 v[RP_A], \off + v_mov_b32 v[RP_D], \ssrc + ds_store_b32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lput_skip\@: + s_mov_b32 exec_lo, s49 +.endm +.macro lds_fetch_add sdst, off, val // sdst <- old LDS[off]; LDS[off]+=val (lane-0 atomic, broadcast) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lfa_skip\@ + v_mov_b32 v[RP_A], \off + v_mov_b32 v[RP_D], \val + ds_add_rtn_u32 v[RP_D], v[RP_A], v[RP_D] // v[RP_D] <- old; LDS[off] += val + s_wait_dscnt 0x0 +.Lfa_skip\@: + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 \sdst, v[RP_D] // broadcast lane-0's old value +.endm +.macro lds_inc off // lane-0-of-wave LDS[off] += 1 (no return) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Linc_skip\@ + v_mov_b32 v[RP_A], \off + v_mov_b32 v[RP_D], 1 + ds_add_u32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Linc_skip\@: + s_mov_b32 exec_lo, s49 +.endm +// ---- FIX 1a ring: RUNTIME-address variants (slot-indexed counters live at SLOTC_BASE+slot*32+field, +// a runtime scalar). Mirror lds_fetch_add / lds_inc but take the address in a sreg. ---- +.macro lds_fetch_add_r sdst, saddr, val // sdst <- old LDS[saddr]; LDS[saddr]+=val (lane-0 atomic, bcast) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lfar_skip\@ + v_mov_b32 v[RP_A], \saddr + v_mov_b32 v[RP_D], \val + ds_add_rtn_u32 v[RP_D], v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lfar_skip\@: + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 \sdst, v[RP_D] +.endm +.macro lds_inc_r saddr // lane-0-of-wave LDS[saddr] += 1 (RUNTIME addr, no return) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lincr_skip\@ + v_mov_b32 v[RP_A], \saddr + v_mov_b32 v[RP_D], 1 + ds_add_u32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lincr_skip\@: + s_mov_b32 exec_lo, s49 +.endm +// lds_put_r (RUNTIME-addr write) is also defined inside the .if DSWS2_CONV||DSWS2_ENVELOPE block below; +// the ring needs it at CONV=0/ENV=0, so define an identical copy here, guarded to avoid a double-def +// when either gate is on (the ring is always built CONV=0 ENV=0). +.if !(DSWS2_CONV || DSWS2_ENVELOPE) +.macro lds_put_r saddr, ssrc // lane-0 write ssrc -> LDS[saddr] (RUNTIME addr) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lputr_skip\@ + v_mov_b32 v[RP_A], \saddr + v_mov_b32 v[RP_D], \ssrc + ds_store_b32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lputr_skip\@: + s_mov_b32 exec_lo, s49 +.endm +.endif + +// ============================================================================================ +// Super-tile decode + resident A/B staging macros (A3..A6). Decode (Naming/symbols): +// ksi = sti & mask ; t = sti >> shift ; mblk = t / NTL ; tcol = t % NTL. +// FIX 1(d): n_kseg is ALWAYS a power of two (it's KT >> NKSEG_SHIFT, both compile-time-shift-derived), +// so the sti->(t,ksi) split is an exact shift/mask -- no magic-div, no n_kseg==1 special-case (shift=0, +// mask=0 falls out of the general path for free: ksi=0, t=sti). /NTL still goes via magic (s12), +// unsigned-division mul_hi (coop GENDIV idiom), since NTL is not generally a power of two. +// ============================================================================================ +.macro DECODE_STI // in: s17=sti, s67=mask, s68=shift ; out: s19=mblk s30=tcol s31=ksi ; clob: s18,s36 +.if KMAJOR + s_mul_hi_u32 s31, s17, s76 // ksi = sti / TOTAL (magic-div, magic_TOTAL in s76; K-major high bits) + s_mul_i32 s36, s31, s11 // ksi * TOTAL + s_sub_u32 s18, s17, s36 // t = sti - ksi*TOTAL (low) +.else + s_and_b32 s31, s17, s67 // ksi = sti & mask (mask-bounded -> ksi in [0,n_kseg-1]) + s_lshr_b32 s18, s17, s68 // t = sti >> shift +.endif +.if SAFEPROBE + // brick-PROOF ti clamp (the "future ti clamp" line 752 promised; COOP_STATUS.md:145 racy-garbage-ti->OOB). + // A racy/torn sti read (during the claimer's per-super-tile republish) can decode a garbage t -> garbage + // mblk/tcol -> the A/B/C SCALAR base goes out of buffer -> gfxhub page fault -> MODE1 brick. SAFEPROBE + // already pins the per-lane vaddr (v8/v9/v10); this pins the tile index too, so EVERY global address is + // provably in-buffer. s11=TOTAL is userdata, never clobbered. s36 is DECODE_STI scratch (rewritten below). + s_sub_u32 s36, s11, 1 // TOTAL-1 + s_min_u32 s18, s18, s36 // t clamped to [0,TOTAL-1] -> mblk in-bounds) +.endif + s_mul_hi_u32 s19, s18, s12 // mblk = t / NTL + s_mul_i32 s36, s19, s13 // mblk * NTL + s_sub_u32 s30, s18, s36 // tcol = t - mblk*NTL +.endm + +// RESIDENT B FRAG LAYOUT: B frag (kstep ks, frag f) at BRES_OFF + (ks*FN + f)*256 +// (each frag = the SAME 256B block coop stores per B-ring slot; lane*8 vaddr base = v9). +// Built here as: dst vbase = v9 + BRES_OFF + f*256 , ds_store offset:(ks*FN*256). +// B global addr (lift coop B-feed): Bshuf + tcol*(FN*256=s14) + (seg k0)* [ksi*KSEG_STEPS*(NT*256=s10)] +// + f*256 (frag, folded into saddr) + ks*(NT*256=s10) (k-step, folded into saddr). +.macro BSTAGE // in: s30=tcol s31=ksi ; clob: s20,s21,s23,s25,s26,s27,v13,v[BSTG..] + s_mul_i32 s20, s30, s14 // tcol * FN*256 + s_mul_i32 s21, s31, KSEG_STEPS // ksi * KSEG_STEPS + s_mul_i32 s21, s21, s10 // * NT*256 -> segment k-start byte offset + s_add_u32 s20, s20, s21 + s_add_u32 s20, s4, s20 + s_addc_u32 s21, s5, 0 // s[20:21] = B base (tcol,ksi, seg k-step 0) +.Lbcl\@: + lds_fetch_add s23, BFRAG_NEXT_OFF, 1 // claim frag f + s_cmp_ge_u32 s23, FN + s_cbranch_scc1 .Lbsd\@ // f>=FN -> all frags claimed + s_lshl_b32 s25, s23, 8 // f*256 + s_add_u32 s26, s20, s25 + s_addc_u32 s27, s21, 0 // s[26:27] = frag f base (seg k0) + v_add_nc_u32 v13, v9, BRES_OFF + v_add_nc_u32 v13, v13, s25 // resident B dst vbase for frag f + .set ks, 0 + .rept KSEG_STEPS + global_load_tr_b64 v[BSTG+ks*2:BSTG+ks*2+1], v9, s[26:27] + s_add_u32 s26, s26, s10 // next k-step (last iter over-advances; unused) + s_addc_u32 s27, s27, 0 + .set ks, ks+1 + .endr + s_wait_loadcnt 0x0 + .set ks, 0 + .rept KSEG_STEPS + ds_store_b64 v13, v[BSTG+ks*2:BSTG+ks*2+1] offset:(ks*FN*256) + .set ks, ks+1 + .endr + s_wait_dscnt 0x0 + lds_inc BFRAG_DONE_OFF // frag f STORED -> publish completion (compute gates on this) + s_branch .Lbcl\@ +.Lbsd\@: +.endm + +// RESIDENT A FRAG LAYOUT: A frag (kstep ks, rowblk r, mi) at ARES_OFF + ((ks*G + r)*FM + mi)*256 +// Built as: dst vbase = v9 + ARES_OFF + r*(FM*256) , ds_store offset:((ks*G*FM + mi)*256). +// A global addr (lift coop compute/A-feed): A + rowblk_abs*(16*FM)*K + mi*16*K + koff, rowblk_abs=mblk*G+r, +// koff = ksi*SEGK (segment K byte offset, fp8 1B/elem), k-step within segment via global offset:ks*16. +.macro ASTAGE // in: s19=mblk s31=ksi ; clob: s22,s23,s25,s32,s36,s40,s41,s44,s45,v13,v[BSTG..] + s_lshl_b32 s32, s9, 4 // rowstride16 = 16*K +.Lacl\@: + lds_fetch_add s23, AROW_NEXT_OFF, 1 // claim rowblk r + s_cmp_ge_u32 s23, G + s_cbranch_scc1 .Lasd\@ + s_mul_i32 s36, s19, G + s_add_u32 s36, s36, s23 // rowblk_abs = mblk*G + r + s_mul_i32 s22, s36, (16*FM) + s_mul_i32 s22, s22, s9 // rowblk_abs*(16*FM)*K + s_mul_i32 s25, s31, SEGK // ksi*SEGK (segment K byte offset) + s_add_u32 s22, s22, s25 + s_add_u32 s40, s2, s22 + s_addc_u32 s41, s3, 0 // s[40:41] = A base (rowblk_abs, mi0, seg k0) + s_mul_i32 s25, s23, (FM*256) // r*FM*256 + v_add_nc_u32 v13, v9, ARES_OFF + v_add_nc_u32 v13, v13, s25 // resident A dst vbase for rowblk r + .set mi, 0 + .rept FM + .if mi == 0 + s_mov_b32 s44, s40 + s_mov_b32 s45, s41 + .else + s_add_u32 s44, s44, s32 // += 16*K (next M-frag) + s_addc_u32 s45, s45, 0 + .endif + .set ks, 0 + .rept KSEG_STEPS + global_load_b64 v[BSTG:BSTG+1], v8, s[44:45] offset:(ks*16) + s_wait_loadcnt 0x0 + ds_store_b64 v13, v[BSTG:BSTG+1] offset:((ks*G*FM + mi)*256) + s_wait_dscnt 0x0 + .set ks, ks+1 + .endr + .set mi, mi+1 + .endr + lds_inc AROW_DONE_OFF // rowblk r fully STAGED -> publish completion + s_branch .Lacl\@ +.Lasd\@: +.endm + +// ============================================================================================ +// FIX 1a -- RING staging macros: slot-indexed BSTAGE_R / ASTAGE_R. Identical math to BSTAGE/ASTAGE +// but claim/done counters live in the per-slot control block (\scb = SLOTC_BASE + slot*32, runtime) +// and operands land in the per-slot buffer (\sob = OP_BASE + slot*OPSTRIDE, runtime; B at +// +BRES_ROFF=0, A at +ARES_ROFF). \scb and \sob are READ-only (never clobbered). Internal address +// scratch: s46/s47 (free in the feed context). ds offset immediates are vbase-relative -> unchanged. +// ============================================================================================ +.macro BSTAGE_R scb, sob // in: s30=tcol s31=ksi ; clob: s20,s21,s23,s25,s26,s27,s46,s47,v13,v[BSTG..] + s_mul_i32 s20, s30, s14 // tcol * FN*256 + s_mul_i32 s21, s31, KSEG_STEPS // ksi * KSEG_STEPS + s_mul_i32 s21, s21, s10 // * NT*256 -> segment k-start byte offset + s_add_u32 s20, s20, s21 + s_add_u32 s20, s4, s20 + s_addc_u32 s21, s5, 0 // s[20:21] = B base (tcol,ksi, seg k-step 0) + s_add_u32 s46, \scb, SL_BFNEXT // &SL_BFNEXT[slot] +.Lbclr\@: + lds_fetch_add_r s23, s46, 1 // claim frag f + s_cmp_ge_u32 s23, FN + s_cbranch_scc1 .Lbsdr\@ // f>=FN -> all frags claimed + s_lshl_b32 s25, s23, 8 // f*256 + s_add_u32 s26, s20, s25 + s_addc_u32 s27, s21, 0 // s[26:27] = frag f base (seg k0) + v_add_nc_u32 v13, v9, \sob // + slot operand base + v_add_nc_u32 v13, v13, s25 // + f*256 (BRES_ROFF = 0) + .set ks, 0 + .rept KSEG_STEPS + global_load_tr_b64 v[BSTG+ks*2:BSTG+ks*2+1], v9, s[26:27] + s_add_u32 s26, s26, s10 + s_addc_u32 s27, s27, 0 + .set ks, ks+1 + .endr + s_wait_loadcnt 0x0 + .set ks, 0 + .rept KSEG_STEPS + ds_store_b64 v13, v[BSTG+ks*2:BSTG+ks*2+1] offset:(ks*FN*256) + .set ks, ks+1 + .endr + s_wait_dscnt 0x0 + s_add_u32 s47, \scb, SL_BFDONE + lds_inc_r s47 // frag f STORED -> compute gates on SL_BFDONE==FN + s_branch .Lbclr\@ +.Lbsdr\@: +.endm + +.macro ASTAGE_R scb, sob // in: s19=mblk s31=ksi ; clob: s22,s23,s25,s32,s36,s40,s41,s44,s45,s46,s47,v13,v[BSTG..] + s_lshl_b32 s32, s9, 4 // rowstride16 = 16*K + s_add_u32 s46, \scb, SL_ARNEXT // &SL_ARNEXT[slot] +.Laclr\@: + lds_fetch_add_r s23, s46, 1 // claim rowblk r + s_cmp_ge_u32 s23, G + s_cbranch_scc1 .Lasdr\@ + s_mul_i32 s36, s19, G + s_add_u32 s36, s36, s23 // rowblk_abs = mblk*G + r + s_mul_i32 s22, s36, (16*FM) + s_mul_i32 s22, s22, s9 // rowblk_abs*(16*FM)*K + s_mul_i32 s25, s31, SEGK // ksi*SEGK + s_add_u32 s22, s22, s25 + s_add_u32 s40, s2, s22 + s_addc_u32 s41, s3, 0 // s[40:41] = A base + s_mul_i32 s25, s23, (FM*256) // r*FM*256 + v_add_nc_u32 v13, v9, \sob // + slot operand base + v_add_nc_u32 v13, v13, ARES_ROFF // + A-within-slot offset (BRES_BYTES) + v_add_nc_u32 v13, v13, s25 // + r*FM*256 + .set mi, 0 + .rept FM + .if mi == 0 + s_mov_b32 s44, s40 + s_mov_b32 s45, s41 + .else + s_add_u32 s44, s44, s32 // += 16*K (next M-frag) + s_addc_u32 s45, s45, 0 + .endif + .set ks, 0 + .rept KSEG_STEPS + global_load_b64 v[BSTG:BSTG+1], v8, s[44:45] offset:(ks*16) + s_wait_loadcnt 0x0 + ds_store_b64 v13, v[BSTG:BSTG+1] offset:((ks*G*FM + mi)*256) + s_wait_dscnt 0x0 + .set ks, ks+1 + .endr + .set mi, mi+1 + .endr + s_add_u32 s47, \scb, SL_ARDONE + lds_inc_r s47 // rowblk r STAGED -> compute gates on SL_ARDONE==G + s_branch .Laclr\@ +.Lasdr\@: +.endm + +// ============================================================================================ +// Phase-B (DSWS2_CONV) consume-point ring-occupancy sensor -- Task 3, READ-ONLY (actuation is Task 5). +// Mirrors the coop occ_a/occ_b sensor (occ = producer - consumer, sampled where the value is +// CONSUMED, not at the segment boundary). The claimer's A7 wait-done spin runs CONCURRENTLY with the +// compute drain, so it observes the ring mid-flight; at the segment boundary the resident region has +// fully drained and occ would read a stuck ~0 (permanent false-starvation) -- exactly what SPEC warns. +// +// COUNTER IDENTITIES (confirmed against the live claim/consume sites -- see report): +// producer = the STORE-completion counters the compute wave actually gates on: +// A-ring: AROW_DONE_OFF (A rowblks resident, monotonic in [0,G]; lds_inc @ ASTAGE) +// B-ring: BFRAG_DONE_OFF (B frags resident, monotonic in [0,FN]; lds_inc @ BSTAGE) +// NOT the *_NEXT claim counters: AROW_NEXT/BFRAG_NEXT overshoot the ring depth by the role +// terminal-bails (G+NAFEED / FN+NBFEED), which would break the occ <= depth bound. +// consumer = ROWBLK_NEXT_OFF, the compute rowblk-claim clock (consume progress through the super-tile: +// each claimed rowblk r consumes A(r) and re-reads all FN shared B frags). +// min-clamp: cons is clamped to prod before the subtract so the u32 result cannot underflow when the +// consume clock outruns a shallower ring (G=6 > FN=4 -> ROWBLK_NEXT can exceed BFRAG_DONE). +// INVARIANT preserved: occ_A in [0,G], occ_B in [0,FN] (nonnegative, bounded by ring depth). +// +// REGISTER DISCIPLINE (brick-critical; this path is reachable pre-grow -- a >v15 vector temp is +// OOR-poison under dyn-VGPR, SPEC S4): scalars <= s65 only (s60/s61 scratch; callers pass dst in +// [s62,s65]); the only vector temps are inside lds_get, which uses v11/v14 (INTERIOR to the launch +// 16-VGPR block) -- NO >v15 temp is introduced here. +// GATE: DSWS2_CONV || DSWS2_ENVELOPE. reserve_try + the BUDGET default are the pool-economy primitives the +// rolling envelope needs INDEPENDENTLY of role conversion (they touch only VRESV_OFF/lds_fetch_add), so the +// envelope must be able to run at CONV=0 (the isolation config). Everything in this block is macro/.set +// definition (emits ZERO bytes), so widening the gate is byte-identical at CONV=0/ENV=0 and CONV=1. +.if DSWS2_CONV || DSWS2_ENVELOPE +.macro occ_sample dst_a, dst_b // out: \dst_a=occ_A in [0,G], \dst_b=occ_B in [0,FN]; clob s60,s61 + lds_get \dst_a, AROW_DONE_OFF // prod_a: A rowblks resident (store-completion) + lds_get \dst_b, BFRAG_DONE_OFF // prod_b: B frags resident (store-completion) + lds_get s60, ROWBLK_NEXT_OFF // cons : compute rowblk-claim consume clock + s_min_u32 s61, s60, \dst_a // cons_a = min(clock, prod_a) (clamp -> no u32 underflow) + s_sub_u32 \dst_a, \dst_a, s61 // occ_A = prod_a - cons_a in [0,G] + s_min_u32 s61, s60, \dst_b // cons_b = min(clock, prod_b) + s_sub_u32 \dst_b, \dst_b, s61 // occ_B = prod_b - cons_b in [0,FN] +.endm + +// ---- DSWS2_GQUIESCE: device-scoped GLOBAL QUIESCE handshake (mirrors the green occ[20]/occ[0] pattern). +// All three ops are lane-0-masked (v2==0), exec saved/restored via s49 (the LDS-macro convention -- s49 is +// never live across a macro boundary, so it is provably free at every site these replace an lds_* op). +// vaddr = v4 (the stable occ-base per-lane offset, =0, prologue-set), data/dst = v3/v5 (occ scratch vregs, +// same as the claim/live ops). scope:SCOPE_DEV + uncached occ buffer => device-coherent visibility (the +// fix). s_wait_storecnt/loadcnt drain before proceeding so the poll observes committed bumps. +.macro gq_reset // claimer: occ[QUIESCE_GOFF] = 0 (committed before EPOCH publish) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lgqr_skip\@ + v_mov_b32 v3, 0 + global_store_b32 v4, v3, s[0:1] offset:QUIESCE_GOFF scope:SCOPE_DEV + s_wait_storecnt 0x0 +.Lgqr_skip\@: + s_mov_b32 exec_lo, s49 +.endm +.macro gq_bump // follower: occ[QUIESCE_GOFF] += 1 (one bump/wave/super-tile) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lgqb_skip\@ + v_mov_b32 v3, 1 + global_atomic_add_u32 v4, v3, s[0:1] offset:QUIESCE_GOFF scope:SCOPE_DEV + s_wait_storecnt 0x0 +.Lgqb_skip\@: + s_mov_b32 exec_lo, s49 +.endm +.macro gq_read dst // claimer: \dst = occ[QUIESCE_GOFF] (lane0 load + broadcast) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lgqrd_skip\@ + global_load_b32 v5, v4, s[0:1] offset:QUIESCE_GOFF scope:SCOPE_DEV + s_wait_loadcnt 0x0 +.Lgqrd_skip\@: + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 \dst, v5 +.endm + +// ---- Pool-T7 chunk-2 wedge localization (DIAG-only; DSWS2_CONV=0 emits nothing -> .text byte-identical). +// epoch_mark: lane-0 publishes this role's live epoch (s35) to a host-streamed occ slot so a hung dispatch +// shows how far each role advanced (stream field roles[C/A/B]). v14<=v15 (feeds/compute are lean-32 at the +// _quiesce call sites), v4=0 (occ base lane offset, prologue), s49 exec-save (LDS-macro convention). ---- +.macro epoch_mark off +.if DSWS2_CONV && DIAG + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lem_skip\@ + v_mov_b32 v14, s35 + global_store_b32 v4, v14, s[0:1] offset:\off scope:SCOPE_DEV +.Lem_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +// bail_mark: PER-WAVE localization mark. Lane-0 writes this wave's epoch (s35) to occ[BAIL_BASE + wid*4] +// (runtime vaddr since the offset depends on wid=s24). s48 scratch, s49 exec-save (macro-local; free at the +// _quiesce bail sites), v13 vaddr, v14 data (both <=v15; the wave is lean-32 at every bail site). One-shot +// per super-tile -> negligible perturbation vs the DIAG per-spin claimer stores. Enabled by DIAG OR BAILMARK. +.macro bail_mark +.if DSWS2_CONV && (DIAG || DSWS2_BAILMARK) + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lbmk_skip\@ + s_lshl_b32 s48, s24, 2 // wid*4 + s_add_u32 s48, s48, BAIL_BASE // occ byte offset for THIS wave + v_mov_b32 v13, s48 // vaddr = per-wave byte offset (lane0) + v_mov_b32 v14, s35 // data = this wave's current epoch + global_store_b32 v13, v14, s[0:1] scope:SCOPE_DEV +.Lbmk_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +// -------------------------------------------------------------------------------------------- +// Phase-B controller thresholds + sum-envelope budget (Task 4). EPOCH_SHIFT mirrors coop / +// occ_dispatch (epoch = segcnt >> EPOCH_SHIFT). BUDGET is the per-WG VGPR sum-envelope ceiling +// the reservation counter must never exceed; default = the launch reservation, which makes the +// envelope a strict conservation law (a feed->compute grow can only fit if a compute->feed shrink +// already freed the delta). Task 5 may re-tune via `-defsym BUDGET=` if per-SIMD headroom exists. +// -------------------------------------------------------------------------------------------- +.ifndef EPOCH_SHIFT + .set EPOCH_SHIFT, 3 // decision clock: epoch = segcnt >> EPOCH_SHIFT (small = reactive) +.endif +.ifndef BUDGET +.if DSWS2_ENVELOPE + .set BUDGET, (WAVES*VLEAN + PEAK_CONC*(NFV-VLEAN)) // rolling: lean floor + concurrent-peak headroom +.else + .set BUDGET, (NCOMP*NFV + (NAFEED+NBFEED)*VLEAN) // = VRESV_OFF init (conservation ceiling) +.endif +.endif + +.if DSWS2_CONV +// compile-time no-parking invariant: every launched wave must fit lean at once +.if (WAVES * VLEAN) > BUDGET + .error "WAVES*VLEAN exceeds BUDGET — pool cannot stay all-lean (parking is out of scope)" +.endif +.endif +.if DSWS2_ENVELOPE +// forward-progress: the budget must admit at least one concurrent peak or a claimed wave can never grow +.if (WAVES*VLEAN + (NFV-VLEAN)) > BUDGET + .error "ENVELOPE: BUDGET admits < 1 concurrent peak — forward progress impossible" +.endif +.endif + +// try_gate: the lock-free single-winner conversion ticket (transcribed VERBATIM from occ_kernel_coop.s, +// which transcribes dsws_ctrl_model.cpp gate_try_win + epoch_of EXACTLY). E = segcnt>>EPOCH_SHIFT. +// gate[dir] holds the last epoch dir fired. Among many waves racing the same (gtry_gate->reserve_try point -- +// occ_sample's s62/s63 result is consumed into `dir` BEFORE this runs), v5/v6/v7 (<=v15: pre-grow / +// lean-safe). CAS operand order (gfx1201, GCN order -- NOT flipped, KG 9ed04f3c): +// ds_cmpstore_rtn_b32 vdst,vaddr,vNEW,vCMP -> MEM=(MEM==vCMP)?vNEW:MEM, vdst<-old. So vsrc0=E (new), +// vsrc1=g (compare). WIN iff returned-old == g. (Swapping them leaves gate stuck so old==g for ALL +// racers -> every racer "wins" -> would-win ~= NCOMP*epochs instead of ~= epochs.) +.macro try_gate dir, swin + lds_get s62, SEGCNT_OFF // E = epoch_of(segcnt, EPOCH_SHIFT) + s_lshr_b32 s62, s62, EPOCH_SHIFT + lds_get s63, (GATE_OFF + (\dir)*4) // g = gate[dir] + s_mov_b32 \swin, 0 + s_cmp_ge_u32 s63, s62 // g >= E -> dir already fired this/later epoch -> lose + s_cbranch_scc1 .Ltg_done\@ + s_mov_b32 s65, exec_lo // lane0-only CAS (one ticket attempt per WAVE) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltg_restore\@ + v_mov_b32 v5, (GATE_OFF + (\dir)*4) // vaddr = &gate[dir] + v_mov_b32 v6, s63 // v6 = g (vsrc1 = CMP/expected) + v_mov_b32 v7, s62 // v7 = E (vsrc0 = NEW value to store) + ds_cmpstore_rtn_b32 v6, v5, v7, v6 // gate[dir] = (gate[dir]==g) ? E : gate[dir]; v6 <- old + s_wait_dscnt 0x0 +.Ltg_restore\@: + s_mov_b32 exec_lo, s65 + v_readfirstlane_b32 s64, v6 // s64 = old (lane0's CAS result, broadcast) + s_cmp_eq_u32 s64, s63 // WIN iff old == g (we were the swapper) + s_cbranch_scc0 .Ltg_done\@ + s_mov_b32 \swin, 1 +.Ltg_done\@: +.endm + +// reserve_try: the VGPR sum-envelope reservation (transcribes reserve_grow, dsws_ctrl_model.cpp:47). +// Reserve first (atomic add of SIGNED \delta on vgpr_reserved), then validate prev+delta <= BUDGET; +// on over-budget cleanly UNDO (atomic add of -\delta) and reject. The LDS atomic serializes the <=2 +// concurrent grows an epoch permits: the second to validate sees the first's reservation and backs off. +// GROW (feed->compute): pass \delta = +(NFV-VLEAN). Over-budget -> undo, \won=0 (stay in role). +// SHRINK(compute->feed): pass \delta = -(NFV-VLEAN). new = prev+delta < prev <= BUDGET, so the +// validate branch is a proven no-op -> \won=1 ALWAYS (shrink never fails). +// One macro, one call site (Task 5 `reserve_try delta, s_ok`); direction is the sign of \delta. +// Scratch: s62/s63 (free at the bail-commit point -- try_gate's s62..s65 are long dead by then). +.macro reserve_try delta, won + lds_fetch_add s62, VRESV_OFF, (\delta) // s62 = prev reserved; vgpr_reserved += delta + s_add_u32 s63, s62, (\delta) // s63 = new reservation = prev + delta + s_mov_b32 \won, 1 + s_cmp_le_u32 s63, BUDGET // new <= BUDGET -> commit (win); shrink always passes + s_cbranch_scc1 .Lrt_done\@ + lds_fetch_add s62, VRESV_OFF, -(\delta) // over-budget: undo the reservation, reject + s_mov_b32 \won, 0 +.Lrt_done\@: +.endm + +// -------------------------------------------------------------------------------------------- +// Phase-B (Task 5) watermark thresholds + LDS put-runtime helper + bail-time commit macros. +// Watermark decision (SPEC; mirrors coop CTRL_LOW/CTRL_HIGH, occ_dispatch DSWS_LOW/HIGH): +// occ_X < CTRL_LOW -> compute STARVED for X -> shrink a compute wave into feed-X. +// occ_X > CTRL_HIGH_X -> feed-X OVER-SERVING -> grow a feed-X wave into compute. +// occ_A in [0,G], occ_B in [0,FN] (occ_sample bounds), so the HIGH marks are per-ring-depth. +// -------------------------------------------------------------------------------------------- +.ifndef CTRL_LOW + .set CTRL_LOW, 1 // occ_X < 1 (== 0, ring empty at consume) -> starved +.endif +.ifndef CTRL_HIGH_A + .set CTRL_HIGH_A, (G-1) // occ_A > G-1 -> A-ring saturated -> A-feed over-serving +.endif +.ifndef CTRL_HIGH_B + .set CTRL_HIGH_B, (FN-1) // occ_B > FN-1 -> B-ring saturated -> B-feed over-serving +.endif + +// lds_put_r: lane-0-of-wave write scalar \ssrc -> LDS[\saddr] (RUNTIME byte offset in a sreg). Mirrors +// the coop lds_put_v idiom but takes a SCALAR address (symmetry with lds_get_r). Used by the claimer's +// Step-4 snapshot write into the runtime parity half of SNAP_BASE. Temps RP_A/RP_D are v11/v14 (<=v15, +// pre-grow safe); s49 is the exec save (matches lds_put). +.macro lds_put_r saddr, ssrc + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lputr_skip\@ + v_mov_b32 v[RP_A], \saddr + v_mov_b32 v[RP_D], \ssrc + ds_store_b32 v[RP_A], v[RP_D] + s_wait_dscnt 0x0 +.Lputr_skip\@: + s_mov_b32 exec_lo, s49 +.endm + +// conv_dec_floor: floor-guarded ATOMIC decrement of a role slot -- \ok <- 1 iff it decremented \slot_off +// (only when the current value was > 1), else 0 (floor hit; source role must keep >= 1 wave). A +// ds_cmpstore_rtn_b32 CAS loop (re-reads on a lost race), so two same-source converters in one epoch +// (e.g. compute->Afeed and compute->Bfeed both dec NCOMP_SLOT) can never drive the slot below 1. +// Clob: s52 (read value), s53 (new/CAS-return), s65 (exec save); v5/v6/v7 (<=v15, pre-grow safe). +.macro conv_dec_floor slot_off, ok + s_mov_b32 \ok, 0 +.Lcdf_retry\@: + lds_get s52, \slot_off // s52 = current source-slot count + s_cmp_le_u32 s52, 1 + s_cbranch_scc1 .Lcdf_done\@ // <=1 -> at floor, cannot convert away (ok stays 0) + s_sub_u32 s53, s52, 1 // new = old - 1 + s_mov_b32 s65, exec_lo // lane0-only CAS (one attempt per WAVE) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lcdf_restore\@ + v_mov_b32 v5, \slot_off // vaddr = &slot + v_mov_b32 v6, s52 // v6 = expected old (CMP) + v_mov_b32 v7, s53 // v7 = new value (NEW) + ds_cmpstore_rtn_b32 v6, v5, v7, v6 // slot = (slot==old)? new : slot ; v6 <- prior + s_wait_dscnt 0x0 +.Lcdf_restore\@: + s_mov_b32 exec_lo, s65 + v_readfirstlane_b32 s53, v6 // s53 = prior (lane0 CAS result, broadcast) + s_cmp_eq_u32 s53, s52 // success iff prior == expected (we were the swapper) + s_cbranch_scc0 .Lcdf_retry\@ // lost the race -> re-read and retry + s_mov_b32 \ok, 1 +.Lcdf_done\@: +.endm + +// conv_apply: the bail-time role-conversion COMMIT (SPEC 3.4 Approach A). Precondition: s58 = s_win +// (1 iff this wave won the (dir,epoch) ticket). Ordered strictly BEFORE the QUIESCE_CNT bump the +// CALLER emits after this macro (the quiesce counter is the snapshot handshake). +// ORDER: (a) floor-guarded dec of \src_slot -> (b) reserve the VGPR sum-envelope \delta (shrink +// always ok; grow may abort over BUDGET) -> (c) on ok: inc \dst_slot, flip private role reg (s59), +// s_alloc_vgpr \alloc_sz (GROW=NFV feed->compute / SHRINK=32 compute->feed) with SCC-retry -> +// (d) on floor-fail or reserve-abort: cancel, remain current role (undo the source dec if a +// reservation abort happened after the dec). +// PRE-GROW OOR WINDOW (SPEC 4, #1 brick risk): the wave is lean-32 on entry; every LDS/atomic temp +// read before the s_alloc_vgpr GROW is <=v15 (occ_sample/try_gate v5/v6/v7 + v11/v14; conv_dec_floor +// v5/v6/v7; lds_fetch_add v11/v14) and every carried scalar is <=s65. NO >v15 source before GROW. +// Clob: s52,s53,s54 (+ conv_dec_floor / reserve_try scratch); s59 = new role slot id (record). +.macro conv_apply src_slot, dst_slot, delta, alloc_sz + s_cmp_eq_u32 s58, 0 + s_cbranch_scc1 .Lca_skip\@ // lost the ticket -> no conversion this bail + conv_dec_floor \src_slot, s54 // (a) floor-guarded atomic dec of source slot + s_cmp_eq_u32 s54, 0 + s_cbranch_scc1 .Lca_skip\@ // floor-fail (source at 1) -> cancel, remain current role + reserve_try (\delta), s53 // (b) reserve VGPR envelope (grow may abort; shrink ok) + s_cmp_eq_u32 s53, 0 + s_cbranch_scc0 .Lca_commit\@ + lds_fetch_add s52, \src_slot, 1 // (d) reserve aborted: UNDO the source dec, cancel + s_branch .Lca_skip\@ +.Lca_commit\@: + lds_fetch_add s52, \dst_slot, 1 // (c) inc dest slot (unbounded -> plain atomic add) + s_mov_b32 s59, \dst_slot // flip private current-role reg (records new role slot id) +.if DIAG || TRACE + // conversion-commit counter (proves a wave ACTUALLY switched role). Lean-32 pre-grow here -> v3/v4<=v15 + // OOR-safe; s49 exec-save (macro-local). (DIAG||TRACE)-gated -> DSWS2_CONV/DIAG=0/TRACE=0 byte-identical. + s_mov_b32 s49, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lca_cm_skip\@ + v_mov_b32 v3, 1 + global_atomic_add_u32 v4, v3, s[0:1] offset:CONVCNT_OFF scope:SCOPE_DEV // occ[48] += 1 + s_wait_storecnt 0x0 +.Lca_cm_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.if CONV_COOLDOWN > 0 + s_mov_b32 s66, CONV_COOLDOWN // Task 4: committed conversion -> arm cooldown +.endif + // ---- s_alloc_vgpr resize: THE pre-grow OOR window closes here; all reads above were <=v15 ---- +.Lca_alloc\@: + s_alloc_vgpr \alloc_sz // GROW(NFV) / SHRINK(32); SCC-retry (brick-class rule) + s_cbranch_scc0 .Lca_alloc\@ +.Lca_skip\@: +.endm +.endif + +// ============================================================================================ +// TFPROBE wall-span capture (TF throughput probe). Realtime-tick min/max into occ[2]/occ[3], +// mirroring occ_kernel_coop.s's proven timer idiom. Each wave stamps lane-0 only (exec-masked +// via s49, the DSWS exec-save convention); base addr v4==0 holds kernel-wide (prologue, line +// ~765; invariant per the "v4=0 occ base lane offset" note). s[30:31] free at entry (DECODE_STI +// outputs, computed only inside role bodies) and dead at every terminal. Emits ZERO bytes at +// TFPROBE=0 -> production .text byte-identical. +// ============================================================================================ +.macro tfspan op:req, off:req // op = min (entry, occ[2]/off 8) | max (exit, occ[3]/off 12) +.if TFPROBE + s_sendmsg_rtn_b64 s[30:31], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + v_cmp_eq_u32 vcc_lo, 0, v2 // lane 0 of each wave only (v2 = tid & 31) + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltfspan_skip\@ + v_mov_b32 v5, s30 // low 32 bits of the realtime tick + global_atomic_\op\()_u32 v4, v5, s[0:1] offset:\off scope:SCOPE_DEV +.Ltfspan_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +// ============================================================================================ +// PHASEPROBE: in-kernel per-phase tick timer for the COMPUTE wave (the critical path). s77 holds +// this wave's last-stamp RTC (low 32b). phase_reset seeds it; phase_stamp accumulates (now-s77) +// into occ[\off] and re-seeds. Lane-0-only atomic add; occ slots are ABOVE the per-chunk memset +// so they sum over the whole run. Scratch s62/s63 (RTC), s64 (delta) -- all free in CONV=0 compute. +// Six phases -> six occ accumulators (bytes 256..276): +.set PH_FOLLOW_OFF, 256 // occ[64]: waiting on claimer to publish next super-tile +.set PH_STAGE_OFF, 260 // occ[65]: waiting on A/B feeds to stage this super-tile +.set PH_GROW_OFF, 264 // occ[66]: claim rowblk + s_alloc_vgpr GROW 32->112 +.set PH_WMMA_OFF, 268 // occ[67]: LDS frag loads + v_wmma compute +.set PH_FLUSH_OFF, 272 // occ[68]: global_atomic_add_f32 C flush (split-K reduction) +.set PH_SHRINK_OFF, 276 // occ[69]: s_alloc_vgpr SHRINK 112->32 +// Per-wave phase accumulators live in SGPRs s78..s83 (NO per-stamp store -> zero memory perturbation, no +// s_wait_storecnt pollution). s77 = last-stamp RTC. phase_flush emits them ONCE at compute retire. +.macro phase_reset +.if PHASEPROBE + s_mov_b32 s78, 0 + s_mov_b32 s79, 0 + s_mov_b32 s80, 0 + s_mov_b32 s81, 0 + s_mov_b32 s82, 0 + s_mov_b32 s83, 0 + s_sendmsg_rtn_b64 s[62:63], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + s_mov_b32 s77, s62 +.endif +.endm +.macro phase_stamp acc:req // \acc += (now - s77); s77 = now (pure scalar, no store) +.if PHASEPROBE + s_sendmsg_rtn_b64 s[62:63], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + s_sub_u32 s64, s62, s77 // delta (mod 2^32; phase << 2^32 so wrap-safe) + s_mov_b32 s77, s62 + s_add_u32 \acc, \acc, s64 +.endif +.endm +.macro phase_flush // lane0 atomic-adds s78..s83 -> occ[64..69]; drained here (not the hot loop) +.if PHASEPROBE + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lphf_skip\@ + v_mov_b32 v5, s78 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_FOLLOW_OFF scope:SCOPE_DEV + v_mov_b32 v5, s79 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_STAGE_OFF scope:SCOPE_DEV + v_mov_b32 v5, s80 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_GROW_OFF scope:SCOPE_DEV + v_mov_b32 v5, s81 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_WMMA_OFF scope:SCOPE_DEV + v_mov_b32 v5, s82 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_FLUSH_OFF scope:SCOPE_DEV + v_mov_b32 v5, s83 + global_atomic_add_u32 v4, v5, s[0:1] offset:PH_SHRINK_OFF scope:SCOPE_DEV + s_wait_storecnt 0x0 +.Lphf_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +// ============================================================================================ +// TRACE: per-super-tile time-series row (claimer, lane 0). Written once per super-tile at the +// quiesce-satisfied drain-exit (.Lqc_q_ok). Captures the adaptive wave-role economy over time: +// the LIVE role slots (do waves convert?), the per-super-tile ring-occupancy PEAK (s73/s74, +// tracked across the wait_done spins), the cumulative conversion count, and the envelope vresv. +// 16 u32/row -> buffer[segcnt*64]; bounded by MAXROWS (s72). Emits ZERO bytes at TRACE=0. +// Persistent trace regs: s70:s71 = buffer VA, s72 = MAXROWS, s73/s74 = ring occA/occB peak. +// ============================================================================================ +.macro alllive_dec // TRACE: --live on wave exit (pairs with the entry ++ for peak-concurrent) +.if TRACE + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lalld_skip\@ + v_mov_b32 v3, -1 + global_atomic_add_u32 v4, v3, s[0:1] offset:ALLLIVE_OFF scope:SCOPE_DEV +.Lalld_skip\@: + s_mov_b32 exec_lo, s49 +.endif +.endm + +.macro trace_row +.if TRACE + // claim a GLOBALLY-unique row index (all WGs' claimers share occ[55]) -> no per-WG SEGCNT collision. + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + v_mov_b32 v14, 1 + global_atomic_add_u32 v15, v4, v14, s[0:1] offset:TRACE_IDX_OFF th:TH_ATOMIC_RETURN scope:SCOPE_DEV // v15=old idx, v4=addr(0), v14=data(1) + s_wait_loadcnt 0x0 + s_mov_b32 exec_lo, s49 + v_readfirstlane_b32 s52, v15 // s52 = unique row index (old value returned by the atomic) + s_cmp_ge_u32 s52, s72 // row >= MAXROWS -> skip (buffer bound) + s_cbranch_scc1 .Ltrow_skip\@ + s_lshl_b32 s53, s52, 6 // row * TRACE_ROW_BYTES(64) + s_add_u32 s60, s70, s53 + s_addc_u32 s61, s71, 0 // s[60:61] = row base VA + s_sendmsg_rtn_b64 s[58:59], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + lds_get s55, SEGCNT_OFF // this WG's per-WG super-tile counter (data field, not index) + v_mov_b32 v14, s58 // 0 tick_lo + v_mov_b32 v15, s55 // 1 segcnt (per-WG) + v_mov_b32 v16, s35 // 2 epoch + lds_get s54, NCOMP_SLOT + v_mov_b32 v17, s54 // 3 nComp (live role slot) + lds_get s54, NAFEED_SLOT + v_mov_b32 v18, s54 // 4 nAfeed + lds_get s54, NBFEED_SLOT + v_mov_b32 v19, s54 // 5 nBfeed + v_mov_b32 v20, s73 // 6 occA peak (across wait_done spins) + v_mov_b32 v21, s74 // 7 occB peak + global_load_b32 v22, v4, s[0:1] offset:CONVCNT_OFF scope:SCOPE_DEV // 8 convCount (cumulative) + lds_get s54, VRESV_OFF + v_mov_b32 v23, s54 // 9 vresv (envelope budget) + v_mov_b32 v24, s17 // 10 sti (claimed super-tile id) + lds_get s54, QUIESCE_CNT_OFF + v_mov_b32 v25, s54 // 11 quiesce (final) + v_mov_b32 v26, s59 // 12 tick_hi + v_mov_b32 v27, s69 // 13 chunkHi (context) + v_mov_b32 v28, s75 // 14 wg_id (which workgroup's economy this row belongs to) + v_mov_b32 v29, 0 // 15 reserved + s_wait_loadcnt 0x0 // convCount load drained before the row store + v_cmp_eq_u32 vcc_lo, 0, v2 // lane 0 of the claimer writes the row + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltrow_wskip\@ + global_store_b128 v4, v[14:17], s[60:61] offset:0 scope:SCOPE_DEV + global_store_b128 v4, v[18:21], s[60:61] offset:16 scope:SCOPE_DEV + global_store_b128 v4, v[22:25], s[60:61] offset:32 scope:SCOPE_DEV + global_store_b128 v4, v[26:29], s[60:61] offset:48 scope:SCOPE_DEV +.Ltrow_wskip\@: + s_mov_b32 exec_lo, s49 +.Ltrow_skip\@: +.endif +.endm + +// ============================================================================================ +// KERNEL +// ============================================================================================ + .text + .globl occ_kernel + .p2align 8 + .type occ_kernel,@function +occ_kernel: + // ---- FIX 1(b,c): derive n_kseg from KT (s8) + the compile-time NKSEG_SHIFT, then the shift/mask + // decode pair, into the reserved high SGPRs s66/s67/s68 BEFORE any clobber (SAFEPROBE below reuses + // s16 purely as scratch; the role bodies keep all transients <= s65). No v2 kernargs are read here + // anymore -- s15/s16/s17 are NOT hardware-preloaded under USER_SGPR=15 (see KERNARG CONTRACT above). ---- + s_lshr_b32 s66, s8, NKSEG_SHIFT // n_kseg = KT >> NKSEG_SHIFT (KT=s8) + s_ff1_i32_b32 s68, s66 // shift = log2(n_kseg) (bit index of the single set bit; n_kseg=1 -> 0) + s_sub_u32 s67, s66, 1 // mask = n_kseg - 1 + // ---- identity (lifted from coop prologue; v0=tid hardware-preloaded) ---- + v_lshrrev_b32 v1, 5, v0 // wid = tid >> 5 + v_and_b32 v2, 31, v0 // lane = tid & 31 + v_and_b32 v6, 15, v0 // lane & 15 (A vaddr) + v_mov_b32 v4, 0 +.if KMAJOR + global_load_b32 v3, v4, s[0:1] offset:248 scope:SCOPE_DEV // occ[62] = magic(TOTAL), host-written + s_wait_loadcnt 0x0 + v_readfirstlane_b32 s76, v3 // s76 = magic_TOTAL, persistent for every DECODE_STI (K-major) +.endif + // ---- per-lane address constants (mbgemm-identical; dyn-VGPR arming compatible) ---- + v_mul_lo_u32 v8, v6, s9 // (lane&15)*K + v_bfe_u32 v7, v0, 4, 1 + v_lshlrev_b32 v7, 3, v7 + v_add_nc_u32 v8, v8, v7 // v8 = A vaddr = (lane&15)*K + colhi*8 + v_lshlrev_b32 v9, 3, v2 // v9 = B/ds vaddr = lane*8 + v_lshlrev_b32 v10, 5, v2 // v10 = C store vaddr = lane*32 +.if SAFEPROBE + // brick-PROOF: clamp the per-lane VECTOR address regs to a loose upper bound (>= true max) so even a + // grow-corrupted vaddr cannot push a global access past the data+guard (pairs with the future ti clamp). + s_lshl_b32 s16, s9, 4 // 16*K (>= v8 max = (lane&15)*K + colhi*8 = 15*K+8) + v_min_u32 v8, s16, v8 // clamp A vaddr + v_min_u32 v9, 0x100, v9 // clamp B/ds vaddr (256 >= lane*8 max 248) + v_min_u32 v10, 0x400, v10 // clamp C vaddr (1024 >= lane*32 max 992) +.endif + + tfspan min, 8 // TFPROBE: every wave stamps occ[2] = min entry tick (wall-span start) +.if TRACE + // total-occupancy: every wave ++live at entry, atomic-max the peak concurrent resident count (occ[1]). + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s49, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lall_enter_skip + v_mov_b32 v3, 1 + global_atomic_add_u32 v5, v4, v3, s[0:1] offset:ALLLIVE_OFF th:TH_ATOMIC_RETURN scope:SCOPE_DEV + s_wait_loadcnt 0x0 + v_add_nc_u32 v5, v5, 1 + global_atomic_max_u32 v4, v5, s[0:1] offset:4 scope:SCOPE_DEV // occ[1] = peak concurrent resident waves +.Lall_enter_skip: + s_mov_b32 exec_lo, s49 +.endif + +.if DSWS2 + // ===== DSWS v2 role branch (wid uniform per wave; scalar-only -> exec stays full for every role). + // wid == 0 -> claimer (pinned super-tile broadcaster; A3) + // wid [0,NBFEED) -> B-feed (A4) + // wid [NBFEED,NBFEED+NAFEED) -> A-feed (A5) + // wid [NBFEED+NAFEED, WAVES) -> compute (A6) ===== + // A1: every role label is just a distinct s_endpgm stub (unique s50 tag keeps them at distinct addresses). + v_readfirstlane_b32 s24, v1 // wid (uniform per wave) + s_cmp_eq_u32 s24, 0 + s_cbranch_scc1 .Ldispatcher // FIX 1a: wid0 = ring dispatcher (was .Lclaimer) +.if DSWS2_CONV && DIAG && DSWS2_TICKET_SELFTEST + // Task 4 Step 3 -- try_gate single-winner SMOKE (assemble-only stub; default off). Every non-claimer + // wave races the (dir=0) ticket ONCE and atomic-adds its win (0/1) into occ[28] (byte offset 112, + // clear of the 0/20/24/104/108 control+probe words). On GPU (Task 6, if enabled) the sum should land + // near #epochs, NOT NCOMP*#epochs -- the harness-side proof the LDS-CAS yields <=1 winner/(dir,epoch). + // v4=0 (set in prologue), v2=lane; try_gate temps v5/v6/v7 are <=v15 (pre-grow safe). wid (s24) + // survives -- try_gate touches only s62..s65 / s16. NOTE: pre-init-rendezvous placement -> a real run + // reads gate/segcnt before the claimer publishes them; fine for an assemble/smoke stub. + try_gate 0, s50 + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ltg_selftest_skip + v_mov_b32 v5, s50 // win flag (0/1) for THIS wave + global_atomic_add_u32 v4, v5, s[0:1] offset:112 scope:SCOPE_DEV // occ[28] += win +.Ltg_selftest_skip: + s_mov_b32 exec_lo, s16 +.endif +.if DSWS2_CONV +.if CONV_COOLDOWN > 0 + s_mov_b32 s66, 0 // Task 4: init cooldown ctr (un-cooled at entry) +.endif + // FIRST-time entry seeds the role reg s59 then falls into the role's FULL entry + // (.Lcompute/.Lafeed/.Lbfeed -> _alloc -> _init -> _follow). It must NOT jump to + // .Ldispatch: that trampoline lands on _follow and is correct ONLY for RE-dispatch + // (a wave that already ran _alloc/_init once). First entry via _follow would skip the + // s_alloc_vgpr 32 allocator handshake, the INITFLAG==0xACED rendezvous, and the s35=0 + // epoch seed -> followers desync and the claimer hangs in .Lclaimer_wait_done (Pool-T7 brick). + s_cmp_lt_u32 s24, NBFEED + s_cbranch_scc1 .Lseed_bfeed + s_cmp_lt_u32 s24, (NBFEED+NAFEED) + s_cbranch_scc1 .Lseed_afeed + s_mov_b32 s59, NCOMP_SLOT + s_branch .Lcompute +.Lseed_afeed: + s_mov_b32 s59, NAFEED_SLOT + s_branch .Lafeed +.Lseed_bfeed: + s_mov_b32 s59, NBFEED_SLOT + s_branch .Lbfeed +.else + s_cmp_lt_u32 s24, NBFEED + s_cbranch_scc1 .Lbfeed + s_cmp_lt_u32 s24, (NBFEED+NAFEED) + s_cbranch_scc1 .Lafeed + s_branch .Lcompute +.endif + +// ============================================================================================ +// FIX 1a -- .Ldispatcher : pinned wid-0 (was .Lclaimer). PURE COORDINATION -- NO BSTAGE (deviation +// from the old B-feed-class claimer; effective B-stager count is NBFEED-1, so pick the launch mix +// accordingly -- the exact feed:compute ratio is fix #3, not 1a). Owns: the barrier-free LDS init +// (shared frontier + BOTH slot control blocks), the completion live++/live-- (harness occ[0]==0 +// gate), the GLOBAL super-tile claim (occ[20]), and the D=2 ring fill/recycle. Registers: +// s36 = gen -- GLOBAL-monotonic publish counter (0 == "slot never published") +// s37 = fill_slot -- which slot to fill next (alternates 0,1,0,1 with D=2) +// s34 = fill_slot's control base (SLOTC_BASE + fill_slot*SLOTC_STRIDE) +// s17 = sti, s69 = chunkHi, s44/s45 = scratch, s16 = exec save +// Steady state: fill a FREE slot, publish (STI -> FILL_IDX -> SL_GEN LAST = release fence), then +// wait THAT slot READY (SL_BFDONE==FN && SL_ARDONE==G) before alternating fill_slot -- so feeds +// never split across slots and compute drains the OTHER slot throughout (STAGE/FOLLOW hidden). +// Reset of a slot's counters happens ONLY here, ONLY after the prior occupant's SL_RBDONE>=G, so +// the barrier-free reset-race is structurally impossible (see RING_SLOTS_DESIGN.md). +// occ[0]=live counter (harness polls ==0); occ[20]=global super-tile claim; occ[24]=chunkHi. +// ============================================================================================ +.Ldispatcher: +.if DYNVGPR +.Ldisp_alloc: + s_alloc_vgpr 32 // dispatcher stays lean (never grows) + s_cbranch_scc0 .Ldisp_alloc +.endif + // live++ : lane0 occ[0] += 1 + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ldisp_live + v_mov_b32 v3, 1 + global_atomic_add_u32 v4, v3, s[0:1] scope:SCOPE_DEV +.Ldisp_live: + s_mov_b32 exec_lo, s16 + // --- barrier-free LDS init: shared frontier + BOTH slot control blocks; RINGINIT = 0xACED LAST --- + lds_put FILL_IDX_OFF, 0 + lds_put DRAIN_IDX_OFF, 0 + // slot 0 control block (8 words) + lds_put (SLOTC_BASE + 0*SLOTC_STRIDE + SL_STI), 0 + lds_put (SLOTC_BASE + 0*SLOTC_STRIDE + SL_GEN), 0 + lds_put (SLOTC_BASE + 0*SLOTC_STRIDE + SL_RBNEXT), 0 + lds_put (SLOTC_BASE + 0*SLOTC_STRIDE + SL_RBDONE), 0 + lds_put (SLOTC_BASE + 0*SLOTC_STRIDE + SL_BFNEXT), 0 + lds_put (SLOTC_BASE + 0*SLOTC_STRIDE + SL_BFDONE), 0 + lds_put (SLOTC_BASE + 0*SLOTC_STRIDE + SL_ARNEXT), 0 + lds_put (SLOTC_BASE + 0*SLOTC_STRIDE + SL_ARDONE), 0 + // slot 1 control block (8 words) + lds_put (SLOTC_BASE + 1*SLOTC_STRIDE + SL_STI), 0 + lds_put (SLOTC_BASE + 1*SLOTC_STRIDE + SL_GEN), 0 + lds_put (SLOTC_BASE + 1*SLOTC_STRIDE + SL_RBNEXT), 0 + lds_put (SLOTC_BASE + 1*SLOTC_STRIDE + SL_RBDONE), 0 + lds_put (SLOTC_BASE + 1*SLOTC_STRIDE + SL_BFNEXT), 0 + lds_put (SLOTC_BASE + 1*SLOTC_STRIDE + SL_BFDONE), 0 + lds_put (SLOTC_BASE + 1*SLOTC_STRIDE + SL_ARNEXT), 0 + lds_put (SLOTC_BASE + 1*SLOTC_STRIDE + SL_ARDONE), 0 + lds_put RINGINIT_OFF, 0xACED // LAST: publishes "LDS ready" to all follower waves + // load this dispatch's chunk terminal bound (host writes occ[24] per chunk); stable -> load ONCE + global_load_b32 v6, v4, s[0:1] offset:24 scope:SCOPE_DEV + s_wait_loadcnt 0x0 + v_readfirstlane_b32 s69, v6 // s69 = chunkHi (terminal sti bound) + s_mov_b32 s36, 0 // gen = 0 (first real publish will be gen 1) + s_mov_b32 s37, 0 // fill_slot = 0 +.Ldisp_loop: + s_lshl_b32 s34, s37, 5 // fill_slot * SLOTC_STRIDE(32) + s_add_u32 s34, s34, SLOTC_BASE // s34 = fill_slot control base + // wait slot FREE: SL_GEN==0 (never published) OR SL_RBDONE>=G (prior occupant drained) +.Ldisp_wait_free: + s_sleep SLEEPN + s_add_u32 s45, s34, SL_GEN + lds_get_r s44, s45 + s_cmp_eq_u32 s44, 0 + s_cbranch_scc1 .Ldisp_free_ok + s_add_u32 s45, s34, SL_RBDONE + lds_get_r s44, s45 + s_cmp_lt_u32 s44, G + s_cbranch_scc1 .Ldisp_wait_free +.Ldisp_free_ok: + // claim next global super-tile: lane0 occ[20] += 1 (return old) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ldisp_grabbed + v_mov_b32 v3, 1 + global_atomic_add_u32 v5, v4, v3, s[0:1] offset:20 th:TH_ATOMIC_RETURN scope:SCOPE_DEV + s_wait_loadcnt 0x0 +.Ldisp_grabbed: + s_mov_b32 exec_lo, s16 + v_readfirstlane_b32 s17, v5 // sti + s_cmp_ge_u32 s17, s69 // sti >= chunkHi -> terminal + s_cbranch_scc1 .Ldisp_terminal + // reset this slot's per-super-tile counters BEFORE publishing gen (followers gate on SL_GEN) + s_add_u32 s45, s34, SL_RBNEXT + lds_put_r s45, 0 + s_add_u32 s45, s34, SL_RBDONE + lds_put_r s45, 0 + s_add_u32 s45, s34, SL_BFNEXT + lds_put_r s45, 0 + s_add_u32 s45, s34, SL_BFDONE + lds_put_r s45, 0 + s_add_u32 s45, s34, SL_ARNEXT + lds_put_r s45, 0 + s_add_u32 s45, s34, SL_ARDONE + lds_put_r s45, 0 + // publish STI first, then FILL_IDX (feeds now target this slot), then SL_GEN LAST (release fence) + s_add_u32 s45, s34, SL_STI + lds_put_r s45, s17 + lds_put FILL_IDX_OFF, s37 + s_add_u32 s36, s36, 1 // gen++ + s_add_u32 s45, s34, SL_GEN + lds_put_r s45, s36 // release: consumers observe the new occupant here + // wait THIS slot READY (fully staged) before advancing fill_slot -> feeds never split across slots +.Ldisp_wait_ready: + s_sleep SLEEPN + s_add_u32 s45, s34, SL_BFDONE + lds_get_r s44, s45 + s_cmp_lt_u32 s44, FN + s_cbranch_scc1 .Ldisp_wait_ready + s_add_u32 s45, s34, SL_ARDONE + lds_get_r s44, s45 + s_cmp_lt_u32 s44, G + s_cbranch_scc1 .Ldisp_wait_ready + s_xor_b32 s37, s37, 1 // alternate fill_slot (D=2) + s_branch .Ldisp_loop +.Ldisp_terminal: + // Publish SENTINEL into fill_slot (the confirmed-FREE slot; s34 still holds its base) + point + // FILL_IDX at it + bump its gen LAST. Feeds (via FILL_IDX) and compute (via the DRAIN_IDX + // frontier-advance that lands here after the OTHER slot's last real work drains) then wake and + // retire. The other slot, if it still holds live work, is drained normally first -- we do NOT + // sentinel it (that would retire compute before its rowblks are done). See RING_SLOTS_DESIGN.md. + s_add_u32 s36, s36, 1 // gen++ (sentinel publish) + s_mov_b32 s44, 0xFFFFFFFF + s_add_u32 s45, s34, SL_STI + lds_put_r s45, s44 // STI[fill_slot] = SENTINEL + lds_put FILL_IDX_OFF, s37 // feeds target the sentineled slot + s_add_u32 s45, s34, SL_GEN + lds_put_r s45, s36 // release LAST + // live-- : lane0 occ[0] -= 1 (harness completion gate fires) + v_cmp_eq_u32 vcc_lo, 0, v2 + s_mov_b32 s16, exec_lo + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Ldisp_dead + v_mov_b32 v3, -1 + global_atomic_add_u32 v4, v3, s[0:1] scope:SCOPE_DEV +.Ldisp_dead: + s_mov_b32 exec_lo, s16 + tfspan max, 12 // TFPROBE: dispatcher stamps occ[3] = max exit tick (wall-span end) + alllive_dec + s_endpgm + +// ============================================================================================ +// FIX 1a -- .Lbfeed : B-feed wave. Gates on SL_GEN[FILL_IDX] > last-seen-gen (new occupant), reads +// STI[FILL_IDX] (sentinel -> retire), stages its claimed B frags into the FILL_IDX slot's operand +// buffer via BSTAGE_R. A feed that skips a generation is SAFE: the dispatcher's wait-READY gate +// guarantees the skipped slot was fully staged by other feeds before FILL_IDX advanced off it, and +// compute gates on SL_BFDONE (a count), not on which feed contributed (see RING_SLOTS_DESIGN.md). +// Registers: s35 = last-seen gen, s38 = FILL_IDX, s48 = scb, s52 = sob, s44/s45 = scratch. +// ============================================================================================ +.Lbfeed: +.if DYNVGPR +.Lbfeed_alloc: + s_alloc_vgpr 32 + s_cbranch_scc0 .Lbfeed_alloc +.endif +.Lbfeed_init: + s_sleep 1 + lds_get s44, RINGINIT_OFF + s_cmp_eq_u32 s44, 0xACED + s_cbranch_scc0 .Lbfeed_init // wait the dispatcher's barrier-free LDS init + s_mov_b32 s35, 0 // last-seen gen (0 = none staged yet) +.Lbfeed_follow: + s_sleep SLEEPN + lds_get s38, FILL_IDX_OFF // current fill slot + s_lshl_b32 s48, s38, 5 + s_add_u32 s48, s48, SLOTC_BASE // s48 = scb (FILL_IDX control base) + s_add_u32 s45, s48, SL_GEN + lds_get_r s44, s45 // SL_GEN[FILL_IDX] + s_cmp_le_u32 s44, s35 // gen <= last-seen -> no new occupant yet -> spin + s_cbranch_scc1 .Lbfeed_follow + s_mov_b32 s35, s44 // adopt the new gen + s_add_u32 s45, s48, SL_STI + lds_get_r s17, s45 // STI[FILL_IDX] + s_cmp_eq_u32 s17, 0xFFFFFFFF // sentinel -> retire + s_cbranch_scc1 .Lretire + DECODE_STI // s30=tcol s31=ksi (mblk unused) + s_lshl_b32 s52, s38, 14 // FILL_IDX * OPSTRIDE(16384) + s_add_u32 s52, s52, OP_BASE // s52 = sob (FILL_IDX operand base) + BSTAGE_R s48, s52 + s_branch .Lbfeed_follow + +// ============================================================================================ +// FIX 1a -- .Lafeed : A-feed wave. Symmetric to .Lbfeed; stages A rowblks into FILL_IDX via ASTAGE_R. +// ============================================================================================ +.Lafeed: +.if DYNVGPR +.Lafeed_alloc: + s_alloc_vgpr 32 + s_cbranch_scc0 .Lafeed_alloc +.endif +.Lafeed_init: + s_sleep 1 + lds_get s44, RINGINIT_OFF + s_cmp_eq_u32 s44, 0xACED + s_cbranch_scc0 .Lafeed_init + s_mov_b32 s35, 0 +.Lafeed_follow: + s_sleep SLEEPN + lds_get s38, FILL_IDX_OFF + s_lshl_b32 s48, s38, 5 + s_add_u32 s48, s48, SLOTC_BASE + s_add_u32 s45, s48, SL_GEN + lds_get_r s44, s45 + s_cmp_le_u32 s44, s35 + s_cbranch_scc1 .Lafeed_follow + s_mov_b32 s35, s44 + s_add_u32 s45, s48, SL_STI + lds_get_r s17, s45 + s_cmp_eq_u32 s17, 0xFFFFFFFF + s_cbranch_scc1 .Lretire + DECODE_STI // s19=mblk s31=ksi (tcol unused) + s_lshl_b32 s52, s38, 14 + s_add_u32 s52, s52, OP_BASE + ASTAGE_R s48, s52 + s_branch .Lafeed_follow + +// ============================================================================================ +// FIX 1a -- .Lcompute : compute wave. Gates on SL_GEN[DRAIN_IDX] > last-seen-gen, waits the slot +// fully STAGED (SL_BFDONE==FN && SL_ARDONE==G), claims rowblks from SL_RBNEXT[DRAIN_IDX], grows, +// runs WMMA over the slot's resident operand buffer, flushes fp32 partials into C via +// global_atomic_add_f32 (split-K segments accumulate), shrinks, and bumps SL_RBDONE. The wave whose +// SL_RBDONE increment reaches G (old==G-1) is the CLOSER: it advances DRAIN_IDX to the other slot +// (lock-free frontier advance). Because the next slot is pre-published AND pre-staged (dispatcher +// keeps one ahead + waits READY), FOLLOW_WAIT/STAGE_WAIT collapse toward zero in steady state. +// Registers: s35=last-seen gen, s46=DRAIN_IDX, s48=scb (control base), s52=sob (operand base) -- +// all survive the whole rowblk loop; s44/s45/s47 scratch; s33=rowblk; s37/s38/s39/s28/s29 = C addr. +// ============================================================================================ +.Lcompute: +.if DYNVGPR +.Lcompute_alloc: + s_alloc_vgpr 32 // lean baseline; grow per rowblk + s_cbranch_scc0 .Lcompute_alloc +.endif +.Lcompute_init: + s_sleep 1 + lds_get s44, RINGINIT_OFF + s_cmp_eq_u32 s44, 0xACED + s_cbranch_scc0 .Lcompute_init + s_mov_b32 s35, 0 + phase_reset // PHASEPROBE: seed this compute wave's phase clock +.Lcompute_follow: + s_sleep SLEEPN + lds_get s46, DRAIN_IDX_OFF // current drain slot + s_lshl_b32 s48, s46, 5 + s_add_u32 s48, s48, SLOTC_BASE // s48 = scb (DRAIN control base) + s_add_u32 s45, s48, SL_GEN + lds_get_r s44, s45 // SL_GEN[DRAIN_IDX] + s_cmp_le_u32 s44, s35 // gen <= last-seen -> no new occupant -> spin + s_cbranch_scc1 .Lcompute_follow + s_mov_b32 s35, s44 + phase_stamp s78 // close FOLLOW_WAIT (spun waiting for the next super-tile) + s_add_u32 s45, s48, SL_STI + lds_get_r s17, s45 // STI[DRAIN_IDX] +.if PHASEPROBE + s_cmp_eq_u32 s17, 0xFFFFFFFF // sentinel -> flush phase accumulators, then retire + s_cbranch_scc0 .Lcompute_go + phase_flush + s_branch .Lretire +.Lcompute_go: +.else + s_cmp_eq_u32 s17, 0xFFFFFFFF // sentinel -> retire + s_cbranch_scc1 .Lretire +.endif + DECODE_STI // s19=mblk s30=tcol s31=ksi + s_lshl_b32 s52, s46, 14 // DRAIN_IDX * OPSTRIDE(16384) + s_add_u32 s52, s52, OP_BASE // s52 = sob (DRAIN operand base) + // wait until resident A AND B fully STAGED (B: FN frags stored, A: G rowblks stored) +.Lcompute_staged: + s_sleep SLEEPN + s_add_u32 s45, s48, SL_BFDONE + lds_get_r s44, s45 + s_cmp_lt_u32 s44, FN + s_cbranch_scc1 .Lcompute_staged + s_add_u32 s45, s48, SL_ARDONE + lds_get_r s44, s45 + s_cmp_lt_u32 s44, G + s_cbranch_scc1 .Lcompute_staged + phase_stamp s79 // close STAGE_WAIT (spun waiting for A/B feeds) + // C tile-term: ti = mblk*NTL + tcol ; ti*(G*FM*FN*1024) (ksi-INDEPENDENT -> split-K accumulates) + s_mul_i32 s38, s19, s13 + s_add_u32 s38, s38, s30 + s_mul_i32 s38, s38, (G*FM*FN*1024) +.Lcompute_claim: + s_add_u32 s45, s48, SL_RBNEXT + lds_fetch_add_r s33, s45, 1 // claim rowblk r in [0,G) + s_cmp_ge_u32 s33, G + s_cbranch_scc1 .Lcompute_drained +.if DYNVGPR + s_wait_loadcnt 0x0 + s_wait_storecnt 0x0 +.Lcompute_grow: + s_alloc_vgpr NFV // grow (SCC-retry guarded, brick-class rule; M=576 safe) + s_cbranch_scc0 .Lcompute_grow +.endif + phase_stamp s80 // close GROW (rowblk claim + s_alloc_vgpr 32->112) + // zero FM*FN fp32 accumulators + .set idx, 0 + .rept FM*FN + v_mov_b32 v[ACC+idx*8+0], 0 + v_mov_b32 v[ACC+idx*8+1], 0 + v_mov_b32 v[ACC+idx*8+2], 0 + v_mov_b32 v[ACC+idx*8+3], 0 + v_mov_b32 v[ACC+idx*8+4], 0 + v_mov_b32 v[ACC+idx*8+5], 0 + v_mov_b32 v[ACC+idx*8+6], 0 + v_mov_b32 v[ACC+idx*8+7], 0 + .set idx, idx+1 + .endr + // resident operand bases: v9 + sob + region [+ r*FM*256 for A] (BRES_ROFF=0, ARES_ROFF=BRES_BYTES) + v_add_nc_u32 v12, v9, s52 + s_mul_i32 s37, s33, (FM*256) + v_add_nc_u32 v13, v9, s52 + v_add_nc_u32 v13, v13, ARES_ROFF + v_add_nc_u32 v13, v13, s37 + // WMMA over the SEGK segment (KSEG_STEPS k-steps); read resident B(ks) + A(ks,r) from LDS + .set ks, 0 + .rept KSEG_STEPS + .set ni, 0 + .rept FN + ds_load_b64 v[FB+ni*2:FB+ni*2+1], v12 offset:((ks*FN+ni)*256) + .set ni, ni+1 + .endr + .set mi, 0 + .rept FM + ds_load_b64 v[FA+mi*2:FA+mi*2+1], v13 offset:((ks*G*FM+mi)*256) + .set mi, mi+1 + .endr + s_wait_dscnt 0x0 + .set mi, 0 + .rept FM + .set ni, 0 + .rept FN + v_wmma_f32_16x16x16_fp8_fp8 v[ACC+(mi*FN+ni)*8:ACC+(mi*FN+ni)*8+7], v[FA+mi*2:FA+mi*2+1], v[FB+ni*2:FB+ni*2+1], v[ACC+(mi*FN+ni)*8:ACC+(mi*FN+ni)*8+7] + .set ni, ni+1 + .endr + .set mi, mi+1 + .endr + .set ks, ks+1 + .endr + phase_stamp s81 // close WMMA (LDS frag loads + v_wmma over the segment) + // flush: C base = C + ti-term + r*(FM*FN*1024) ; per (frag,elem) atomic-add one fp32 (vaddr v10=lane*32) + s_mul_i32 s39, s33, (FM*FN*1024) + s_add_u32 s39, s38, s39 + s_add_u32 s28, s6, s39 + s_addc_u32 s29, s7, 0 +.if NOCFLUSH == 0 + .set frag, 0 + .rept FM*FN + .set e, 0 + .rept 8 + .if CSTORE + global_store_b32 v10, v[ACC+frag*8+e], s[28:29] offset:(frag*1024 + e*4) scope:SCOPE_DEV // probe: non-atomic, same count + .else + global_atomic_add_f32 v10, v[ACC+frag*8+e], s[28:29] offset:(frag*1024 + e*4) scope:SCOPE_DEV + .endif + .set e, e+1 + .endr + .set frag, frag+1 + .endr +.endif + s_wait_storecnt 0x0 // atomic-adds READ ACC -> must drain before shrink frees ACC + phase_stamp s82 // close FLUSH (global_atomic_add_f32 C reduction + drain) +.if DYNVGPR +.Lcompute_shrink: + s_alloc_vgpr 32 // shrink (SCC-retry guarded) + s_cbranch_scc0 .Lcompute_shrink +.endif + phase_stamp s83 // close SHRINK (s_alloc_vgpr 112->32) + // rowblk r computed+flushed: bump SL_RBDONE; the wave whose inc reaches G is the CLOSER and advances + // the drain frontier. old==G-1 detects the closer atomically (exactly one wave). + s_add_u32 s45, s48, SL_RBDONE + lds_fetch_add_r s44, s45, 1 // s44 = old SL_RBDONE[DRAIN_IDX] + s_cmp_eq_u32 s44, (G-1) + s_cbranch_scc0 .Lcompute_claim // not the closer -> keep claiming (next rowblk / over-claim) + s_xor_b32 s47, s46, 1 // closer: DRAIN_IDX -> other slot (lock-free frontier advance) + lds_put DRAIN_IDX_OFF, s47 + s_branch .Lcompute_claim +.Lcompute_drained: + s_branch .Lcompute_follow // this slot drained -> re-check drain frontier / terminal + +.if DSWS2_CONV +// ============================================================================================ +// Role RE-DISPATCH trampoline (Task 3): scalar-only. Reads the role register s59 (flipped by +// conv_apply on a role conversion, else unchanged) and branches to the matching role's +// per-epoch _follow loop. RE-DISPATCH ONLY -- reached from a per-super-tile _quiesce bail, +// never from first-time entry (the seed arms fall into the full role entry so _alloc/_init +// run once; see the seed block). Lands on _follow (NOT _alloc/_init): the wave's VGPR +// footprint is already correct (conv_apply set it) and INIT already ran -- re-entering _alloc +// would wrongly resize, and re-running _init would reset s35=0, breaking the "wait for the +// NEXT epoch" contract (the wave would re-process the current super-tile). INITFLAG is written +// once (0xACED) and never cleared, so it is the s35 reset -- not INITFLAG -- that _follow preserves. +// s35 (last-seen-epoch) is untouched here, which is what makes a re-dispatched wave wait for +// the NEXT epoch at the top of its new role's _follow loop. Scalar-only (s59 read + s_branch) -> +// adds ZERO OOR/VGPR exposure. +// ============================================================================================ +.Ldispatch: + s_cmp_eq_u32 s59, NCOMP_SLOT + s_cbranch_scc1 .Lcompute_follow + s_cmp_eq_u32 s59, NAFEED_SLOT + s_cbranch_scc1 .Lafeed_follow + s_branch .Lbfeed_follow +.endif + +// ---- A7 role-agnostic terminal (followers): retire. (Claimer retires via .Lclaimer_terminal.) ---- +.Lretire: + tfspan max, 12 // TFPROBE: every follower stamps occ[3] = max exit tick (wall-span end) + alllive_dec // TRACE: follower exit -> --live (peak-concurrent occupancy) + s_endpgm +.else + s_endpgm // DSWS2=0 has no v2 body (this file is always built DSWS2=1) +.endif + .size occ_kernel, .-occ_kernel + +// ---- RGADESC: analysis-only descriptor so `rga -s bin --co` can enumerate + livereg this kernel. +// vgpr 256 ceiling so livereg reports the true s_alloc-grown peak-live. NOT emitted for the PM4 .bin. ---- +.if RGADESC +.amdhsa_kernel occ_kernel + .amdhsa_next_free_vgpr 256 + .amdhsa_next_free_sgpr 72 // body uses up to s69 (s66=n_kseg s67=mask s68=shift s69=chunkHi, FIX 1) + .amdhsa_group_segment_fixed_size 65536 // FIX 1a ring: D=2 needs 33024B (RGA-analysis descriptor only) + .amdhsa_user_sgpr_count 15 // FIX 1(g): v2 contract now s0..s14 only (n_kseg/TOTAL_super/ + // magic_kseg dropped -- derived in-kernel / memory-carried) + .amdhsa_wavefront_size32 1 +.end_amdhsa_kernel +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: occ_kernel + .symbol: occ_kernel.kd + .kernarg_segment_size: 60 + .kernarg_segment_align: 8 + .group_segment_fixed_size: 65536 + .private_segment_fixed_size: 0 + .wavefront_size: 32 + .sgpr_count: 72 + .vgpr_count: 256 + .max_flat_workgroup_size: 256 + .args: [] +.end_amdgpu_metadata +.endif diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_grind.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_grind.s new file mode 100644 index 000000000000..e6c8ac69726b --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_grind.s @@ -0,0 +1,341 @@ +// occ_kernel_grind.s (gfx1201, wave32) -- MAD-305 "GRIND" CONTROL fp8 WMMA GEMM. +// +// PURPOSE (control experiment, 2026-07-03): the smallest NON-split-K, ONE-OUTPUT-TILE-PER-WORKGROUP +// fp8 GEMM. It deliberately does NOT use the dyn-VGPR / split-K "moat" of occ_kernel_dsws.s. One wave +// (one workgroup) OWNS one output tile (mblk,tcol), accumulates the FULL K dimension with the fp32 +// accumulators live in VGPR across the whole K loop, and writes C EXACTLY ONCE with a plain +// global_store (NO C atomics, NO cross-WG reduction, NO ksi split). This is the baseline against which +// the split-K kernel's 2.1 TF is measured: if grind beats it, split-K may not be worth defending. +// +// STRUCTURE (reused idioms from occ_kernel_dsws.s, machinery stripped): +// - WMMA microkernel: v_wmma_f32_16x16x16_fp8_fp8 (identical fragment shapes, FM x FN frags). +// - A/B staging: global_load(_tr)_b64 -> ds_store -> ds_load -> WMMA, streaming K through LDS in +// KCHUNK-k16-step chunks (single-buffer; one wave so NO barrier -- s_wait_dscnt suffices). +// - Tile hand-out: a global atomic claim counter (occ[20]) -- a persistent grid-stride over output +// tiles (raw-PM4 DISPATCH_DIRECT does NOT deliver TGID_X on gfx1201, so a claim counter is the +// proven work-distribution mechanism; see occ_kernel_wggemm.s deprecation note). +// STRIPPED vs dsws: the (mblk,tcol,ksi) super-tile pool, the atomic-add C flush, the split-K ksi +// dimension, the claimer/feed/compute role machinery, the s_alloc_vgpr grow/shrink churn. +// +// ============================================================================================ +// KERNARG CONTRACT (USER_SGPR=15, s0..s14; hardware-preloaded user SGPRs). BYTE-IDENTICAL to the +// occ_kernel_dsws.s v2 contract EXCEPT s11=TOTAL is the *grind* tile count (finer M tiling: MTL = +// M/(FM*16), not M/(G*FM*16)) and there is no ksi. The host may reuse the dsws2 userdata packing: +// s0:s1 = occ buffer base (>=0x1000B; host zero-inits; occ[20]=claim counter, host sets to `base`) +// s2:s3 = A base (fp8 e4m3, row-major MxK, 1 byte/elem) +// s4:s5 = Bshuf base (shuffled-B layout consumed by global_load_tr_b64; same as dsws) +// s6:s7 = C base (fp32, FRAGMENT-TILED: tile ti -> C + ti*(FM*FN*1024); plain store, NO memset +// required for correctness, but zeroing is harmless) +// s8 = KT (total K16-steps for the whole matrix = K/16; MUST be a multiple of KCHUNK) +// s9 = K (bytes per A-row = K, fp8 1 byte/elem) +// s10 = NT*256 (B-saddr advance per K16-step) +// s11 = TOTAL (grind tile count = MTL*NTL, MTL=M/(FM*16); ALSO the claim terminal / chunkHi) +// s12 = magic(ceil(2^32/NTL)) (unsigned-div magic for /NTL -> mblk/tcol decode) +// s13 = NTL (number of N tile-columns = N/(FN*16)) +// s14 = FN*256 (B-saddr stride per N-frag) +// (TGID_X lands in s15 -- UNUSED.) +// occ layout (occ base = s0:s1; host zero-inits, sets occ[20]=base): +// occ[0] (byte 0) = live counter (lane0 +1 at entry, -1 at exit; host polls ==0) +// occ[1] (byte 4) = maxlive (bookkeeping) +// occ[2] (byte 8) = min start realtime clock ; occ[3] (byte 12) = max end realtime clock +// occ[20] (byte 20) = GLOBAL tile claim counter (host sets to `base`; grind atomic-adds 1/tile) +// +// G and SEGK/split-K are GONE. KCHUNK (k16-steps staged per LDS chunk) is a COMPILE-TIME defsym. +// ============================================================================================ + +.amdgcn_target "amdgcn-amd-amdhsa--gfx1201" + +// ---- tile defsyms ---- +.ifndef FM + .set FM, 2 // per-wave M-frags (M tile = FM*16 rows) +.endif +.ifndef FN + .set FN, 4 // per-wave N-frags (N tile = FN*16 cols) +.endif +.ifndef KCHUNK + .set KCHUNK, 4 // k16-steps staged through LDS per chunk (power of two) +.endif +.ifndef RGADESC + .set RGADESC, 0 // 1 = emit analysis-only AMDHSA descriptor for RGA livereg +.endif + +// log2(KCHUNK) so n_chunks = KT >> KCH_SHIFT (KCHUNK is a power of two by construction). +.if KCHUNK == 1 + .set KCH_SHIFT, 0 +.elseif KCHUNK == 2 + .set KCH_SHIFT, 1 +.elseif KCHUNK == 4 + .set KCH_SHIFT, 2 +.elseif KCHUNK == 8 + .set KCH_SHIFT, 3 +.else + .error "KCHUNK must be a power of two in {1,2,4,8}" +.endif + +// ---- LDS layout (single-buffer; A then B; frag = 256B, lane*8 base) ---- +.set ALDS_OFF, 0 // A frag (ks,mi) at ALDS_OFF + (ks*FM + mi)*256 +.set ALDS_BYTES, (KCHUNK*FM*256) +.set BLDS_OFF, ALDS_BYTES // B frag (ks,ni) at BLDS_OFF + (ks*FN + ni)*256 +.set BLDS_BYTES, (KCHUNK*FN*256) +.set LDS_TOTAL, (BLDS_OFF + BLDS_BYTES) // = KCHUNK*(FM+FN)*256 (=6144 @ KCHUNK=4,FM=2,FN=4) +.if LDS_TOTAL > 32768 + .error "grind LDS layout exceeds 32768B group segment" +.endif + +// ---- VGPR layout (STATIC alloc; full footprint live for the whole K loop -- this is the control's +// high-VGPR-duty structure by design). Accumulators live from pre-K-loop zero through the single C store. +.set STG, 16 // staging pair (v16:v17) for one in-flight A/B frag +.set ACC, 32 // FM*FN fp32 accumulators x 8 (v32..) +.set FA, (ACC + 8*FM*FN) // compute A frags (from LDS): FM x 2 (=v96 @ 2x4) +.set FB, (FA + 2*FM) // compute B frags (from LDS): FN x 2 (=v100 @ 2x4) +.set NFV, (FB + 2*FN) // next-free vgpr (=v108 @ 2x4 -> ~112 alloc, under 128 cap) + + .text + .globl occ_kernel + .p2align 8 + .type occ_kernel,@function +occ_kernel: + // ---- per-thread identity (v0=tid hardware-preloaded); one wave -> wid=0, lane=tid ---- + v_and_b32 v2, 31, v0 // lane = tid & 31 + v_and_b32 v6, 15, v0 // lane & 15 (A row within frag) + v_mov_b32 v4, 0 // vaddr-0 for atomic claim (addr = occ_base + offset) + // ---- per-lane address constants (dsws-identical fragment maps) ---- + v_mul_lo_u32 v8, v6, s9 // (lane&15)*K + v_bfe_u32 v7, v0, 4, 1 // colhi = (tid>>4)&1 + v_lshlrev_b32 v7, 3, v7 // colhi*8 + v_add_nc_u32 v8, v8, v7 // v8 = A vaddr = (lane&15)*K + colhi*8 + v_lshlrev_b32 v9, 3, v2 // v9 = B/LDS vaddr = lane*8 + v_lshlrev_b32 v10, 5, v2 // v10 = C store vaddr = lane*32 + + // ---- live++ : lane0 occ[0] += 1 ; maxlive book ---- + s_mov_b32 s16, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lafter_live + v_mov_b32 v5, 1 + global_atomic_add_u32 v11, v4, v5, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV // v11 = old live + s_wait_loadcnt 0x0 + v_add_nc_u32 v11, v11, 1 + global_atomic_max_u32 v4, v11, s[0:1] offset:4 scope:SCOPE_DEV // maxlive +.Lafter_live: + s_mov_b32 exec_lo, s16 + + // ---- timer t0 (min start) ---- + s_sendmsg_rtn_b64 s[36:37], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + s_mov_b32 s16, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lafter_t0 + v_mov_b32 v5, s36 + global_atomic_min_u32 v4, v5, s[0:1] offset:8 scope:SCOPE_DEV +.Lafter_t0: + s_mov_b32 exec_lo, s16 + + // n_chunks = KT >> KCH_SHIFT (KT = s8) + s_lshr_b32 s33, s8, KCH_SHIFT + // A mi-stride = 16*K bytes + s_lshl_b32 s30, s9, 4 + +// ============================================================================================ +// CLAIM LOOP: ti = atomicAdd(occ[20], 1) ; if ti >= TOTAL(s11) done ; compute tile ti full-K ; store C. +// ============================================================================================ +.Lclaim: + s_mov_b32 s16, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lclaim_bcast + v_mov_b32 v5, 1 + global_atomic_add_u32 v11, v4, v5, s[0:1] offset:20 th:TH_ATOMIC_RETURN scope:SCOPE_DEV // v11=old ti + s_wait_loadcnt 0x0 +.Lclaim_bcast: + s_mov_b32 exec_lo, s16 + v_readfirstlane_b32 s20, v11 // ti (uniform) + s_cmp_ge_u32 s20, s11 // ti >= TOTAL -> drained + s_cbranch_scc1 .Ldone + + // ---- decode: mblk = ti/NTL (magic) ; tcol = ti - mblk*NTL ---- + s_mul_hi_u32 s21, s20, s12 // mblk + s_mul_i32 s22, s21, s13 // mblk*NTL + s_sub_u32 s22, s20, s22 // tcol = ti - mblk*NTL + + // ---- tile scalar bases (k=0) ---- + // A: A + (mblk*FM*16)*K + s_mul_i32 s38, s21, (FM*16) // row_start = mblk*FM*16 + s_mul_i32 s34, s38, s9 // low(row_start*K) + s_mul_hi_u32 s35, s38, s9 // high + s_add_u32 s34, s2, s34 + s_addc_u32 s35, s3, s35 // s[34:35] = A tile base (k=0), advances per chunk + // B: Bshuf + tcol*(FN*256) + s_mul_i32 s36, s22, s14 // low(tcol*FN*256) + s_mul_hi_u32 s37, s22, s14 // high + s_add_u32 s36, s4, s36 + s_addc_u32 s37, s5, s37 // s[36:37] = B tile base (k=0), advances per chunk + // C: C + ti*(FM*FN*1024) + s_mul_i32 s28, s20, (FM*FN*1024) // low + s_mul_hi_u32 s29, s20, (FM*FN*1024) // high + s_add_u32 s28, s6, s28 + s_addc_u32 s29, s7, s29 // s[28:29] = C tile base + + // ---- zero the FM*FN fp32 accumulators (live across the whole K loop) ---- + .set idx, 0 + .rept FM*FN + v_mov_b32 v[ACC+idx*8+0], 0 + v_mov_b32 v[ACC+idx*8+1], 0 + v_mov_b32 v[ACC+idx*8+2], 0 + v_mov_b32 v[ACC+idx*8+3], 0 + v_mov_b32 v[ACC+idx*8+4], 0 + v_mov_b32 v[ACC+idx*8+5], 0 + v_mov_b32 v[ACC+idx*8+6], 0 + v_mov_b32 v[ACC+idx*8+7], 0 + .set idx, idx+1 + .endr + + s_mov_b32 s31, 0 // chunk = 0 +// ---- K-CHUNK LOOP: stage KCHUNK k16-steps of A/B into LDS, then WMMA-accumulate ---- +.Lkloop: + s_cmp_ge_u32 s31, s33 + s_cbranch_scc1 .Lkdone + + // ===== STAGE A into LDS: A frag (ks,mi) ; global_load_b64(v8, base + ks*16) -> ds_store ===== + .set mi, 0 + .rept FM + // sAmi = A_chunk_base + mi*(16*K) + .if mi == 0 + s_mov_b32 s44, s34 + s_mov_b32 s45, s35 + .else + s_add_u32 s44, s44, s30 + s_addc_u32 s45, s45, 0 + .endif + .set ks, 0 + .rept KCHUNK + global_load_b64 v[STG:STG+1], v8, s[44:45] offset:(ks*16) + s_wait_loadcnt 0x0 + ds_store_b64 v9, v[STG:STG+1] offset:(ALDS_OFF + (ks*FM + mi)*256) + s_wait_dscnt 0x0 + .set ks, ks+1 + .endr + .set mi, mi+1 + .endr + + // ===== STAGE B into LDS: B frag (ks,ni) ; global_load_tr_b64(v9, base + ks*s10 + ni*256) -> ds_store + s_mov_b32 s42, s36 // sBc = B_chunk_base (ks=0) + s_mov_b32 s43, s37 + .set ks, 0 + .rept KCHUNK + s_mov_b32 s40, s42 // sf = sBc + s_mov_b32 s41, s43 + .set ni, 0 + .rept FN + global_load_tr_b64 v[STG:STG+1], v9, s[40:41] + s_wait_loadcnt 0x0 + ds_store_b64 v9, v[STG:STG+1] offset:(BLDS_OFF + (ks*FN + ni)*256) + s_wait_dscnt 0x0 + s_add_u32 s40, s40, 256 // += frag stride (FN dir) + s_addc_u32 s41, s41, 0 + .set ni, ni+1 + .endr + s_add_u32 s42, s42, s10 // += k16-step stride (NT*256) + s_addc_u32 s43, s43, 0 + .set ks, ks+1 + .endr + + // ===== COMPUTE: read resident A/B from LDS, WMMA-accumulate into live ACC ===== + .set ks, 0 + .rept KCHUNK + .set ni, 0 + .rept FN + ds_load_b64 v[FB+ni*2:FB+ni*2+1], v9 offset:(BLDS_OFF + (ks*FN + ni)*256) + .set ni, ni+1 + .endr + .set mi, 0 + .rept FM + ds_load_b64 v[FA+mi*2:FA+mi*2+1], v9 offset:(ALDS_OFF + (ks*FM + mi)*256) + .set mi, mi+1 + .endr + s_wait_dscnt 0x0 + .set mi, 0 + .rept FM + .set ni, 0 + .rept FN + v_wmma_f32_16x16x16_fp8_fp8 v[ACC+(mi*FN+ni)*8:ACC+(mi*FN+ni)*8+7], v[FA+mi*2:FA+mi*2+1], v[FB+ni*2:FB+ni*2+1], v[ACC+(mi*FN+ni)*8:ACC+(mi*FN+ni)*8+7] + .set ni, ni+1 + .endr + .set mi, mi+1 + .endr + .set ks, ks+1 + .endr + + // advance chunk bases: A += KCHUNK*16 bytes ; B += KCHUNK*(NT*256) + s_add_u32 s34, s34, (KCHUNK*16) + s_addc_u32 s35, s35, 0 + s_mul_i32 s46, s10, KCHUNK + s_add_u32 s36, s36, s46 + s_addc_u32 s37, s37, 0 + s_add_u32 s31, s31, 1 // chunk++ + s_branch .Lkloop +.Lkdone: + + // ===== C STORE (once, plain non-atomic): C_tile + frag*1024 + {0,16} ; vaddr v10=lane*32 ===== + .set frag, 0 + .rept FM*FN + global_store_b128 v10, v[ACC+frag*8+0:ACC+frag*8+3], s[28:29] offset:(frag*1024 + 0) + global_store_b128 v10, v[ACC+frag*8+4:ACC+frag*8+7], s[28:29] offset:(frag*1024 + 16) + .set frag, frag+1 + .endr + s_wait_storecnt 0x0 // stores must be issued before we reuse ACC next claim + s_branch .Lclaim + +.Ldone: + s_wait_storecnt 0x0 + // ---- timer t1 (max end) ---- + s_sendmsg_rtn_b64 s[36:37], sendmsg(MSG_RTN_GET_REALTIME) + s_wait_kmcnt 0x0 + s_mov_b32 s16, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lafter_t1 + v_mov_b32 v5, s36 + global_atomic_max_u32 v4, v5, s[0:1] offset:12 scope:SCOPE_DEV +.Lafter_t1: + s_mov_b32 exec_lo, s16 + // ---- live-- : lane0 occ[0] -= 1 ---- + s_mov_b32 s16, exec_lo + v_cmp_eq_u32 vcc_lo, 0, v2 + s_and_b32 exec_lo, exec_lo, vcc_lo + s_cbranch_execz .Lafter_dec + v_mov_b32 v5, -1 + global_atomic_add_u32 v4, v5, s[0:1] scope:SCOPE_DEV +.Lafter_dec: + s_mov_b32 exec_lo, s16 + s_endpgm + .size occ_kernel, .-occ_kernel + +// ---- RGADESC: analysis-only AMDHSA descriptor so `rga -s bin --co` can enumerate + livereg. NOT +// emitted for the PM4 .bin (the host provides RSRC1/RSRC2 directly). ---- +.if RGADESC +.amdhsa_kernel occ_kernel + .amdhsa_next_free_vgpr NFV + .amdhsa_next_free_sgpr 50 + .amdhsa_group_segment_fixed_size LDS_TOTAL + .amdhsa_user_sgpr_count 15 + .amdhsa_wavefront_size32 1 +.end_amdhsa_kernel +.amdgpu_metadata +--- +amdhsa.version: [ 1, 2 ] +amdhsa.kernels: + - .name: occ_kernel + .symbol: occ_kernel.kd + .kernarg_segment_size: 60 + .kernarg_segment_align: 8 + .group_segment_fixed_size: 6144 + .private_segment_fixed_size: 0 + .wavefront_size: 32 + .sgpr_count: 50 + .vgpr_count: 112 + .max_flat_workgroup_size: 32 + .args: [] +.end_amdgpu_metadata +.endif From 7ec9b6a4da78f0dc5709c1ea0b8ae641be3d86ae Mon Sep 17 00:00:00 2001 From: mad-lab-kbando <69054773+kmbandy@users.noreply.github.com> Date: Sun, 5 Jul 2026 20:32:50 -0400 Subject: [PATCH 16/19] DSWS flow: throttle deadman message-bus read off the idle-coast hot path Root cause of the W=16 chunked-dispatch brick + wall instability (measured 2026-07-05): deadman_check does an s_sendmsg_rtn RTC read (an SQ-front-end op) at every loop-head. With 24 WGs x 16 waves = 384 waves and only 2 tiles/chunk, ~382 idle waves spin the coast loop and hammer that message bus ~41k times per chunk. That front-end spam (a) destabilized the coast wall -- identical STAGINSTR work measured 0.324s vs 1.996s across two chunks -- and (b) starved the compositor's SQC(inst) instruction fetch -> GCVM_L2 page fault -> MODE1. Fix: throttle the RTC read to once every DEADMAN_EVERY (=64) iterations via a per-wave counter in s71 (the high-RTC reg, unused at TRACE=0; deadman_check only reads s70). Force-retire slack = DEADMAN_EVERY iters (negligible vs 0.5s deadline). Result (12-chunk W=16, same config as the bricking run): all chunks complete, wall flat at 0.325s (6x variance gone), oracle CLEAN bad=0 max_rel=0, no brick. grow-fail=0 throughout also disproves the VGPR-starvation-livelock hypothesis. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_0132aDSBLwusCJ4KzHQTnvdu --- .../spike/dvgpr_occ/occ_kernel_dsws_flow.s | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s index d1c4cbe6e9d1..89849bfdcd11 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s @@ -517,20 +517,33 @@ .Lcasadv_skip\@: s_mov_b32 exec_lo, s49 .endm -// ---- DEADMAN watchdog: s[70:71] = this wave's start RTC; deadman_check force-retires past the deadline ---- +// ---- DEADMAN watchdog: s70 = this wave's start RTC (low 32b); s71 = throttle counter (repurposed high-RTC +// reg, which is unused at TRACE=0 -- deadman_check only reads s70). The message-bus RTC read (s_sendmsg_rtn) +// is an SQ-front-end op; hundreds of idle COAST waves hitting it EVERY loop iteration spam the front-end, +// starving the compositor's SQC(inst) fetch (2026-07-05 MODE1 brick) AND destabilizing the coast wall +// (identical STAGINSTR work measured 0.32s vs 2.0s). THROTTLE: only read the RTC every DEADMAN_EVERY iters. ---- +.ifndef DEADMAN_EVERY + .set DEADMAN_EVERY, 64 // message-bus RTC-read cadence (in loop iters); force-retire slack = DEADMAN_EVERY iters +.endif .macro deadman_stamp // stamp start RTC (low 32b in s70) once at entry .if DEADMAN s_sendmsg_rtn_b64 s[70:71], sendmsg(MSG_RTN_GET_REALTIME) s_wait_kmcnt 0x0 + s_mov_b32 s71, 0 // repurpose the (TRACE=0-unused) high-RTC reg as the throttle counter .endif .endm .macro deadman_check // if alive > DEADMAN_TICKS -> clean force-retire (no wedge) .if DEADMAN + s_add_u32 s71, s71, 1 // THROTTLE: touch the SQ-front-end message bus only every + s_cmp_ge_u32 s71, DEADMAN_EVERY // DEADMAN_EVERY iters (idle coast waves would else spam it -> brick) + s_cbranch_scc0 .Ldm_skip\@ + s_mov_b32 s71, 0 s_sendmsg_rtn_b64 s[62:63], sendmsg(MSG_RTN_GET_REALTIME) s_wait_kmcnt 0x0 s_sub_u32 s62, s62, s70 // elapsed = now_lo - start_lo (u32 wrap-safe; deadline << 42s) s_cmp_ge_u32 s62, DEADMAN_TICKS s_cbranch_scc1 .Lflow_retire +.Ldm_skip\@: .endif .endm // lds_put_r (RUNTIME-addr write) is also defined inside the .if DSWS2_CONV||DSWS2_ENVELOPE block below; From 87b29b9f28acea379d09c73b1404dddc35a8fc2b Mon Sep 17 00:00:00 2001 From: mad-lab-kbando <69054773+kmbandy@users.noreply.github.com> Date: Mon, 6 Jul 2026 08:23:48 -0400 Subject: [PATCH 17/19] chore(dsws): ignore dvgpr-spike build artifacts + scratch, drop stray '=' file The rdna4_fp8_gemm/dvgpr_occ spike accumulated local-only clutter (build dirs, compiled sim/probe binaries, .err/.co/.amdisa dumps, benchmark result logs, root-level results.db/json, a generated phase_profile.html, and scratch sources). Add scoped .gitignore rules so `git status` stays clean and these never leak into commits. Patterns are anchored per-path and use explicit prefixes so tracked CMakeLists.txt / TRITON_PIN.txt / *_DESIGN.md are untouched. Also removes a 0-byte file literally named '=' left by a redirect typo. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_0132aDSBLwusCJ4KzHQTnvdu --- .gitignore | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.gitignore b/.gitignore index 966acb97a854..a2b2735d7309 100644 --- a/.gitignore +++ b/.gitignore @@ -203,3 +203,39 @@ Makefile tests/perf-baseline/**/*.log tests/perf-baseline/**/rocprof-*/ tests/perf-baseline/calibration-logs/ + +# --- rdna4_fp8_gemm / dvgpr spike: local build artifacts, scratch + result logs (2026-07) --- +# Tracked source (.s/.cpp/.cu/.cuh, CMakeLists.txt, *_DESIGN.md, etc.) is added explicitly; +# everything below is local-only and must stay out of the tree. +# repo-root run dumps +/results.db +/results.json +/results.sysinfo.txt +# scoped scratch doc + build output dirs +ggml/src/ggml-cuda/aiter-integration/vulkan-port-scope.md +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/out/ +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/bench/gemm_fp8_levers +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_pm4/ref_gfx12/ +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build/ +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/rga_out/ +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/rga_ml8/ +# compiler/assembler dumps, disassembly, objects, scratch copies, generated viz (dvgpr_occ) +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/*.err +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/*.amdisa +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/*.co +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/*.new +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/phase_profile.html +# result / benchmark logs (explicit prefixes so tracked CMakeLists.txt / TRITON_PIN.txt survive) +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/dynfull_*.txt +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/fairrace_*.txt +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/mbml8_*.txt +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/nofeed_*.txt +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/o_pf_yield_test.txt +# compiled sim/probe binaries + their scratch sources +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/lean_bounds_sim +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/lean_bounds_sim.cpp +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/simdprobe +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/simdprobe.hip +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/rocprof_smoke +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/rocprof_smoke.hip +ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/t_ctrl From c33d06539396ddc53b5fdb7081ae4477515f90de Mon Sep 17 00:00:00 2001 From: mad-lab-kbando <69054773+kmbandy@users.noreply.github.com> Date: Mon, 6 Jul 2026 08:23:48 -0400 Subject: [PATCH 18/19] docs(dsws): G_resident design v2 (burst-owns-a-ksi-run) + WOFLUSH lever (gated off) Step-2 design doc rewritten to v2, folding the council (Fable) review of v1: - H1 (silent reduction race) fixed: zero-init banks + all-ds_add => order- independent merge, the precondition for J>1 and POOL_N>1. - H3 (LDS arithmetic) fixed: real +256 header; measured OPSTRIDE 8192@SEGK32, 4096@SEGK16 from Run-4 LDS=57600; both feasibility tables corrected. - H5 (group/tile boundary bank collision) dodged: Path A keeps whole-tile groups (G_resident=G); Path B (partition) demoted to an optional post-bind throughput lever with drain-before-admit spelled out. - Reframes concurrent-fat as count x duty-cycle, with burst length J as the free (no-LDS) knob on the duty axis; adds the Gate-3 honesty check (does TF actually move with J, or is the cap structural?) and the occ[58] FATMAX instrumentation TODO. Kernel/build changes are the WOFLUSH (atomic-flush-to-C) experiment lever, flag-gated OFF: WOFLUSH=0 is byte-identical to the shipped throttle bin. Kept in-tree as a rejected-but-documented alternative to the on-chip reduction. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_0132aDSBLwusCJ4KzHQTnvdu --- .../spike/dvgpr_occ/DSWS_GRESIDENT_DESIGN.md | 201 ++++++++++++++++++ .../spike/dvgpr_occ/build_flow.sh | 2 +- .../spike/dvgpr_occ/occ_kernel_dsws_flow.s | 35 +++ 3 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/DSWS_GRESIDENT_DESIGN.md diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/DSWS_GRESIDENT_DESIGN.md b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/DSWS_GRESIDENT_DESIGN.md new file mode 100644 index 000000000000..755d0a5fcb10 --- /dev/null +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/DSWS_GRESIDENT_DESIGN.md @@ -0,0 +1,201 @@ +# DSWS — burst-owns-a-ksi-run: engaging the dyn-VGPR stagger without a flush + +**Date:** 2026-07-06 **Kernel:** `occ_kernel_dsws_flow.s` (gfx1201 / RDNA4 / R9700, wave32, raw PM4) +**Status:** design **v2** — folds council review (Fable). Supersedes the shelved `POOL_N=2` framing, the +rejected `WOFLUSH` (atomic-flush) lever, and v1's *count-only* concurrent-fat model. + +> **What changed v1 → v2 (why read this):** v1 modeled concurrent-fat as a pure *count* problem +> (`POOL_N·G_resident`) and proposed shrinking banks (`G_resident < G`) to afford a high `POOL_N`. Fable's +> review found three defects in that: **H1** a silent correctness race in the reduction, **H3** an +> off-by-256 that made every "feasible 64KB" row actually overflow, **H5** a group/tile-boundary bank +> collision the partition scheme creates. v2 adopts Fable's **burst-owns-a-ksi-run**: it makes the +> reduction commutative (kills H1), keeps **whole-tile groups** (`G_resident=G`, no boundary → kills H5), +> and reframes concurrent-fat as **count × duty-cycle** with **burst length** as a free knob on the +> duty axis that costs *no* LDS. Bank-shrink (`G_resident throughput lever behind a proven bind. + +--- + +## 1. The problem (measured, not inferred) + +The dyn-VGPR "traveling-peak" stagger only does work if enough compute waves are **simultaneously fat** +that the per-SIMD VGPR pool (~1536 VGPR; a fat wave = NFV≈112 → ~13 fat/SIMD) actually **binds** → +`s_alloc_vgpr` starts failing → `grow-fail > 0`. + +**Run 4 (2026-07-05):** full grid (128 WGs, saturated 256-tile 3072×1024×2048 shape), `grow-fail = 0`, +98.8% coast. The pool never binds. Root cause is the LDS accumulator: + +- Write-once-C keeps each rowblk's fp32 C sub-tile in an **8KB LDS bank**, persisting across all + `n_kseg` split-K segments; `G=6` rowblks → **48KB** of banks. +- LDS/WG = 57,600B → only **~1 WG/CU** resident (gfx1201 CU ≈ 64KB LDS) → few fat compute waves/SIMD. + Nothing forces waves to compete for VGPR. + +## 2. The trilemma (why this is hard) — and the escape + +The K-reduction running-sum must live *somewhere* on-chip to be cheap, and to stagger (short fat bursts) +the fat accumulator must be **evicted between bursts** — a dyn-VGPR *shrink* is only legal if its contents +are safe first. v1 framed the three eviction targets as a hard trilemma: + +| Running-sum home | Cost | +|---|---| +| **VGPR, whole K** (grind) | square wave — no trapezoid, no stagger; wave held fat entire K-loop | +| **LDS bank** (write-once) | 48KB persistent → caps occupancy at 1 WG/CU → **pool never binds** | +| **Global atomic** (WOFLUSH) | RMW round-trip + same-cell serialization (the 26–51% FLUSH we deleted) | + +**The escape (Fable):** the trilemma is false because "VGPR whole-K" and "LDS bank every ksi" are the two +*endpoints* of a knob, not the only choices. Let a fat burst own a **run of J consecutive ksi** for one +rowblk: accumulate those J ksi in the **VGPR ACC** (no LDS touch between them), then do **one** `ds_add` +merge of that partial into the rowblk's LDS bank. `J = n_kseg` is the whole-K endpoint (square wave); +`J = 1` is the write-once endpoint (bank every ksi). **J is a continuous duty-cycle knob between them**, +and it costs no LDS. The bank still persists across bursts (it holds the running sum of merged runs), but +it is *touched* only once per burst, not once per ksi. + +WOFLUSH stays rejected: it re-introduces the exact atomic-flush wait write-once removed. + +## 3. The insight — concurrent-fat is *count × duty-cycle*, not count + +v1's error: it treated instantaneous-fat as `min(POOL_N·G_resident, WAVES)` — a pure **count**. But a +wave is only fat *while it grinds*; between bursts it goes thin (fetch operands, merge, re-claim). So: + +> **instantaneous-fat ≈ (waves resident) × (fraction of time each is fat) = count × duty-cycle.** + +To bind the pool you need instantaneous-fat/SIMD ≳ 13–14. Two independent levers get you there: + +1. **Count** — more concurrent `(rowblk, ksi-run)` claims live at once (`POOL_N`, `G_resident`, `WAVES`). + Costs LDS (operand slots + banks). This was v1's only lever. +2. **Duty-cycle** — longer fat bursts (**larger J**) → each resident wave spends more of its life fat → + more are simultaneously fat. Costs **no LDS**. This is the lever v1 was blind to. + +Burst length is the free knob on the duty axis. That is what lets us keep **whole-tile groups** +(`G_resident = G`, 48KB banks, 1 WG/CU — the config Run 4 *already* had) and still reach the bind, by +cranking J instead of shrinking banks. + +**The H1 fix that makes any of this legal:** the reduction must be **order-independent**. Today it is not — +`ksi==0` does `ds_store` (init the bank), `ksi>0` does `ds_add`; with concurrent runs of the same rowblk +the init can land *after* an add → silent wrong C. v2: **zero-initialize the banks up front, then *every* +merge (all ksi, all runs) is `ds_add`.** `ds_add` is atomic per-op in LDS, so concurrent merges into one +bank serialize correctly and the result is order-independent. (fp32 add is non-associative, so ordering +still perturbs rounding — but write-once *already* has non-deterministic `ds_add` order across ksi and +passes oracle `bad=0` within tolerance; v2 adds no new nondeterminism class.) + +## 4. The binding model (corrected arithmetic — H3 fixed) + +Per-SIMD fat waves ≈ `instantaneous-fat / 2` (a WG spans 2 SIMDs). Bind ⇒ need **≳ 13–14 fat/SIMD**. + +**LDS constraint (the H3 fix):** `256 + POOL_N·OPSTRIDE + G_resident·8192 ≤ 65536`. +Measured OPSTRIDE: **8192 @ SEGK=32, 4096 @ SEGK=16** (from Run-4 `LDS=57600 = 256+8192+6·8192`). +*(v1 omitted the +256 header; every "64KB" row there actually summed to 65792 = 256 **over** the limit and +the kernel's `.error` would have fired.)* + +### Path A — whole-tile groups (`G_resident = G = 6`), the correctness+bind proof + +Dodges the boundary problem (§7) entirely. Banks = 48KB fixed → **1 WG/CU**. Binding rides on **duty-cycle +(J)**, since the count ceiling is low: + +| SEGK | OPSTRIDE | POOL_N max | banks | LDS (+256) | count = POOL_N·G | +|---|---|---|---|---|---| +| 32 | 8192 | 1 | 48KB | 57,600 | 6 ← **Run 4** | +| 16 | 4096 | 3 | 48KB | 61,696 | 18 | + +Both are **below** the count-threshold of ~26 — so Path A **cannot bind by count alone**; it must bind by +**duty-cycle**. That is the whole point: crank **J** (burst length) up until enough of the ≤30 resident +waves are simultaneously fat. This is the cheapest possible first step — it changes *scheduling* +(burst granularity), not the LDS layout, and reuses Run-4's exact bank geometry. + +### Path B — partition (`G_resident < G`), an *optional* throughput lever behind a proven bind + +Only if Path A binds but leaves throughput on the table do we shrink banks to raise the **count** lever +too. Needs the §7 boundary resolution. Corrected feasible points (SEGK=16, OPSTRIDE=4096, 1 WG/CU), +requiring `POOL_N·4096 + G_resident·8192 ≤ 65280`: + +| POOL_N | G_resident | LDS (+256) | count = POOL_N·G_r | binds by count? | +|---|---|---|---|---| +| 7 | 4 | 61,696 | 28 | **yes** (≥26) | +| 9 | 3 | 61,696 | 27 | **yes** | +| 5 | 5 | 61,696 | 25 | marginal | + +**Key finding (revised):** one WG at `WAVES=30` can bind *either* by count (Path B, `POOL_N·G_resident≳26`) +*or* by duty-cycle (Path A, large J) *or* by both. Path A is the milestone-1 default because it is +correct-by-construction (no boundary hazard) and free (no layout change). `SEGK=16` (halved operand slots) +is what buys headroom on *both* paths. A further lever — dropping banks small enough for **2 WG/CU** — is +noted but not pursued until 1-WG/CU binding is characterized. + +## 5. B-stationary residency + the cache math (why re-fetch is free) + +*(Relevant only to Path B, where splitting a tile into `G/G_resident` groups re-consumes B once per group. +Path A re-fetches nothing new.)* Measured gfx1201 caches: **L2 = 8MB, L3/Infinity-Cache = 64MB** +(`rocminfo`, chip 0x7551). + +- One B panel `B[:,tcol]` = `K·FN·16` fp8 = **128KB** (oracle) … **576KB** (real ml8 down, K=9216). +- Under **B-stationary** scan order (co-resident WGs share `tcol` — the existing `KMAJOR` hook), the L2 + working set = a *handful* of shared panels = **0.5–4.6MB** → fits the 8MB L2 with margin. +- Safety net: the **entire B** fits L3 for our shapes — real down B = 9216×2560 fp8 = **23.6MB < 64MB**; + oracle B = 2MB < 8MB L2. After first touch, every B (re)access is **at worst an L3 hit, never HBM.** + +Re-fetching B per group is a cache hit **by construction**. (DE frame: a partitioned hash-agg — fewer +group-by states in `work_mem`/LDS → more concurrent workers; the re-scan of the small dimension table B +stays in the buffer pool.) + +## 6. Design + +- **Reduction (H1 fix, both paths):** at rowblk-claim time, **zero-initialize** the `G_resident` banks + (`ds_store 0`). Drop the `ksi==0` special case. *Every* merge is `ds_add_f32`. +- **Burst = a ksi-run:** a fat compute wave claims `(tile, rowblk, ksi_lo)` and grinds **J** consecutive + ksi `[ksi_lo, ksi_lo+J)` in the **VGPR ACC** (WMMA-accumulate, no LDS between them), then does **one** + `ds_add` merge of the ACC into the rowblk's bank, shrinks, and re-claims. **J is a build/dispatch knob.** +- **Enumeration:** coordinator emits `(tile, rowblk, ksi_run)` where `ksi_run` indexes `⌈n_kseg/J⌉` + runs. Completer stores each rowblk's bank to C **once** (plain `global_store`, no atomic) after its last + run merges; banks recycle. +- **Path A (default):** `G_resident = G` (whole tile, all rowblks' banks co-resident). No group axis, so + no group-boundary drain — only the existing tile-boundary drain. `SEGK=16` → `POOL_N ≤ 3`. +- **Path B (optional, post-bind):** `ACC_N = G_resident < G`; add a `group ∈ [0, G/G_resident)` axis; + B re-staged per group from L2/L3; requires §7 drain-before-admit. *(Impl caveat: OPSTRIDE/operand-staging + currently bake `G=6`; Path B must re-derive staging for `G_resident`.)* +- **Waves:** `WAVES = 30` (kernel cap; barrier/mailbox already supports ≤30). +- **Unchanged:** on-chip `ds_add_f32` reduction (fp32), count-to-WAVES exit barrier, deadman throttle, + chunking + compositor yield. + +## 7. Correctness + +- fp32 on-chip reduction preserved → oracle `bad=0` expected (same math class as write-once). +- **H1 (was a silent race — now resolved):** zero-init banks + all-`ds_add` ⇒ order-independent ⇒ correct + under any number of concurrent same-rowblk runs. This is a *precondition* for J>1 and for `POOL_N>1`. +- **Path A boundary hazard: none.** Whole-tile groups have no intra-tile group boundary; the pool can only + span a tile boundary, handled by the existing tile-drain. +- **Path B boundary hazard (H5):** with `POOL_N>1` the pool can span two groups whose rowblks map to the + *same* (group-relative) banks; group g+1's runs would `ds_add` into group g's not-yet-stored sums. + **Resolution = drain-before-admit:** the coordinator does not admit group g+1 until group g's banks are + stored + `s_wait` (serializes at *group* granularity only). Slot-indexed banks (`POOL_N·G_resident` + banks) are rejected — too much LDS. This is why Path A goes first. + +## 8. Gates (sequenced, per kmbandy) + +1. **Correctness** — builds, W-anything, oracle `bad=0`, brick-free. Flag-gated so `J=1, G_resident=G` + is byte-identical to today's write-once (the safe fallback). +2. **Bind** — at a work-heavy shape, sweep **J** (Path A) until `grow-fail > 0` — the proof the stagger + engages. *This is the milestone Run 4 failed, and J is the new lever to clear it.* +3. **Is the bind the *real* constraint? (the honesty gate — Fable's caveat):** duty-cycle headroom only + converts to throughput **if VGPR is the binding occupancy constraint.** A prior TF finding hinted one + cap was **structural (tile geometry), not VGPR.** So at Gate 2 also record whether TF *moves* with J — + if `grow-fail>0` but TF is flat, the wall is elsewhere and Path B/POOL_N gymnastics won't help. Measure + before building more. +4. **Throughput** — TF vs the write-once bin; sweep `J × POOL_N × (G_resident) × SEGK × WAVES`. + +## 9. Open questions for the council + +1. Does `ds_add` **contention on a shared bank** (many concurrent same-rowblk runs merging into one bank[r]) + become the new wall as J shrinks / POOL_N grows? (J>1 *reduces* merge frequency — is it enough?) +2. Is `instantaneous-fat = count × duty-cycle` the right silicon model, or does something else cap first + (claim-rate, coordinator throughput, feed bandwidth, the structural/tile-geometry cap from Gate 3)? +3. Optimal `J` for a target shape — and does the best J leave *enough* occupancy headroom to still stagger, + or does binding Path A force J so high it's effectively a square wave (H2's 91%-duty fragility)? +4. If Path A binds but Gate-3 says the wall is structural, is Path B worth building at all — or does the + answer become "fix tile geometry," not "shrink banks"? +5. Any reason B-stationary + Path-B re-staging thrashes L2 in a way the §5 math misses? + +## 10. Instrumentation TODO (before Gate 2) + +Wire the **dead `occ[58]` FATMAX** counter (defined, never written; non-TRACE) to record the running max of +instantaneous-fat, so Gate 2 can *measure* duty-cycle × count directly instead of inferring it from +`grow-fail`. `occ[57]` FATLIVE (current fat count) pairs with it. Without this we can only see *whether* the +pool binds, not *how close* a non-binding config came — which is exactly what J-sweeping needs. diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_flow.sh b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_flow.sh index ee6a8bfc0f53..c761f7303f65 100755 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_flow.sh +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/build_flow.sh @@ -11,7 +11,7 @@ mkflow() { # EMERGENT economy: no mix args. Env: WAVES VBUDGET G SEGK POOL_N ACC local tag="occ_dsws2_w${WAVES:-16}_flow_gd" nice -19 ionice -c3 "$L/clang" -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx1201 \ -Wa,-defsym,DSWS2=1 -Wa,-defsym,FM=2 -Wa,-defsym,FN=4 -Wa,-defsym,G=${G:-6} -Wa,-defsym,SEGK=${SEGK:-64} \ - -Wa,-defsym,SAFEPROBE=1 -Wa,-defsym,DIAG=${DIAG:-0} -Wa,-defsym,POOL_N=${POOL_N:-3} -Wa,-defsym,ACC_N=${ACC_N:-1} \ + -Wa,-defsym,SAFEPROBE=1 -Wa,-defsym,DIAG=${DIAG:-0} -Wa,-defsym,POOL_N=${POOL_N:-3} -Wa,-defsym,ACC_N=${ACC_N:-1} -Wa,-defsym,WOFLUSH=${WOFLUSH:-0} \ -Wa,-defsym,WAVES=${WAVES:-16} -Wa,-defsym,VBUDGET=${VBUDGET:-1536} \ -Wa,-defsym,PHASEPROBE=${PHASEPROBE:-0} -Wa,-defsym,NOCFLUSH=${NOCFLUSH:-0} -Wa,-defsym,CSTORE=${CSTORE:-0} \ -Wa,-defsym,SLEEPN=${SLEEPN:-2} -Wa,-defsym,COORD_PERIOD=${COORD_PERIOD:-64} -Wa,-defsym,TFPROBE=${TFPROBE:-0} -Wa,-defsym,DEADMAN=${DEADMAN:-1} -Wa,-defsym,DEADMAN_TICKS=${DEADMAN_TICKS:-50000000} -Wa,-defsym,STAGINSTR=${STAGINSTR:-0} -Wa,-defsym,TRACE=${TRACE:-0} \ diff --git a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s index 89849bfdcd11..a2c16c5c4147 100644 --- a/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s +++ b/ggml/src/ggml-cuda/aiter-integration/rdna4_fp8_gemm/spike/dvgpr_occ/occ_kernel_dsws_flow.s @@ -87,6 +87,15 @@ .endif // phase boundary and atomic-adds the delta into per-phase occ accumulators // (occ[64..69], bytes 256..276, ABOVE the per-chunk memset -> accumulate over // the whole run). Host prints ticks + % per phase. Default 0 => byte-identical. +.ifndef WOFLUSH + .set WOFLUSH, 0 // BURST-SCOPED FLUSH (LDS-halving lever, council 2026-07-05): 1 = drop the +.endif // per-rowblk LDS accumulator banks entirely; each compute burst atomic-adds + // its fp32 ACC frags DIRECTLY to C (global_atomic_add_f32, fp32-exact, same + // addresses as the write-once completer store). Build with ACC_N=0 (host + // DSWS2_ACC_N=0) -> LDS/WG ~8KB -> ~7 WGs/CU -> per-SIMD VGPR pool BINDS -> + // the dyn-VGPR traveling-peak finally engages (grow-fail>0). Re-incurs the + // n_kseg-x C-write atomic traffic write-once removed; tunable later by burst + // K-depth J + KMAJOR. Default 0 => byte-identical to the write-once bin. .ifndef NOCFLUSH .set NOCFLUSH, 0 // PERF PROBE ONLY: 1 = skip the global_atomic_add_f32 C-flush loop (keep ALL .endif // other bookkeeping/handshake). Isolates the device-atomic C-reduction cost @@ -1579,6 +1588,29 @@ occ_kernel: .endr .set ks, ks+1 .endr +.if WOFLUSH + // BURST-SCOPED FLUSH (no LDS bank): atomic-add this segment's fp32 ACC frags STRAIGHT to C[rowblk r]. + // C is memset 0 by the host; every segment of every rowblk atomic-adds -> C = full split-K sum. + // Same addressing as the write-once completer store (v10=lane*32, offset frag*1024+e*4) so it lands + // in the identical C locations -> correct by construction. s19=mblk s30=tcol s33=rowblk r (all live). + s_mul_i32 s38, s19, s13 // mblk*NTL + s_add_u32 s38, s38, s30 // + tcol + s_mul_i32 s38, s38, (G*FM*FN*1024) // * per-tile C bytes + s_mul_i32 s40, s33, (FM*FN*1024) // + rowblk r * per-rowblk C bytes + s_add_u32 s38, s38, s40 + s_add_u32 s28, s6, s38 + s_addc_u32 s29, s7, 0 // s[28:29] = C rowblk base + .set frag, 0 + .rept FM*FN + .set e, 0 + .rept 8 + global_atomic_add_f32 v10, v[ACC+frag*8+e], s[28:29] offset:(frag*1024 + e*4) scope:SCOPE_DEV + .set e, e+1 + .endr + .set frag, frag+1 + .endr + s_wait_storecnt 0x0 // J=1 correctness baseline: drain this wave's atomics +.else // WRITE-ONCE REDUCE: accumulate this segment's partial into LDS bank[r] (mirrors C frag layout; // vaddr = v10=lane*32, base = ACC_BASE + r*ACC_STRIDE). ksi==0 (tile's first segment, POOL_N=1 // guarantees it drains before any later ksi) WRITES; ksi>0 ADDS. C is stored ONCE at ksi==mask @@ -1609,6 +1641,7 @@ occ_kernel: .endr .Lflow_bankdn: s_wait_dscnt 0x0 +.endif instr_inc STINSTR_COMP // diag: a rowblk-segment was actually computed+reduced s_add_u32 s45, s48, SL_RBDONE lds_fetch_add_r s47, s45, 1 // s47 = old RBDONE; old==G-1 -> I am the UNIQUE completer @@ -1626,6 +1659,7 @@ occ_kernel: // s19/s30/s31 still hold mblk/tcol/ksi from this wave's own DECODE_STI (untouched by the reduce). s_cmp_eq_u32 s31, s67 // ksi == mask (n_kseg-1) -> tile complete? s_cbranch_scc0 .Lflow_drain_adv // not last ksi -> just advance DRAIN (no store) +.if !WOFLUSH s_mul_i32 s38, s19, s13 // mblk*NTL s_add_u32 s38, s38, s30 // + tcol s_mul_i32 s38, s38, (G*FM*FN*1024) // * per-tile C bytes @@ -1649,6 +1683,7 @@ occ_kernel: .set r, r+1 .endr s_wait_storecnt 0x0 // store COMPLETE before DRAIN++ -> banks safe to reuse +.endif // WOFLUSH: atomics already wrote C incrementally -> no store, just DRAIN++ .Lflow_drain_adv: lds_get s44, DRAIN_HEAD_OFF lds_cmpstore_adv DRAIN_HEAD_OFF, s44 // completer advances DRAIN (unique wave; store already done) From 8a7d9c5f1f4a9257447769eb31ab2a3b5f4b5b09 Mon Sep 17 00:00:00 2001 From: mad-lab-kbando <69054773+kmbandy@users.noreply.github.com> Date: Mon, 6 Jul 2026 08:23:48 -0400 Subject: [PATCH 19/19] wip(pagedattn): snapshot in-progress mt_pagedattn fp8/aiter tweaks Snapshot of local work-in-progress on the paged-attention path (mt_pagedattn.cu, mt_pagedattn_aiter.cu, mt_pagedattn_turbo_fp8.cuh) so it rides along in this branch push. Small, self-contained (14 insertions / 14 deletions); unrelated to the DSWS spike work in the rest of the branch. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_0132aDSBLwusCJ4KzHQTnvdu --- ggml/src/ggml-cuda/mt_pagedattn.cu | 13 ++++++------- ggml/src/ggml-cuda/mt_pagedattn_aiter.cu | 13 +++++++------ ggml/src/ggml-cuda/mt_pagedattn_turbo_fp8.cuh | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-cuda/mt_pagedattn.cu b/ggml/src/ggml-cuda/mt_pagedattn.cu index fbf33325d414..5e86ebbf7304 100644 --- a/ggml/src/ggml-cuda/mt_pagedattn.cu +++ b/ggml/src/ggml-cuda/mt_pagedattn.cu @@ -95,9 +95,8 @@ static int get_paged_decode_mode() { // ───────────────────────── helpers ───────────────────────── -// Warp-level reduce across 32 lanes (HIP wavefront is 64 on some -// GPUs but ggml-cuda's WARP_SIZE is fixed at 32 — works correctly on -// gfx1xxx because __shfl_xor_sync over the active mask). +// Warp-level reduce across 32 lanes. HIP wavefronts can be 64 lanes, while +// ggml-cuda's WARP_SIZE is fixed at 32, so every shuffle is width-scoped. template __device__ __forceinline__ T warp_reduce_sum(T v) { #pragma unroll @@ -373,7 +372,7 @@ __global__ void mt_scatter_kv_turbo4_0_kernel( { float v_sq = x[j] * x[j]; for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { - v_sq += __shfl_xor_sync(0xffffffffu, v_sq, offset); + v_sq += __shfl_xor_sync(0xffffffffu, v_sq, offset, WARP_SIZE); } if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = v_sq; } @@ -431,7 +430,7 @@ __global__ void mt_scatter_kv_turbo4_0_kernel( const float c = TURBO_CENTROIDS_4BIT[idx]; float rc = c * c; for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { - rc += __shfl_xor_sync(0xffffffffu, rc, offset); + rc += __shfl_xor_sync(0xffffffffu, rc, offset, WARP_SIZE); } if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = rc; } @@ -505,7 +504,7 @@ __global__ void mt_scatter_kv_turbo4_64_kernel( { float v_sq = x[j] * x[j]; for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { - v_sq += __shfl_xor_sync(0xffffffffu, v_sq, offset); + v_sq += __shfl_xor_sync(0xffffffffu, v_sq, offset, WARP_SIZE); } if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = v_sq; } @@ -540,7 +539,7 @@ __global__ void mt_scatter_kv_turbo4_64_kernel( const float c = TURBO_CENTROIDS_4BIT[idx]; float rc = c * c; for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { - rc += __shfl_xor_sync(0xffffffffu, rc, offset); + rc += __shfl_xor_sync(0xffffffffu, rc, offset, WARP_SIZE); } if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = rc; } diff --git a/ggml/src/ggml-cuda/mt_pagedattn_aiter.cu b/ggml/src/ggml-cuda/mt_pagedattn_aiter.cu index 7a90c0b3cbf2..bf0552b8905b 100644 --- a/ggml/src/ggml-cuda/mt_pagedattn_aiter.cu +++ b/ggml/src/ggml-cuda/mt_pagedattn_aiter.cu @@ -296,7 +296,7 @@ __global__ void mt_scatter_kv_turbo4_aiter_kernel( { float v_sq = x[j] * x[j]; for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { - v_sq += __shfl_xor_sync(0xffffffffu, v_sq, offset); + v_sq += __shfl_xor_sync(0xffffffffu, v_sq, offset, WARP_SIZE); } if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = v_sq; } @@ -332,7 +332,7 @@ __global__ void mt_scatter_kv_turbo4_aiter_kernel( const float c = TURBO_CENTROIDS_4BIT[idx]; float rc = c * c; for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { - rc += __shfl_xor_sync(0xffffffffu, rc, offset); + rc += __shfl_xor_sync(0xffffffffu, rc, offset, WARP_SIZE); } if (j % WARP_SIZE == 0) warp_accum[j / WARP_SIZE] = rc; } @@ -456,7 +456,7 @@ __global__ void mt_scatter_kv_turbo4_fp8_aiter_kernel( // ── Stage 2: per-block max-abs scale ── float v_abs = fabsf(x[j]); for (int off = 16; off > 0; off >>= 1) { - v_abs = fmaxf(v_abs, __shfl_xor_sync(0xffffffffffffffffull, v_abs, off)); + v_abs = fmaxf(v_abs, __shfl_xor_sync(0xffffffffffffffffull, v_abs, off, WARP_SIZE)); } __shared__ float warp_max[8]; if ((j % 32) == 0) warp_max[j / 32] = v_abs; @@ -496,7 +496,7 @@ __global__ void mt_scatter_kv_turbo4_fp8_aiter_kernel( } const uint8_t my_nib = (uint8_t)(best_idx & 0xF); - const uint8_t partner_nib = (uint8_t) __shfl_xor_sync(0xffffffffffffffffull, (int) my_nib, 1); + const uint8_t partner_nib = (uint8_t) __shfl_xor_sync(0xffffffffffffffffull, (int) my_nib, 1, WARP_SIZE); if ((j & 1) == 0) { blk[2 + j / 2] = my_nib | (uint8_t)(partner_nib << 4); } @@ -504,8 +504,9 @@ __global__ void mt_scatter_kv_turbo4_fp8_aiter_kernel( const uint64_t sign_mask = __ballot_sync(0xffffffffffffffffull, sgn); if ((j & 7) == 0) { const int byte_idx = j / 8; - const int warp_off = (j % 32) / 8; - blk[130 + byte_idx] = (uint8_t)((sign_mask >> (warp_off * 8)) & 0xFF); + const int hw_lane_base = (threadIdx.x % warpSize) & ~(WARP_SIZE - 1); + const int bit_off = hw_lane_base + ((j % WARP_SIZE) & ~7); + blk[130 + byte_idx] = (uint8_t)((sign_mask >> bit_off) & 0xFF); } } diff --git a/ggml/src/ggml-cuda/mt_pagedattn_turbo_fp8.cuh b/ggml/src/ggml-cuda/mt_pagedattn_turbo_fp8.cuh index bb1bb3f53bd6..4dd1c67a7020 100644 --- a/ggml/src/ggml-cuda/mt_pagedattn_turbo_fp8.cuh +++ b/ggml/src/ggml-cuda/mt_pagedattn_turbo_fp8.cuh @@ -126,7 +126,7 @@ static __device__ __forceinline__ void coop_stage_turbo4_fp8_bs256_tile( // Broadcast scale from lane 0 to all 32 lanes of this warp. // 64-bit mask literal required on RDNA4 (gfx12, wave32) per // amd_warp_sync_functions.h static_assert. - scale_f = __shfl_sync(0xFFFFFFFFFFFFFFFFull, scale_f, 0); + scale_f = __shfl_sync(0xFFFFFFFFFFFFFFFFull, scale_f, 0, WARP_SIZE); // Read this lane's 4-byte qs window (8 nibbles) and 1-byte signs window. uint32_t qs_word = 0;