fix(qmoe): choose the decode GEMV tiling before computing, not on real calls

lib/Runtime/Kernels/include/hip_custom_kernels.h

@@ -3137,6 +3137,10 @@ HIP_KERNEL_API int hip_qmoe_reduce_sorted_pairs(
  *
  * Constraints: fp16 only (element_size_bytes == 2); hidden_size and
  * inter_size both multiples of 32; block_size > 0 and even.
+ *
+ * num_experts (E) lets the first call of a shape time the GEMV tile
+ * configurations on other experts' weights before computing; every call
+ * then uses the configuration chosen, so equal inputs give equal results.
  */
 HIP_KERNEL_API int hip_qmoe_decode_fused(
     void* stream,
@@ -3153,7 +3157,7 @@ HIP_KERNEL_API int hip_qmoe_decode_fused(
     int64_t hidden_size, int64_t inter_size,
     int64_t k, int64_t block_size,
     float swiglu_alpha, float swiglu_beta, float swiglu_limit,
-    int64_t element_size_bytes);
+    int64_t element_size_bytes, int64_t num_experts);
 
 // W4A8 dp4a decode variant of hip_qmoe_decode_fused (env-gated via
 // HIPDNN_EP_MATMUL_DP4A). Additionally takes 4 scratch buffers: quantized
@@ -3176,7 +3180,7 @@ HIP_KERNEL_API int hip_qmoe_decode_fused_dp4a(
     int64_t hidden_size, int64_t inter_size,
     int64_t k, int64_t block_size,
     float swiglu_alpha, float swiglu_beta, float swiglu_limit,
-    int64_t element_size_bytes);
+    int64_t element_size_bytes, int64_t num_experts);
 
 /* =========================================================================
  * com.amd QMoE Sub-Kernels (LatentMoE)

lib/Runtime/Kernels/unit_test/CMakeLists.txt

@@ -150,6 +150,9 @@ foreach(_arch IN LISTS HIP_ARCHITECTURES)
   add_kernel_package_ut(${_arch}
     hipdnn-kernel-ut-gather-block-quantized-xi64-wfp8-yfp16
     gather_block_quantized/Xi64_Wfp8_Yfp16/test_gather_block_quantized_fp8.cpp)
+  add_kernel_package_ut(${_arch}
+    hipdnn-kernel-ut-qmoe-decode-xfp16-wu4-yfp16
+    qmoe_decode/Xfp16_Wu4_Yfp16/test_qmoe_decode.cpp)
 endforeach()
 
 # One package-facing entry point finds the live GPU arch, invokes every leaf

lib/Runtime/Kernels/unit_test/Makefile

@@ -43,7 +43,7 @@ LEAVES := gather_block_quantized/Xi64_Wfp8_Yfp16 \
 	matmul_nbits/Xfp16_Wu4_Yfp16 matmul_nbits/Xfp16_Wi8_Yfp16 \
 	matmul_nbits/Xfp32_Wu2_Yfp32 matmul_nbits/Xfp32_Wu3_Yfp32 \
 	matmul_nbits/Xfp32_Wu4_Yfp32 matmul_nbits/Xfp32_Wi8_Yfp32 \
-	qmoe_ragged_mm/Xfp16_Wu4_Yfp16
+	qmoe_decode/Xfp16_Wu4_Yfp16 qmoe_ragged_mm/Xfp16_Wu4_Yfp16
 
 ARCH  := $(patsubst --offload-arch=%,%,$(OFFLOAD))
 HIPCC := $(HIP_SDK)/bin/hipcc.exe

lib/Runtime/Kernels/unit_test/README.md

@@ -90,7 +90,7 @@ controller from that `bin` directory:
 | `--mode` | `lookup`, `autotune` | `lookup` | `lookup` uses the tables embedded in `custom_kernels_<arch>.dll` and falls through to autotune on a miss, like production. `autotune` bypasses the tables (production's `online` mode). |
 | `--coverage` | `1`, `2`, `3` | `3` | Shape tier; see "Shape coverage". `1` suits a per-PR job, `3` a nightly one. |
 
-The controller runs the 16 leaves serially, with `bin` as the working
+The controller runs the 17 leaves serially, with `bin` as the working
 directory so every leaf finds `custom_kernels_<arch>.dll`. It prints one
 `[<op>/<leaf>] PASS|FAIL (<ms> ms)` line per leaf and finally
 `Combined report: <path>`.
@@ -193,7 +193,7 @@ Concretely:
   never drops a categorical situation.
 
 A tier3 (default) run of one leaf is on the order of tens to ~150 cases, and
-the whole 16-leaf suite at tier3 is budgeted at **up to ~30 min total**
+the whole 17-leaf suite at tier3 is budgeted at **up to ~30 min total**
 (measured ~20-25 min in practice) — this is intentionally wider than a
 "finishes in a minute" smoke test, because the shape axis (M=1 vs M>1,
 multiple K/N/seq scenarios) needs real coverage too, not just the categorical
@@ -256,6 +256,7 @@ unit_test/
   gqa/decode/Xfp16_Wi8_Yfp16/
   gqa/prefill/Xfp16_Wfp16_Yfp16/
   gqa/prefill/Xfp16_Wi8_Yfp16/
+  qmoe_decode/Xfp16_Wu4_Yfp16/
   qmoe_ragged_mm/Xfp16_Wu4_Yfp16/
   gather_block_quantized/Xi64_W{u4,u8,fp8}_Yfp16/
 ```
@@ -362,6 +363,12 @@ COVERAGE=N` (any leaf, or the top-level aggregator) builds and runs it.
   (`genCase()`'s `O(M*K*N)`, no BLAS) is now multithreaded over `N` to keep
   this affordable; see the comment above `genCase()` in any
   `matmul_nbits/*/test_matmul_nbits*.cpp`.
+- **qmoe_decode** (the fused single-token decode): `{fp, dp4a}` x block
+  size `{32,64,128}` x zero points `{off,on}` x bias `{off,on}` (24 combos)
+  on a shape list COVERAGE thins (tier1 = 1 shape, tier2 = 2 with
+  GPT-OSS-20B, tier3 = 3), each checked against a CPU reference and then
+  for 128 more bit-identical calls. Every tier also runs a first call made
+  over a pending HIP error and the rejected arguments.
 - **qmoe_ragged_mm**: routing `{balanced, empty experts, skewed}` x input
   `{contiguous, FC1 gather}` x bias `{off,on}` x zero points `{off,on}` (24
   combos) on a shape list COVERAGE thins (tier1 = 1 shape, tier2 = 2, tier3 =

lib/Runtime/Kernels/unit_test/kernel_ut_main.cpp

@@ -62,6 +62,8 @@ static const Leaf kLeaves[] = {
      "hipdnn-kernel-ut-gather-block-quantized-xi64-wu8-yfp16.exe"},
     {"gather_block_quantized", "Xi64_Wfp8_Yfp16",
      "hipdnn-kernel-ut-gather-block-quantized-xi64-wfp8-yfp16.exe"},
+    {"qmoe_decode", "Xfp16_Wu4_Yfp16",
+     "hipdnn-kernel-ut-qmoe-decode-xfp16-wu4-yfp16.exe"},
 };
 
 static std::string executableDirectory() {

lib/Runtime/Kernels/unit_test/qmoe_decode/Xfp16_Wu4_Yfp16/Makefile

@@ -0,0 +1,129 @@
+# ============================================================
+# custom_kernels qmoe_decode (fused single-token QMoE decode: fp16 input,
+# 4-bit weights, fp16 output; fp and dp4a variants) Test
+#
+#   make test        OFFLOAD=--offload-arch=gfx1151 HIP_SDK=<sdk> [COVERAGE=1|2|3]
+#   make test_custom OFFLOAD=... HIP_SDK=... ARGS="--coverage 1"
+#   make clean
+#
+# MODE is accepted for the uniform API but only labels out/results.csv: the
+# decode picks its GEMV tiling itself, on the first call of each shape.
+#
+# `test` runs every categorical situation (fp / dp4a x block size x zero
+# points x bias, a pending HIP error, rejected arguments) on the shapes
+# COVERAGE selects (default 3; see unit_test/README.md "Shape coverage").
+# Inputs and the CPU reference are generated in-process -- no python, no data
+# files.
+# ============================================================
+
+ifeq ($(OS),Windows_NT)
+    DETECTED_OS := Windows
+    SHELL := cmd.exe
+    .SHELLFLAGS := /c
+else
+    DETECTED_OS := WSL
+endif
+
+# ---- Required from CI / CLI ----
+
+OFFLOAD  ?= --offload-arch=gfx1151
+HIP_SDK  ?= $(HIP_PATH)
+MODE     ?= lookup
+COVERAGE ?= 3
+ARGS     ?=
+
+ifeq ($(HIP_SDK),)
+    $(error pass HIP_SDK=<HIP SDK root>)
+endif
+
+# ---- Paths ----
+
+HIPCC       := $(HIP_SDK)/bin/hipcc.exe
+INCLUDE_DIR := ../../../include
+KERNEL_HIP  := ../../../hip/qmoe_kernel.hip
+
+BUILD_DIR ?= out
+TARGET_DIRECT := $(BUILD_DIR)/test_direct.exe
+RESULTS_CSV   := $(BUILD_DIR)/results.csv
+
+ARCH := $(patsubst --offload-arch=%,%,$(OFFLOAD))
+
+# ---- Build settings ----
+
+KERNEL_FLAGS := -O3 -std=c++17 -x hip -ffp-contract=fast
+CXXFLAGS     := -O3 -std=c++17
+INCLUDES     := -I$(INCLUDE_DIR)
+
+# ---- OS-specific: run command + clean ----
+
+ifeq ($(DETECTED_OS),Windows)
+    HIP_SDK_WIN := $(subst /,\,$(HIP_SDK))
+    RUN_CMD     = set "PATH=$(HIP_SDK_WIN)\bin;%PATH%" && set "HIPDNN_RESULTS_CSV=$(subst /,\,$(RESULTS_CSV))" && set "HIPDNN_RESULTS_OP=qmoe_decode" && set "HIPDNN_RESULTS_LEAF=Xfp16_Wu4_Yfp16" && set "HIPDNN_RESULTS_ARCH=$(ARCH)" && set "HIPDNN_RESULTS_MODE=$(MODE)" && $(subst /,\,.\$(1))
+    define DO_CLEAN
+	if exist $(BUILD_DIR) rmdir /s /q $(BUILD_DIR)
+    endef
+else
+    HIP_SDK_WIN := $(shell wslpath -w "$(HIP_SDK)")
+    RUN_CMD     = cmd.exe /c "set PATH=$(HIP_SDK_WIN)\bin;%PATH% && set HIPDNN_RESULTS_CSV=$(RESULTS_CSV) && set HIPDNN_RESULTS_OP=qmoe_decode && set HIPDNN_RESULTS_LEAF=Xfp16_Wu4_Yfp16 && set HIPDNN_RESULTS_ARCH=$(ARCH) && set HIPDNN_RESULTS_MODE=$(MODE) && %CD%\$(1)"
+    define DO_CLEAN
+	rm -rf $(BUILD_DIR)
+    endef
+endif
+
+CSV_FRESH ?= 1
+RESET_RESULTS_CSV = if exist $(subst /,\,$(RESULTS_CSV)) del $(subst /,\,$(RESULTS_CSV))
+
+# ============================================================
+# Build
+# ============================================================
+
+ifeq ($(DETECTED_OS),Windows)
+$(BUILD_DIR):
+	@if not exist $(BUILD_DIR) mkdir $(BUILD_DIR)
+else
+$(BUILD_DIR):
+	@mkdir -p $@
+endif
+
+DIRECT_TEST_OBJ   := $(BUILD_DIR)/direct_test.obj
+DIRECT_KERNEL_OBJ := $(BUILD_DIR)/direct_kernel.obj
+
+direct: FORCE | $(BUILD_DIR)
+	$(HIPCC) -c $(CXXFLAGS) $(OFFLOAD) $(INCLUDES) -x hip test_qmoe_decode.cpp -o $(DIRECT_TEST_OBJ)
+	$(HIPCC) -c $(KERNEL_FLAGS) $(OFFLOAD) $(INCLUDES) $(KERNEL_HIP) -o $(DIRECT_KERNEL_OBJ)
+	$(HIPCC) $(OFFLOAD) $(DIRECT_TEST_OBJ) $(DIRECT_KERNEL_OBJ) -o $(TARGET_DIRECT)
+
+# ============================================================
+# Run / test targets (uniform API)
+# ============================================================
+
+clean:
+	$(DO_CLEAN)
+
+# make test = build + run every categorical situation at $(COVERAGE)
+# (default 3). No data files -- everything is in-process.
+test: direct
+	@$(RESET_RESULTS_CSV)
+	$(call RUN_CMD,$(TARGET_DIRECT)) --coverage $(COVERAGE)
+
+# Pass the exe's own flags through ARGS= (e.g. --coverage 1).
+test_custom: direct
+ifeq ($(CSV_FRESH),1)
+	@$(RESET_RESULTS_CSV)
+endif
+	$(call RUN_CMD,$(TARGET_DIRECT)) $(ARGS)
+
+# Build only. The unit_test/ aggregator passes PKG_DIR and PKG_NAME to collect
+# the exe into its hipdnn-kernel-ut package layout.
+build: direct
+ifneq ($(PKG_DIR),)
+ifeq ($(DETECTED_OS),Windows)
+	@copy /Y $(subst /,\,$(TARGET_DIRECT)) "$(subst /,\,$(PKG_DIR))\$(PKG_NAME)" >nul
+else
+	@cp $(TARGET_DIRECT) $(PKG_DIR)/$(PKG_NAME)
+endif
+endif
+
+FORCE:
+
+.PHONY: test test_custom clean build direct FORCE

lib/Runtime/Kernels/unit_test/qmoe_decode/Xfp16_Wu4_Yfp16/README.md

@@ -0,0 +1,74 @@
+# qmoe_decode Xfp16_Wu4_Yfp16 standalone test
+
+Standalone `hipcc`-only test for the fused single-token QMoE decode,
+`hip_qmoe_decode_fused` and its W4A8 variant `hip_qmoe_decode_fused_dp4a`:
+fp16 input, 4-bit expert weights with fp16 scales, fp16 output. It does not
+configure or build hip-ep, ONNX Runtime, LLVM, or MLIR. `make` compiles
+`qmoe_kernel.hip` into the leaf; the CMake package build links
+`custom_kernels_<arch>.dll` instead. Inputs and the CPU reference are
+generated in-process -- no python, no data files.
+
+```
+make test        OFFLOAD=--offload-arch=%GFX% HIP_SDK=%HIP_SDK% [COVERAGE=1|2|3]
+make test_custom OFFLOAD=... HIP_SDK=... ARGS="--coverage 1"
+make clean
+```
+
+| Target | Meaning |
+|---|---|
+| `test` | Coverage-tiered sweep -- see "Shape coverage" below. |
+| `test_custom` | Runs the exe with `ARGS=` (e.g. `--coverage 1`). |
+| `clean` | Removes `out/`. |
+
+## What it checks
+
+Every case runs FC1 + SwiGLU, FC2 and the weighted sum over k distinct
+experts, and checks:
+
+- the first call against a double-precision reference that rounds to fp16
+  where the kernels store (SwiGLU output, weighted slot, output) and, for
+  dp4a, quantizes the activations to per-group int8 the way the kernels do.
+  Each element must be within `4e-3` of itself plus `2e-3` of the largest
+  output (the k slots are fp16 before they are summed, so an element where
+  they cancel may be off by an ulp of the slots), and relL2 within `1e-3`;
+- 128 more calls on the same inputs, each bit-identical to the first. The
+  first call of a shape picks the GEMV tiling for the process, and the block
+  size sets the reduction order, so a tiling that changed after the first
+  call would show here;
+- output guard regions, and a NaN fill that fails any element left unwritten.
+
+Categorical situations, at every tier: fp and dp4a, block size 32, 64 and
+128, zero points absent (8) and present, bias absent and present. Only block
+sizes that are multiples of 32 are run: both GEMVs read one group's scale and
+zero point per 32 values.
+
+Also at every tier:
+
+- the first call of an untimed shape made while an earlier failed HIP call's
+  error is pending, for fp and dp4a, must return 0 with a correct, repeatable
+  result. Without the decode clearing the error on entry, dp4a reports it as
+  its own; fp's first tiling launch consumes it, so fp still succeeds but
+  keeps the default tiling, which this test cannot see;
+- an fp32 element size, an odd block size, a dp4a block size of 48, a hidden
+  size of 48 and k = 0 must return -1 and write nothing.
+
+## Shape coverage (3 tiers)
+
+`COVERAGE` only thins the shapes (hidden x inter, experts, k): tier1 =
+`64 x 64`, 4 experts, k 2; tier2 adds GPT-OSS-20B (`2880 x 2880`, 32, 4);
+tier3 (default) adds `2048 x 768`, 128, 8. Each shape runs all 24 categories.
+At startup the exe prints `qmoe_decode coverage=N cases=<n>` (32 / 56 / 80);
+a successful run prints one `PASS` line per case and ends with `ALL PASS`.
+
+## `MODE` and `out/results.csv`
+
+The decode has no autotune table: it times its GEMV tiling candidates on the
+first call of each shape. `MODE` is accepted for consistency with other
+leaves and only fills the `mode` column.
+
+Each case appends one row to `out/results.csv` (see `unit_test/README.md`),
+`shape` like `H2880_I2880_E32_k4`, `config` like `dp4a_bs32_zp_bias`, `relL2`
+against the reference, and `time_ms` the mean of 50 back-to-back calls after
+the repeat check, on warm caches (the same experts every call).
+
+CI passes `OFFLOAD`/`HIP_SDK` explicitly; there is no personal default.