bench: add quality gate for Qwen 3.6 27B AR on R9700 and Strix

.github/workflows/ci.yml

@@ -7,12 +7,22 @@ on:
     branches: [main]
     paths:
       - 'server/**'
+      - 'bench/**'
+      - 'optimizations/**'
+      - 'scripts/check_uv_workspace.sh'
+      - '.github/ci/**'
+      - 'docs/specs/quality-gates.md'
       - 'harness/**'
+      - '.python-version'
       - 'pyproject.toml'
       - 'uv.lock'
       - '.github/workflows/ci.yml'
+      - '.github/workflows/production-quality.yml'
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 concurrency:
   group: ci-${{ github.ref }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
@@ -21,8 +31,11 @@ jobs:
   uv-workspace:
     name: uv workspace (lock + sync + import smoke)
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
         with:
           version: "0.11.x"
@@ -34,12 +47,22 @@ jobs:
       - name: Lint Python surfaces touched by lucebox tooling
         run: uv run --frozen --extra dev ruff check .
 
-      - name: Test DS4 benchmark tools
+      - name: Feature / DS4 benchmark tooling
         run: uv run --frozen --extra dev pytest -q harness/tests/test_ds4_benchmark_tools.py
 
+      - name: Feature / release quality gates (model-free)
+        run: uv run --frozen --extra dev python -m pytest -q bench/tests
+
+      - name: Verify promoted performance baseline history
+        if: github.event_name == 'pull_request'
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: uv run --project bench --frozen python -m bench.performance.history verify-archives --base "$BASE_SHA"
+
   build:
     name: Build (cmake + uv sync --extra megakernel)
     runs-on: ubuntu-latest
+    timeout-minutes: 45
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
@@ -86,13 +109,22 @@ jobs:
             test_seq_engine_contract test_seq_batch_plan test_client_send_buffer \
             test_model_smoke test_batched_gdn test_concat_transpose -j$(nproc)
 
-      - name: Run C++ server unit tests
+      - name: Feature / launch compatibility (model-free)
         run: |
-          cd server/build
-          ctest --output-on-failure \
-            -R "server_unit|deepseek4_unit|feature_gate|seq_slot_manager|seq_engine_contract|seq_batch_plan|client_send_buffer|batched_gdn_cpu" \
+          ctest --test-dir server/build --output-on-failure \
+            -R 'feature_gate' --no-tests=error
+
+      - name: Feature / serving and sequence contracts
+        run: |
+          ctest --test-dir server/build --output-on-failure \
+            -R 'server_unit|seq_slot_manager|seq_engine_contract|seq_batch_plan|client_send_buffer' \
             --no-tests=error
 
+      - name: Feature / DeepSeek 4 and recurrent CPU contracts
+        run: |
+          ctest --test-dir server/build --output-on-failure \
+            -R 'deepseek4_unit|batched_gdn_cpu' --no-tests=error
+
       - name: Populate venv with cu128 torch + setuptools
         # First pass: install the workspace's default deps. dflash declares
         # torch (which uv pulls from the pytorch-cu128 index per
@@ -212,6 +244,7 @@ jobs:
             test_rocmfp_mix_gateup_glu
             test_ds4_mix_registry_teardown
             test_model_smoke
+            test_server_unit
             test_batched_gdn
             test_concat_transpose
           )
@@ -220,41 +253,26 @@ jobs:
           fi
           cmake --build build --target "${targets[@]}" -j"$(nproc)"
 
-      - name: Run focused NVIDIA verifier tests
+      - name: Feature / sparse attention allocator
+        if: matrix.sparse_attention == 'true'
+        run: ./server/build/test_flash_attn_sparse
+
+      - name: Feature / DeepSeek 4 mixed-format kernels
         run: |
-          if [[ "$RUN_SPARSE_ATTENTION_TEST" == "true" ]]; then
-            ./server/build/test_flash_attn_sparse
-          fi
           ./server/build/test_deepseek4_mmid_grouped_cuda
           ./server/build/test_deepseek4_unit
           ctest --test-dir server/build --output-on-failure \
             -R 'rocmfp3_mix_registry|rocmfp_mix_slice_matvec|rocmfp_mix_gateup_glu|ds4_mix_registry_teardown'
 
-      - name: Run concurrent-serving kernel tests
+      - name: Feature / sampler and draft top-k parity
         run: |
           ctest --test-dir server/build --output-on-failure \
-            -R '^(test_model_smoke\.PagedAttention\.|batched_gdn$|concat_transpose$)' --no-tests=error
+            -R '^test_server_unit\.(GpuSamplerCudaFixture|DraftTopkCudaFixture)\.' --no-tests=error
 
-      # Optional model-backed end-to-end smoke (real spec-decode on the 3090),
-      # disabled by default because it builds dflash_server and lazy-loads the
-      # ~16 GB Qwen3.6-27B target + draft (~1-2 min). The weights are already
-      # staged at /opt/models on lucebox-rtx3090 (override with repo var
-      # LUCEBOX_MODELS_DIR). Verified working on the runner. To enable, uncomment;
-      # continue-on-error keeps a heavy/slow run from ever blocking a PR.
-      # - name: dflash end-to-end smoke (model-backed)
-      #   continue-on-error: true
-      #   env:
-      #     MODELS: ${{ vars.LUCEBOX_MODELS_DIR || '/opt/models' }}
-      #   run: |
-      #     cd server
-      #     cmake --build build --target dflash_server -j"$(nproc)"
-      #     ./build/dflash_server "$MODELS/Qwen3.6-27B-Q4_K_M.gguf" \
-      #       --draft "$MODELS/draft/dflash-draft-3.6-q4_k_m.gguf" --port 8099 &
-      #     SRV=$!; trap 'kill $SRV 2>/dev/null' EXIT
-      #     for i in $(seq 1 90); do curl -sf localhost:8099/v1/models >/dev/null && break; sleep 2; done
-      #     curl -sf --max-time 240 localhost:8099/v1/completions \
-      #       -H 'Content-Type: application/json' \
-      #       -d '{"prompt":"The capital of France is","max_tokens":8}' | grep -q '"text"'
+      - name: Feature / Qwen paged concurrent serving
+        run: |
+          ctest --test-dir server/build --output-on-failure \
+            -R '^(test_model_smoke\.PagedAttention\.|batched_gdn$|concat_transpose$)' --no-tests=error
 
   gpu-tests-amd:
     name: GPU tests (${{ matrix.device_name }}, ${{ matrix.arch }} / ROCm)
@@ -270,27 +288,32 @@ jobs:
     needs: [uv-workspace]
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         include:
           - device_name: Radeon AI PRO R9700
             arch: gfx1201
             device_index: "0"
-            concurrency_key: r9700
+            qwen_profile: qwen36-27b-q4-r9700-ar-c4
           - device_name: Strix Halo Radeon 8060S
             arch: gfx1151
             device_index: "1"
-            concurrency_key: strix-halo
+            qwen_profile: qwen36-27b-q4-strix-halo-ar-c4
     runs-on: [self-hosted, lucebox3]
-    timeout-minutes: 20
+    timeout-minutes: 30
     env:
       HIP_VISIBLE_DEVICES: ${{ matrix.device_index }}
       EXPECTED_HIP_ARCH: ${{ matrix.arch }}
-    concurrency:
-      group: lucebox3-${{ matrix.concurrency_key }}-runner
-      cancel-in-progress: false
+      EXPECTED_DEVICE_NAME: ${{ matrix.device_name }}
+      QWEN_PROFILE: ${{ matrix.qwen_profile }}
+      MODELS: ${{ vars.LUCEBOX_MODELS_DIR || '/opt/models' }}
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
 
+      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          version: "0.11.x"
+
       - name: KFD health (diagnose instead of hanging)
         # rocminfo on a wedged KFD blocks in uninterruptible sleep and eats
         # the whole 20-minute job timeout. Probe with a hard timeout first,
@@ -335,12 +358,12 @@ jobs:
             -o "$RUNNER_TEMP/hip_smoke" .github/ci/hip_smoke.cpp
           "$RUNNER_TEMP/hip_smoke" "$EXPECTED_HIP_ARCH"
 
-      - name: Build + test ROCm formats and inference core
+      - name: Build focused AMD verifier targets
         run: |
           cmake -S server -B "$RUNNER_TEMP/rocmfp-build" \
             -DDFLASH27B_GPU_BACKEND=hip \
             -DDFLASH27B_HIP_ARCHITECTURES="$EXPECTED_HIP_ARCH" \
-            -DDFLASH27B_SERVER=OFF \
+            -DDFLASH27B_SERVER=ON \
             -DDFLASH27B_TESTS=ON \
             -DGGML_HIP_GRAPHS=ON \
             -DCMAKE_BUILD_TYPE=Release \
@@ -351,11 +374,55 @@ jobs:
               test_recurrent_snapshot test_server_unit test_rocmfp3_mix_registry \
               test_rocmfp_mix_slice_matvec test_rocmfp_mix_gateup_glu \
               test_ds4_mix_registry_teardown test_model_smoke test_batched_gdn \
-              test_concat_transpose \
+              test_concat_transpose test_draft_topk_cuda \
             --parallel 8
+
+      - name: Feature / DeepSeek 4 mixed-format kernels
+        run: |
+          ctest --test-dir "$RUNNER_TEMP/rocmfp-build" \
+            --output-on-failure \
+            -R 'rocmfp4_reference|rocmfpx_reference|rocmfp4_hip_tail|rocmfpx_mmq|rocmfp3_mix_registry|rocmfp_mix_slice_matvec|rocmfp_mix_gateup_glu|ds4_mix_registry_teardown' \
+            --no-tests=error
+
+      - name: Feature / DeepSeek 4 recurrent core
+        run: |
+          ctest --test-dir "$RUNNER_TEMP/rocmfp-build" \
+            --output-on-failure \
+            -R 'deepseek4_mmid_grouped_cuda|deepseek4_unit|recurrent_snapshot' \
+            --no-tests=error
+
+      - name: Feature / draft rollback and top-k parity
+        run: |
+          ctest --test-dir "$RUNNER_TEMP/rocmfp-build" \
+            --output-on-failure \
+            -R 'ChainRollbackPolicy|draft_topk_cuda' \
+            --no-tests=error
+
+      - name: Feature / Qwen paged concurrent serving
+        run: |
           ctest --test-dir "$RUNNER_TEMP/rocmfp-build" \
             --output-on-failure \
-            -R 'rocmfp4_reference|rocmfpx_reference|rocmfp4_hip_tail|rocmfpx_mmq|deepseek4_mmid_grouped_cuda|deepseek4_unit|recurrent_snapshot|ChainRollbackPolicy|rocmfp3_mix_registry|rocmfp_mix_slice_matvec|rocmfp_mix_gateup_glu|ds4_mix_registry_teardown|test_model_smoke\.PagedAttention\.|batched_gdn$|concat_transpose$'
+            -R 'test_model_smoke\.PagedAttention\.|batched_gdn$|concat_transpose$' \
+            --no-tests=error
+
+      - name: Feature / Qwen 3.6 AR c1 and c4 production smoke on ${{ matrix.device_name }}
+        run: |
+          build="$RUNNER_TEMP/rocmfp-build"
+          cmake --build "$build" --target dflash_server --parallel 8
+          uv run --project bench --frozen python -m bench.release.qwen36_amd validate-ci \
+            --profile "$QWEN_PROFILE"
+          uv run --project bench --frozen python -m bench.release.qwen36_amd smoke \
+            --profile "$QWEN_PROFILE" \
+            --server-bin "$build/dflash_server" \
+            --output-dir "$RUNNER_TEMP/qwen-ar-smoke"
+
+      - name: Upload Qwen 3.6 AR smoke evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: qwen-ar-smoke-${{ matrix.arch }}
+          path: ${{ runner.temp }}/qwen-ar-smoke
+          if-no-files-found: error
 
   build-windows:
     name: Build Windows (MSVC + CUDA, library + server targets)
@@ -397,7 +464,7 @@ jobs:
         shell: cmd
         run: cmake --build build --target dflash_common dflash_server backend_ipc_daemon test_platform_compat -j 4
 
-      - name: Run Windows platform smoke test
+      - name: Feature / platform compatibility
         working-directory: server
         shell: cmd
         run: ctest --test-dir build -R "^platform_compat$" --output-on-failure

.github/workflows/production-quality.yml

@@ -0,0 +1,68 @@
+name: Production quality
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  qwen-ar-amd:
+    name: Qwen 3.6 27B AR qualification (${{ matrix.device_name }})
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        include:
+          - device_name: Radeon AI PRO R9700
+            profile: qwen36-27b-q4-r9700-ar-c4
+            artifact_name: qwen-ar-r9700-qualification
+          - device_name: Strix Halo Radeon 8060S
+            profile: qwen36-27b-q4-strix-halo-ar-c4
+            artifact_name: qwen-ar-strix-qualification
+    runs-on: [self-hosted, lucebox3]
+    timeout-minutes: 120
+    env:
+      MODELS: ${{ vars.LUCEBOX_MODELS_DIR || '/opt/models' }}
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          submodules: recursive
+          token: ${{ secrets.SUBMODULE_PAT || secrets.GITHUB_TOKEN }}
+
+      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          version: "0.11.x"
+
+      - name: Qualify active Qwen 3.6 AR production recipe
+        timeout-minutes: 105
+        run: |
+          bench/release/qualify.sh "${{ matrix.profile }}" \
+            --output-dir "$RUNNER_TEMP/${{ matrix.artifact_name }}"
+
+      - name: Clean up qualification server
+        if: always()
+        run: |
+          uv run --project bench --frozen python -m bench.release.qualify \
+            --cleanup-pid-file "$RUNNER_TEMP/${{ matrix.artifact_name }}/server.pid"
+
+      - name: Publish qualification summary
+        if: always()
+        run: |
+          report="$RUNNER_TEMP/${{ matrix.artifact_name }}/qualification.md"
+          if [[ -f "$report" ]]; then
+            cat "$report" >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "Qualification stopped before a report was produced." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Retain complete qualification evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: ${{ matrix.artifact_name }}-${{ github.run_id }}
+          path: |
+            ${{ runner.temp }}/${{ matrix.artifact_name }}
+            !${{ runner.temp }}/${{ matrix.artifact_name }}/build/**
+          if-no-files-found: error
+          retention-days: 30

.github/workflows/speed-profile.yml

@@ -1,13 +1,4 @@
-name: Speed Profile
-
-# Report-only speed profile for the inference engine. Runs on the self-hosted
-# RTX 3090 (lucebox-rtx3090) on PRs that touch the engine or the optimizations, and on
-# manual dispatch. It NEVER blocks a PR (continue-on-error: true) — it publishes a
-# report to the run summary + uploads the JSON / markdown / nsys trace as artifacts.
-#
-# Why report-only: perf has run-to-run variance (thermals, clocks, scheduling).
-# Gating a merge on a noisy absolute number produces false failures. We surface the
-# trend first; a soft threshold can come later once a baseline + variance band exist.
+name: Exploratory speculative speed profile
 
 on:
   pull_request:
@@ -27,7 +18,7 @@ concurrency:
 
 jobs:
   speed-profile:
-    name: Speed profile (self-hosted RTX 3090, sm_86)
+    name: Exploratory profile (self-hosted RTX 3090, sm_86)
     runs-on: [self-hosted, lucebox-rtx3090]
     timeout-minutes: 30
     continue-on-error: true   # report-only: a slow/failed profile must not block the PR
@@ -46,7 +37,7 @@ jobs:
       TOKENIZER: ${{ vars.LUCEBOX_TOKENIZER || vars.LUCEBOX_SPEED_PROFILE_TOKENIZER || 'Qwen/Qwen3.6-27B' }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           submodules: recursive
           token: ${{ secrets.SUBMODULE_PAT || secrets.GITHUB_TOKEN }}
@@ -169,7 +160,7 @@ jobs:
         if: always() && steps.models.outputs.present == 'true'
         run: |
           if [ -f server/profile.md ]; then
-            { echo "## 🏎️ Speed profile"; echo ""; cat server/profile.md; } >> "$GITHUB_STEP_SUMMARY"
+            { echo "## Exploratory speculative speed profile"; echo ""; cat server/profile.md; } >> "$GITHUB_STEP_SUMMARY"
           else
             echo "Profiler produced no report (the run failed earlier — see logs)." >> "$GITHUB_STEP_SUMMARY"
           fi
@@ -205,9 +196,9 @@ jobs:
 
       - name: Upload artifacts (json + markdown + nsys trace)
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          name: speed-profile-${{ github.run_id }}
+          name: exploratory-speculative-profile-${{ github.run_id }}
           path: |
             server/profile.json
             server/profile.md