cuda: opt-in exact concurrency, byte-identical output regardless of neighbours or parks

.github/workflows/unsloth-pin-preflight.yml

@@ -28,16 +28,10 @@ permissions:
   contents: write
   issues: write
 
-# Two runs of the same ref probe the same pins against the same base, so the
-# second adds nothing and just competes for runners. On 08-04 a dispatch and the
-# schedule sat queued together for an hour. Newest wins: it sees the newest
-# pr-set.json.
-#
-# Per ref, though, not globally. This file also runs on any push that touches
-# pr-set.json, so with one shared group a push to a second branch cancelled the
-# first branch's run: observed on 09-03, where the run that would have said
-# whether a repin fixed the nightly was cancelled by an unrelated branch, and
-# the PR was left showing the failure from before the fix.
+# Two runs of the same ref probe the same pins against the same base, so the second only
+# competes for runners; newest wins, since it sees the newest pr-set.json. Per ref, not
+# globally: this also runs on any push touching pr-set.json, and with one shared group a push
+# to a second branch cancelled the first branch's run (09-03).
 concurrency:
   group: unsloth-pin-preflight-${{ github.ref }}
   cancel-in-progress: true
@@ -56,10 +50,8 @@ jobs:
         id: p
         run: |
           set -uo pipefail
-          # Everything below reports through `status`/`details`, so a death
-          # anywhere else leaves both empty and the alert blank: a red X on a
-          # scheduled run nobody opens. Report the abort through the same
-          # channel as a finding, so the repin bot sees a failure either way.
+          # everything below reports through `status`/`details`, so a death anywhere else
+          # leaves the alert blank; report the abort through the same channel as a finding
           trap 'rc=$?; if [ "$rc" != 0 ]; then {
                   echo "status=failure"
                   echo "details<<ALERT_EOF"
@@ -161,15 +153,10 @@ jobs:
               continue
             fi
             FILES="$(git diff --name-only --diff-filter=U | sed 's/^/  /')"
-            # `|| true` is load-bearing, not tidying. GitHub runs a `run:` block
-            # under `bash -e` whatever this script's own `set` line says, `head`
-            # closes the pipe after 20 lines, and pipefail then makes the whole
-            # assignment fail. So on 09-03 the step died right here, on the first
-            # real conflict, with `grep: write error: Broken pipe` and no alert:
-            # the one path this job exists to report was the one it could not
-            # survive. It only fires when the conflict diff is bigger than the
-            # 64 KiB pipe buffer, since a smaller one is written before `head`
-            # ever closes it, which is why most conflicts got reported fine.
+            # `|| true` is load-bearing: a `run:` block is `bash -e` whatever this script
+            # says, `head` closes the pipe after 20 lines, and pipefail then fails the whole
+            # assignment. On 09-03 that killed the step on the first real conflict with
+            # `grep: write error: Broken pipe`. Only fires above the 64 KiB pipe buffer.
             HUNKS="$(git diff --diff-filter=U -U0 2>/dev/null | grep -E '^\+|^-' | grep -vE '^(\+\+\+|---)' | head -20 || true)"
             git merge --abort 2>/dev/null
             PROBLEMS="${PROBLEMS}- \`${SRC}#${NUM}\` (\`${SHA:0:10}\`) does not merge onto \`${BASE}\` + the pins before it.\n\n  Conflicting files:\n\n\`\`\`\n${FILES}\n\`\`\`\n\n  <details><summary>conflict hunks</summary>\n\n\`\`\`diff\n${HUNKS}\n\`\`\`\n\n  </details>\n"
@@ -193,11 +180,9 @@ jobs:
               PROBLEMS="${PROBLEMS}- the merged tree builds, but \`scripts/unsloth/merge_checks.py\` found a resolution that is silently wrong. See the run log for file and line.\n"
             fi
 
-            # The other half of that question. merge_checks.py asks whether the
-            # tree contains something wrong; this asks whether it still contains
-            # what each pin carries. A pin that has rotted into a no-op, or an
-            # arch registration a resolution quietly dropped, is invisible to
-            # every other check here and to the compiler.
+            # the other half: merge_checks.py asks whether the tree contains something
+            # wrong, this asks whether it still contains what each pin carries. A pin rotted
+            # into a no-op is invisible to every other check here and to the compiler.
             if ! python3 ../scripts/unsloth/pin_contract.py --root . --base "$BASE" \
                  --pr-set ../scripts/unsloth/pr-set.json --report "${RUNNER_TEMP}/pin_contract.json" ; then
               PROBLEMS="${PROBLEMS}- the merged tree is missing code a pin carries. See the run log for the pin and file.\n"
@@ -207,12 +192,10 @@ jobs:
               PROBLEMS="${PROBLEMS}- pins upstream has taken over, safe to delete from \`pr-set.json\`:\n\n\`\`\`\n${NOTES}\n\`\`\`\n"
             fi
 
-            # A clean merge is not a compiling tree. On 09-03 ggml-org#27754
-            # merged with no conflicts at all and did not compile: upstream had
-            # added a parameter to build_attn_mha and the pin's new
-            # build_attn_sparse still called the old signature. Nothing above
-            # can see that. CPU only and the `llama` target only, which is where
-            # that translation unit lives; 59s cold at -j4 with no ccache.
+            # a clean merge is not a compiling tree: on 09-03 ggml-org#27754 merged with no
+            # conflicts and did not compile, upstream having added a parameter to
+            # build_attn_mha that the pin's build_attn_sparse still called without. CPU only,
+            # 59s cold at -j4 with no ccache.
             GATE_OK=1
             if ! cmake -B "${RUNNER_TEMP}/gate" -DCMAKE_BUILD_TYPE=Release \
                    -DGGML_CUDA=OFF -DLLAMA_BUILD_TESTS=ON -DLLAMA_BUILD_SERVER=OFF \
@@ -223,10 +206,8 @@ jobs:
               PROBLEMS="${PROBLEMS}- the pins merge cleanly and the merged tree does not compile. See the run log for the file and line; this is the failure that only shows up in the CUDA leg once the nightly has fanned out.\n"
             fi
 
-            # The last question, and the only one that needs a binary: does each
-            # feature we ship still work. Everything above is about the source.
-            # CPU only, because no runner in this pipeline has a GPU -- see the
-            # note in feature_matrix.py about what that does and does not prove.
+            # the only question that needs a binary: does each feature we ship still work.
+            # CPU only, since no runner here has a GPU; see the note in feature_matrix.py.
             if [ -n "$GATE_OK" ]; then
               if ! python3 ../scripts/unsloth/feature_matrix.py \
                    --build-dir "${RUNNER_TEMP}/gate" \

.github/workflows/unsloth-pr-set-lint.yml

@@ -128,9 +128,9 @@ jobs:
           done
           exit "$fail"
 
-      # A pin nobody decided about is the failure this whole file exists to stop.
-      # Being in `unchecked` with a reason is a fine answer; being in neither map
-      # is how DiffusionGemma went five weeks with no coverage and no record of it.
+      # a pin nobody decided about is the failure this file exists to stop: being in
+      # `unchecked` with a reason is fine, being in neither map is how DiffusionGemma went
+      # five weeks with no coverage and no record of it
       - name: Every pin is either checked or knowingly unchecked
         run: |
           set -euo pipefail

.github/workflows/unsloth-prebuilt.yml

@@ -1,3 +1,3 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
 
@@ -282,8 +282,8 @@
           # .github/workflows, which upstream history routinely does).
           if [ "$EXISTS" != "true" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
             git remote add upstream https://github.com/ggml-org/llama.cpp.git
-            # The upstream checkout below takes scripts/unsloth/ away. Copy the
-            # whole dir out, not file by file: see the note above the step.
+            # the upstream checkout below takes scripts/unsloth/ away; copy the whole dir
+            # out, not file by file, see the note above the step
             cp -r scripts/unsloth "${RUNNER_TEMP}/us"
             ADDITIVE_MERGE="${RUNNER_TEMP}/us/additive_merge.py"
             if [ "$(jq length <<<"$PRS")" != 0 ]; then
@@ -446,8 +446,7 @@
 
       # A bad pin resolution can still build fine, so it must be caught before the source artifact ships. See merge_checks.py.
       # Its own step, not more script in `resolve`: GitHub caps one workflow string at 21000 chars and that step is near it. See check_workflow_scalars.py.
-      # That is also why `resolve` copies all of scripts/unsloth/ to ${RUNNER_TEMP}/us in one line rather than one cp per script: every check added
-      # here would otherwise cost another line inside the capped block, and going over silently disables the whole workflow.
+      # That is also why `resolve` copies all of scripts/unsloth/ to ${RUNNER_TEMP}/us in one line rather than one cp per script: another line inside the capped block per check would eventually go over, which silently disables the whole workflow.
       - name: Check the merged tree for silently wrong resolutions
         if: ${{ env.MERGED_PINS == '1' }}
         run: |
@@ -457,13 +456,11 @@
             exit 1
           fi
 
-      # merge_checks.py asserts the ABSENCE of two known-bad shapes. This asserts the PRESENCE of what each pin carries, which is a different question and
-      # the one that goes unanswered when a pin rots into a no-op or a resolution quietly drops an arch registration. Free, so it runs before the compile gate.
+      # merge_checks.py asserts the ABSENCE of two known-bad shapes; this asserts the PRESENCE of what each pin carries, the question that goes unanswered when a pin rots into a no-op. Free, so it runs before the compile gate.
       - name: Check every pin still contributes what it carries
         if: ${{ env.MERGED_PINS == '1' }}
-        # Through env, never interpolated into the script: `prs` carries PR
-        # titles, which are third-party text, and `${{ }}` pastes them into the
-        # shell source before bash ever sees it.
+        # through env, never interpolated: `prs` carries PR titles, which are third-party
+        # text, and `${{ }}` pastes them into the shell source before bash sees it
         env:
           PRS: ${{ steps.r.outputs.prs }}
           BASE: ${{ steps.r.outputs.base }}
@@ -475,12 +472,8 @@
             exit 1
           fi
 
-      # The gap this closes, observed 09-03: ggml-org#27754 merged with zero conflicts and did not compile, because upstream had added a parameter to
-      # build_attn_mha and the pin's new build_attn_sparse still called the old signature. Nothing before this point can see that, and without it the release
-      # dies in the CUDA leg after the 38-job fan-out. CPU only: a cold `llama` build took 59s at -j4 with no ccache, against 20-60 minutes for a CUDA build.
-      # mtmd is in the gate because `llama` alone is not enough: observed 09-04, ggml-org#25731 built `llama` clean while tools/mtmd did not compile at all,
-      # upstream having made mtmd_image_preprocessor::preprocess const while the pin's Inkling subclass stayed non-const, so it overrode nothing and the
-      # vision and audio towers were abstract. Every vision pin lands in mtmd, so a gate that skips it cannot see the whole class.
+      # The gap this closes, observed 09-03: ggml-org#27754 merged with zero conflicts and did not compile, upstream having added a parameter to build_attn_mha that the pin's build_attn_sparse still called without. Without this the release dies in the CUDA leg after the 38-job fan-out. CPU only: a cold `llama` build took 59s at -j4, against 20-60 minutes for CUDA.
+      # mtmd is in the gate because `llama` alone is not enough: on 09-04 ggml-org#25731 built `llama` clean while tools/mtmd did not compile at all, upstream having made mtmd_image_preprocessor::preprocess const while the pin's Inkling subclass stayed non-const. Every vision pin lands in mtmd.
       - name: Compile gate (CPU, llama and mtmd targets)
         if: ${{ env.MERGED_PINS == '1' }}
         run: |

common/arg.cpp

@@ -1304,6 +1304,13 @@ bool common_params_parse(int argc, char ** argv, common_params & params, llama_e
             exit(0);
         }
         params.lr.init();
+
+        // [TAG_EXACT_CONCURRENCY] refuse a column bound that cannot cover a decode step before
+        // anything is loaded, rather than running with the guarantee quietly switched off
+        if (!common_exact_concurrency_init(ctx_arg.params)) {
+            ctx_arg.params = params_org;
+            return false;
+        }
     } catch (const std::invalid_argument & ex) {
         fprintf(stderr, "%s\n", ex.what());
         ctx_arg.params = params_org;

common/common.cpp

@@ -1,4 +1,5 @@
 #include "ggml.h"
+#include "ggml-backend.h"
 #include "gguf.h"
 
 #include "build-info.h"
@@ -1289,6 +1290,14 @@ struct common_init_result::impl {
 
 common_init_result::common_init_result(common_params & params, bool model_only) :
     pimpl(new impl{}) {
+    // [TAG_EXACT_CONCURRENCY] before any context exists, so one is never created under a figure
+    // the explicit bound does not cover; this also covers a caller that skipped
+    // common_params_parse(). On failure nothing is loaded.
+    if (!model_only && !common_exact_concurrency_init(params)) {
+        COM_ERR("%s", "LLAMA_EXACT_CONCURRENCY: refusing to load the model, see the error above\n");
+        return;
+    }
+
     auto mparams = common_model_params_to_llama(params);
     auto cparams = common_context_params_to_llama(params);
 
@@ -1433,6 +1442,78 @@ std::vector<llama_adapter_lora_ptr> & common_init_result::lora() {
     return pimpl->lora;
 }
 
+// [TAG_EXACT_CONCURRENCY]
+bool common_exact_concurrency() {
+    static const bool enabled = []() {
+        const char * val = getenv("LLAMA_EXACT_CONCURRENCY");
+        return val && atoi(val) != 0;
+    }();
+
+    return enabled;
+}
+
+// [TAG_EXACT_CONCURRENCY]
+int common_exact_decode_width(const common_params & params) {
+    const int64_t n_slots = std::max(1, params.n_parallel);
+
+    // draft tokens a slot carries into the verify ubatch, from the same place the speculation
+    // code takes its own width
+    const int64_t n_draft = std::max(0, (int) common_speculative_n_max(&params.speculative));
+
+    // the product is handed to a backend as an int; one that overflows is reported, not wrapped
+    const int64_t n_cols = n_slots*(1 + n_draft);
+
+    return n_cols > INT32_MAX ? -1 : (int) n_cols;
+}
+
+// [TAG_EXACT_CONCURRENCY]
+bool common_exact_concurrency_init(const common_params & params) {
+    if (!common_exact_concurrency()) {
+        return true;
+    }
+
+    // DFlash drafting turns causal attention off on its draft context, which the paged attention
+    // needs; say so instead of asserting in the graph. DSpark is the same implementation.
+    for (const auto type : params.speculative.types) {
+        if (type == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) {
+            COM_ERR("%s", "LLAMA_EXACT_CONCURRENCY does not support --spec-type draft-dflash or draft-dspark: both disable causal attention on the draft, which the paged attention needs\n");
+            return false;
+        }
+    }
+
+    const int n_cols = common_exact_decode_width(params);
+
+    if (n_cols < 0) {
+        COM_ERR("LLAMA_EXACT_CONCURRENCY: a decode step of %d slots with %d draft tokens each is too wide to report\n",
+                std::max(1, params.n_parallel), std::max(0, (int) common_speculative_n_max(&params.speculative)));
+        return false;
+    }
+
+    const char * bound = getenv("GGML_CUDA_BATCH_INVARIANT_MAX_COLS");
+    if (bound) {
+        const int max_cols = atoi(bound);
+        if (max_cols > 0 && max_cols < n_cols) {
+            COM_ERR("GGML_CUDA_BATCH_INVARIANT_MAX_COLS is %d but LLAMA_EXACT_CONCURRENCY needs at "
+                    "least %d to cover a decode step of %d slots, above which a matmul is left "
+                    "batched and its rows depend on the other rows in the ubatch. Raise it to %d, "
+                    "set it to 0 for no bound, or unset it to let it default to %d.\n",
+                    max_cols, n_cols, std::max(1, params.n_parallel), n_cols, n_cols);
+            return false;
+        }
+    }
+
+    // the batch splitter isolates prompts by width, so tell it how wide one sequence's decode step
+    // is. A context created later reports n_seq_max times that, which is n_cols again; reporting
+    // n_cols here too covers a caller that decodes first, or contexts it created earlier.
+    if (!llama_set_exact_decode_tokens((uint32_t) (n_cols / std::max(1, params.n_parallel))) ||
+        !llama_set_exact_decode_width((uint32_t) n_cols)) {
+        COM_ERR("%s", "LLAMA_EXACT_CONCURRENCY: the decode width could not be reported, see the error above\n");
+        return false;
+    }
+
+    return true;
+}
+
 common_init_result_ptr common_init_from_params(common_params & params, bool model_only) {
     common_init_result_ptr res(new common_init_result(params, model_only));
 

common/common.h

@@ -931,6 +931,17 @@ using common_init_result_ptr = std::unique_ptr<common_init_result>;
 
 common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false);
 
+// [TAG_EXACT_CONCURRENCY] true when LLAMA_EXACT_CONCURRENCY is set for this process
+bool common_exact_concurrency();
+
+// the widest ubatch a decode step can build here: one column per slot times one plus its draft
+// tokens. Under exact mode the CUDA column policy has to cover this, and derives its bound from it.
+int common_exact_decode_width(const common_params & params);
+
+// report that width to the CUDA backend, refusing a smaller explicit
+// GGML_CUDA_BATCH_INVARIANT_MAX_COLS; false if the configuration must not run
+bool common_exact_concurrency_init(const common_params & params);
+
 struct llama_model_params   common_model_params_to_llama  (      common_params & params);
 struct llama_context_params common_context_params_to_llama(const common_params & params);
 

ggml/include/ggml-cuda.h

@@ -38,6 +38,13 @@ GGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char
 GGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);
 
 GGML_BACKEND_API bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);
+
+// [TAG_EXACT_CONCURRENCY] report the widest ubatch a decode step of this process can build: one
+// column per slot times one plus its draft tokens. Under LLAMA_EXACT_CONCURRENCY the column policy
+// defaults to that instead of a fixed number, so --parallel or a wider draft cannot silently push a
+// decode above the bound. An explicit GGML_CUDA_BATCH_INVARIANT_MAX_COLS still wins. Call before
+// the first graph is computed; also available through ggml_backend_reg_get_proc_address().
+GGML_BACKEND_API void ggml_backend_cuda_set_exact_decode_width(int n_cols);
 GGML_BACKEND_API void ggml_backend_cuda_unregister_host_buffer(void * buffer);
 
 GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cuda_reg(void);

ggml/src/ggml-cann/ggml-cann.cpp

@@ -2656,6 +2656,10 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
             return true;
         case GGML_OP_FLASH_ATTN_EXT:
             {
+                // [TAG_EXACT_CONCURRENCY] src[5] is the page table, which only the CUDA backend reads
+                if (op->src[5]) {
+                    return false;
+                }
 #ifdef ASCEND_310P
                 // FA not support on 310p device
                 return false;

ggml/src/ggml-cpu/ggml-cpu.cpp

@@ -474,6 +474,10 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
             return ggml_is_contiguous(op->src[0]);
         case GGML_OP_SSM_SCAN:
             return ggml_get_op_params_i32(op, 0) == 1 || op->src[3]->ne[0] == 1;
+        // [TAG_EXACT_CONCURRENCY] note: FLASH_ATTN_EXT with src[5], the page table, is deliberately
+        // still accepted. The CPU ignores it and attends in physical order, but it is also the
+        // reference test-backend-ops compares the paged CUDA kernel against, and that test's mask
+        // selects exactly the listed cells. A KV layer cannot reach the CPU under the mode anyway.
         default:
             return true;
     }