From e79ffe3dfbaf660a01994d5d06b36808a5f1981e Mon Sep 17 00:00:00 2001 From: Robert Esclapez Garcia Date: Wed, 29 Jul 2026 03:40:31 -0700 Subject: [PATCH 1/2] ggml-cuda: pad matmul weight rows to avoid cache-set aliasing Weight matrices whose packed row size is a multiple of 2048 bytes map every row onto the same L2 cache sets, so a matmul walking down a column thrashes a single set. On RDNA3.5 this shows up as lost bandwidth in the FFN and attention GEMMs. Add one cache line (128 B) to the row stride of such weights at buffer-init time so consecutive rows land in different sets. get_alloc_size reserves the extra bytes, and the loader uploads the packed file data with a strided 2D copy since the destination rows are no longer adjacent. The gaps are zeroed so they cannot hold NaNs. Quantized weights are excluded: 128 is not a multiple of their block size and would misalign the block-indexed kernels. The matmul paths need no change -- cuBLAS, mmf and mmvf all take the leading dimension from nb[1] already. The loader flags the eligible tensors with GGML_TENSOR_FLAG_PAD_ROWS, reusing llm_tensor_info_for() to tell a matmul weight from a bias or a norm; the token-embedding/output duplication rule it shares with the buffer-type selection is factored into resolve_tn_tensor() so the two cannot drift. Gated to RDNA3.5; disable with GGML_CUDA_NO_PAD_WEIGHTS. Co-Authored-By: Claude Opus 4.7 --- ggml/include/ggml.h | 11 ++++---- ggml/src/ggml-cuda/ggml-cuda.cu | 43 +++++++++++++++++++++++++++++ src/llama-model-loader.cpp | 49 +++++++++++++++++++++++++-------- 3 files changed, 86 insertions(+), 17 deletions(-) diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 8c522fd6f856..338443557cd3 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -648,11 +648,12 @@ extern "C" { // this tensor... enum ggml_tensor_flag { - GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph - GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph - GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters - GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up) - GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed + GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph + GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph + GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters + GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up) + GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed + GGML_TENSOR_FLAG_PAD_ROWS = 32, // ...is a matmul weight whose row stride a backend may pad to avoid cache-set aliasing }; enum ggml_tri_type { diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 9e23d64acddb..e50549fe4b0d 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -761,6 +761,33 @@ static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) { return ctx->dev_ptr; } +#define GGML_CUDA_ROW_ALIAS_STRIDE 2048 // rows of this size, or a multiple of it, map to the same cache sets +#define GGML_CUDA_ROW_PAD 128 // one cache line, added to the row stride to break the aliasing + +// quantized weights are excluded: GGML_CUDA_ROW_PAD is not a multiple of their block size and would +// misalign the block-indexed matmul kernels +static bool ggml_cuda_should_pad_weight(const ggml_tensor * tensor, int device) { + static const bool padding_disabled = getenv("GGML_CUDA_NO_PAD_WEIGHTS") != nullptr; + + if (padding_disabled || !(tensor->flags & GGML_TENSOR_FLAG_PAD_ROWS)) { + return false; + } + + if (!GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[device].cc)) { + return false; + } + + if (tensor->view_src != nullptr || ggml_is_quantized(tensor->type) || tensor->ne[1] <= 1) { + return false; + } + + return ggml_row_size(tensor->type, tensor->ne[0]) % GGML_CUDA_ROW_ALIAS_STRIDE == 0; +} + +static size_t ggml_cuda_padded_row_size(const ggml_tensor * tensor) { + return ggml_row_size(tensor->type, tensor->ne[0]) + GGML_CUDA_ROW_PAD; +} + static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; @@ -769,6 +796,17 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer return GGML_STATUS_SUCCESS; } + if (ggml_cuda_should_pad_weight(tensor, ctx->device)) { + tensor->nb[1] = ggml_cuda_padded_row_size(tensor); + tensor->nb[2] = tensor->nb[1]*tensor->ne[1]; + tensor->nb[3] = tensor->nb[2]*tensor->ne[2]; + + // the gaps between rows are never written, initialize them to 0 to avoid possible NaN values + ggml_cuda_set_device(ctx->device); + CUDA_CHECK(cudaMemset(tensor->data, 0, ggml_backend_buft_get_alloc_size(buffer->buft, tensor))); + return GGML_STATUS_SUCCESS; + } + if (ggml_is_quantized(tensor->type) && tensor->view_src == nullptr && ggml_backend_buffer_get_usage(buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) { // initialize padding to 0 to avoid possible NaN values const size_t original_size = ggml_nbytes(tensor); @@ -929,6 +967,11 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t } } + // reserve room for the row stride that ggml_backend_cuda_buffer_init_tensor will assign + if (ggml_cuda_should_pad_weight(tensor, buft_ctx->device)) { + size = ggml_cuda_padded_row_size(tensor)*ggml_nrows(tensor); + } + return size; } diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 43447f57d30b..746ab901b97f 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1038,6 +1038,17 @@ static ggml_backend_buffer_type_t select_weight_buft(const llama_hparams & hpara return nullptr; } +// some models use the token embedding tensor as the output, but since these are used in different layers and with different ops +// the tensor is duplicated +// to handle this, we check if the tensor is duplicated, and if so, we assume that it is being loaded as the output tensor +static llm_tensor resolve_tn_tensor(const LLM_TN_IMPL & tn, int flags) { + if (tn.tensor == LLM_TENSOR_TOKEN_EMBD && (flags & llama_model_loader::TENSOR_DUPLICATED)) { + return LLM_TENSOR_OUTPUT; + } + + return tn.tensor; +} + struct ggml_tensor * llama_model_loader::create_tensor( const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output, const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags) { @@ -1079,17 +1090,9 @@ struct ggml_tensor * llama_model_loader::create_tensor( throw std::runtime_error(format("missing tensor '%s'", tn.str().c_str())); } - // some models use the token embedding tensor as the output, but since these are used in different layers and with different ops - // the tensor is duplicated - // to handle this, we check if the tensor is duplicated, and if so, we assume that it is being loaded as the output tensor - llm_tensor tn_tensor = tn.tensor; - if (tn.tensor == LLM_TENSOR_TOKEN_EMBD && (flags & TENSOR_DUPLICATED)) { - tn_tensor = LLM_TENSOR_OUTPUT; - } - llm_tensor_info info; try { - info = llm_tensor_info_for(tn_tensor); + info = llm_tensor_info_for(resolve_tn_tensor(tn, flags)); } catch (const std::out_of_range & e) { throw std::runtime_error(format("missing tensor info mapping for %s", tn.str().c_str())); } @@ -1271,6 +1274,16 @@ struct ggml_tensor * llama_model_loader::create_tensor( struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur); ggml_set_name(tensor, ggml_get_name(cur)); + // tensors with a "weight" suffix are used as the src0 of the op that they map to; flag the ones + // that feed a matrix multiplication so that backends may pad their row stride + const bool is_weight = tn.suffix == nullptr || strcmp(tn.suffix, "weight") == 0; + if (is_weight) { + const ggml_op op = llm_tensor_info_for(resolve_tn_tensor(tn, flags)).op; + if (op == GGML_OP_MUL_MAT || op == GGML_OP_MUL_MAT_ID) { + tensor->flags |= GGML_TENSOR_FLAG_PAD_ROWS; + } + } + if (duplicated) { size_data += ggml_nbytes(cur); } else { @@ -1527,7 +1540,13 @@ bool llama_model_loader::load_all_data( } } - size_t n_size = ggml_nbytes(cur); + // the data in the file is packed, while the destination tensor may have a padded row + // stride, in which case the rows are uploaded with a strided 2D copy + const size_t packed_row_size = ggml_row_size(cur->type, cur->ne[0]); + const int64_t n_rows = ggml_nelements(cur) / cur->ne[0]; + const bool row_padded = cur->nb[1] != packed_row_size; + + const size_t n_size = row_padded ? packed_row_size*n_rows : ggml_nbytes(cur); if (use_mmap) { const auto & mapping = mappings.at(weight->idx); @@ -1554,6 +1573,8 @@ bool llama_model_loader::load_all_data( auto & mmap_used = mmaps_used[weight->idx]; mmap_used.first = std::min(mmap_used.first, weight->offs); mmap_used.second = std::max(mmap_used.second, weight->offs + n_size); + } else if (row_padded) { + ggml_backend_tensor_set_2d(cur, data, 0, packed_row_size, n_rows, cur->nb[1], packed_row_size); } else { ggml_backend_tensor_set(cur, data, 0, n_size); } @@ -1570,7 +1591,7 @@ bool llama_model_loader::load_all_data( } } else { // If upload_backend is valid load the tensor in chunks to pinned memory and upload the buffers asynchronously to the GPU. - if (upload_backend) { + if (upload_backend && !row_padded) { size_t offset = weight->offs; alignment = file->read_alignment(); size_t aligned_offset = offset & ~(alignment - 1); @@ -1626,7 +1647,11 @@ bool llama_model_loader::load_all_data( read_buf.resize(n_size); file->seek(weight->offs, SEEK_SET); file->read_raw(read_buf.data(), n_size); - ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size); + if (row_padded) { + ggml_backend_tensor_set_2d(cur, read_buf.data(), 0, packed_row_size, n_rows, cur->nb[1], packed_row_size); + } else { + ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size); + } if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) { throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur))); } From 0f6fe20559ffb6e390ad845d167fa164f0367e51 Mon Sep 17 00:00:00 2001 From: Robert Esclapez Garcia Date: Thu, 30 Jul 2026 06:55:57 -0700 Subject: [PATCH 2/2] ggml-cuda: make the weight row padding per-architecture and tunable The alias stride and the pad were compile-time constants gated to RDNA3.5. Both follow from the cache geometry -- line size times number of sets -- so another architecture needs different values, and neither CUDA nor HIP report enough to derive them: hipDeviceProp_t::l2CacheSize is documented as always returning 0, and set associativity is not exposed at all. Move the pair into cuda_device_info, resolved once per device from a table keyed on the compute capability, then overridden by GGML_CUDA_ROW_ALIAS_STRIDE and GGML_CUDA_ROW_PAD. Porting to another architecture becomes a table entry, and the values can be swept without rebuilding. Architectures with no entry keep packed rows, so each one is opted in only after being measured. Resolving once per device is also where the settings are validated: an alias stride that is not a power of two, or a pad that is a multiple of it and so leaves the rows aliasing, warns and falls back to no padding. GGML_CUDA_NO_PAD_WEIGHTS now resolves to a zero pad, leaving a single way for the padding to be off. Add the per-type check that was missing: ggml_cuda_should_use_mmf and ggml_cuda_should_use_mmvf reject a src0 whose strides are not a multiple of 2*type_size, so a pad that misses this would not fail or corrupt anything, it would quietly stop those kernels from being selected. Also correct the reason quantized weights are excluded. The matmul kernels do read the row stride, as nb[1]/type_size in mmq.cu and mmvq.cu; what a 128 byte pad breaks is that division being exact. Co-Authored-By: Claude Opus 4.7 --- ggml/src/ggml-cuda/common.cuh | 10 ++++ ggml/src/ggml-cuda/ggml-cuda.cu | 94 ++++++++++++++++++++++++++++----- 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 9845db10854f..7471f3cef932 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1126,6 +1126,15 @@ struct ggml_cuda_type_traits { ////////////////////// +// Row-stride padding for matrix multiplication weights. A weight whose packed row size is a +// multiple of alias_stride puts every row in the same cache sets, so pad bytes are added to the +// row stride to break the aliasing. Both values follow from the cache geometry (line size times +// number of sets), which neither CUDA nor HIP report, so they are tabulated per architecture. +struct ggml_cuda_row_pad_params { + size_t alias_stride; // packed row sizes that are a multiple of this alias in the cache + size_t pad; // bytes added to the row stride; 0 disables the padding +}; + struct ggml_cuda_device_info { int device_count; // number of (possibly virtual) devices exposed to the rest of ggml int physical_device_count; // number of physical CUDA devices actually present @@ -1144,6 +1153,7 @@ struct ggml_cuda_device_info { int physical_device; // backing physical CUDA device for this (virtual) device int physical_share_count; // number of (virtual) devices sharing this device's physical GPU int virtual_index; // index of this (virtual) device among those sharing its physical GPU + ggml_cuda_row_pad_params row_pad; // weight row-stride padding }; cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {}; diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index e50549fe4b0d..43b9779c64d6 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -220,6 +220,71 @@ static int ggml_cuda_parse_id(char devName[]) { } #endif // defined(GGML_USE_HIP) +// Weight row-stride padding per architecture. An architecture is opted in only once the padding +// has been measured on it; every other one gets {0, 0} and keeps packed rows. +static ggml_cuda_row_pad_params ggml_cuda_row_pad_params_for(int cc) { + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { + return { 2048, 128 }; // one cache line added to rows that alias every 2 KiB + } + + return { 0, 0 }; +} + +// Returns the environment override for name, or -1 when it is unset or not a number. +static int64_t ggml_cuda_row_pad_env(const char * name) { + const char * val = getenv(name); + if (!val || !val[0]) { + return -1; + } + + char * end = nullptr; + const int64_t parsed = strtoll(val, &end, 0); + if (*end != '\0' || parsed < 0) { + GGML_LOG_WARN("%s: ignoring %s=%s, expected a non-negative integer\n", __func__, name, val); + return -1; + } + + return parsed; +} + +// Resolves the padding for one device: the architecture default, then the environment overrides, +// then the sanity checks. Runs once per device so an invalid setting warns once, not per tensor. +static ggml_cuda_row_pad_params ggml_cuda_resolve_row_pad(int cc) { + if (getenv("GGML_CUDA_NO_PAD_WEIGHTS")) { + return { 0, 0 }; + } + + ggml_cuda_row_pad_params params = ggml_cuda_row_pad_params_for(cc); + + const int64_t alias_stride_override = ggml_cuda_row_pad_env("GGML_CUDA_ROW_ALIAS_STRIDE"); + const int64_t pad_override = ggml_cuda_row_pad_env("GGML_CUDA_ROW_PAD"); + if (alias_stride_override >= 0) { + params.alias_stride = alias_stride_override; + } + if (pad_override >= 0) { + params.pad = pad_override; + } + + if (params.pad == 0) { + return { 0, 0 }; + } + + if (params.alias_stride == 0 || (params.alias_stride & (params.alias_stride - 1)) != 0) { + GGML_LOG_WARN("%s: disabling weight row padding, alias stride %zu is not a power of two\n", + __func__, params.alias_stride); + return { 0, 0 }; + } + + // a pad that is itself a multiple of the alias stride leaves the rows aliasing + if (params.pad % params.alias_stride == 0) { + GGML_LOG_WARN("%s: disabling weight row padding, pad %zu is a multiple of alias stride %zu\n", + __func__, params.pad, params.alias_stride); + return { 0, 0 }; + } + + return params; +} + static ggml_cuda_device_info ggml_cuda_init() { ggml_cuda_device_info info = {}; @@ -376,6 +441,8 @@ static ggml_cuda_device_info ggml_cuda_init() { } #endif // defined(GGML_USE_HIP) + + info.devices[id].row_pad = ggml_cuda_resolve_row_pad(info.devices[id].cc); } if (ggml_cuda_highest_compiled_arch(GGML_CUDA_CC_TURING) >= GGML_CUDA_CC_TURING && !turing_devices_without_mma.empty()) { @@ -761,31 +828,30 @@ static void * ggml_backend_cuda_buffer_get_base(ggml_backend_buffer_t buffer) { return ctx->dev_ptr; } -#define GGML_CUDA_ROW_ALIAS_STRIDE 2048 // rows of this size, or a multiple of it, map to the same cache sets -#define GGML_CUDA_ROW_PAD 128 // one cache line, added to the row stride to break the aliasing - -// quantized weights are excluded: GGML_CUDA_ROW_PAD is not a multiple of their block size and would -// misalign the block-indexed matmul kernels +// Quantized weights are excluded: the matmul kernels read the row stride as nb[1]/type_size, an +// exact division that a pad which is not a multiple of the block size would break. static bool ggml_cuda_should_pad_weight(const ggml_tensor * tensor, int device) { - static const bool padding_disabled = getenv("GGML_CUDA_NO_PAD_WEIGHTS") != nullptr; + const ggml_cuda_row_pad_params & row_pad = ggml_cuda_info().devices[device].row_pad; - if (padding_disabled || !(tensor->flags & GGML_TENSOR_FLAG_PAD_ROWS)) { + if (row_pad.pad == 0 || !(tensor->flags & GGML_TENSOR_FLAG_PAD_ROWS)) { return false; } - if (!GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[device].cc)) { + if (tensor->view_src != nullptr || ggml_is_quantized(tensor->type) || tensor->ne[1] <= 1) { return false; } - if (tensor->view_src != nullptr || ggml_is_quantized(tensor->type) || tensor->ne[1] <= 1) { + // ggml_cuda_should_use_mmf and ggml_cuda_should_use_mmvf reject a src0 whose strides are not a + // multiple of 2*type_size, so a pad that misses this would silently cost those kernels + if (row_pad.pad % (2*ggml_type_size(tensor->type)) != 0) { return false; } - return ggml_row_size(tensor->type, tensor->ne[0]) % GGML_CUDA_ROW_ALIAS_STRIDE == 0; + return ggml_row_size(tensor->type, tensor->ne[0]) % row_pad.alias_stride == 0; } -static size_t ggml_cuda_padded_row_size(const ggml_tensor * tensor) { - return ggml_row_size(tensor->type, tensor->ne[0]) + GGML_CUDA_ROW_PAD; +static size_t ggml_cuda_padded_row_size(const ggml_tensor * tensor, int device) { + return ggml_row_size(tensor->type, tensor->ne[0]) + ggml_cuda_info().devices[device].row_pad.pad; } static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { @@ -797,7 +863,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer } if (ggml_cuda_should_pad_weight(tensor, ctx->device)) { - tensor->nb[1] = ggml_cuda_padded_row_size(tensor); + tensor->nb[1] = ggml_cuda_padded_row_size(tensor, ctx->device); tensor->nb[2] = tensor->nb[1]*tensor->ne[1]; tensor->nb[3] = tensor->nb[2]*tensor->ne[2]; @@ -969,7 +1035,7 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t // reserve room for the row stride that ggml_backend_cuda_buffer_init_tensor will assign if (ggml_cuda_should_pad_weight(tensor, buft_ctx->device)) { - size = ggml_cuda_padded_row_size(tensor)*ggml_nrows(tensor); + size = ggml_cuda_padded_row_size(tensor, buft_ctx->device)*ggml_nrows(tensor); } return size;