From e25e52c2b48d22ff8e4d5325ddca8faa77ed67a4 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:35:57 +0200 Subject: [PATCH 01/21] fix(cpu): reject unsafe quantized copy layouts and correct row sizes --- ggml/src/ggml-cpu/copy.h | 48 +++++++++++++++++ ggml/src/ggml-cpu/ggml-cpu.c | 11 ++++ ggml/src/ggml-cpu/ggml-cpu.cpp | 4 ++ ggml/src/ggml-cpu/ops.cpp | 5 +- tests/CMakeLists.txt | 4 ++ tests/test-backend-ops.cpp | 4 ++ tests/test-cpu-quantized-copy.cpp | 86 +++++++++++++++++++++++++++++++ 7 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 ggml/src/ggml-cpu/copy.h create mode 100644 tests/test-cpu-quantized-copy.cpp diff --git a/ggml/src/ggml-cpu/copy.h b/ggml/src/ggml-cpu/copy.h new file mode 100644 index 000000000000..b0ebd94359c2 --- /dev/null +++ b/ggml/src/ggml-cpu/copy.h @@ -0,0 +1,48 @@ +#pragma once + +#include "ggml.h" + +// Quantized blocks cannot be transposed as individual scalar elements. Keep +// this check shared by scheduling and direct/planned CPU graph execution. +static inline bool ggml_cpu_quantized_rows_supported(const struct ggml_tensor * tensor) { + if (!ggml_is_quantized(tensor->type)) { + return true; + } + if (tensor->nb[0] != ggml_type_size(tensor->type)) { + return false; + } + // A one-block row can have nb[0] == nb[1], so strides alone cannot + // distinguish its invalid scalar transpose. Follow view provenance too. + int axis = 0; + for (const struct ggml_tensor * cur = tensor; cur != NULL; cur = cur->src[0]) { + switch (cur->op) { + case GGML_OP_TRANSPOSE: + if (axis < 2) { axis = 1 - axis; } + break; + case GGML_OP_PERMUTE: + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if (cur->op_params[i] == axis) { + axis = i; + break; + } + } + break; + case GGML_OP_VIEW: + case GGML_OP_RESHAPE: + if (axis != 0) { return false; } + break; + default: + return axis == 0; + } + } + return axis == 0; +} + +static inline bool ggml_cpu_copy_layout_supported(const struct ggml_tensor * op) { + if (op->op != GGML_OP_CPY && op->op != GGML_OP_CONT && op->op != GGML_OP_DUP) { + return true; + } + const struct ggml_tensor * src = op->src[0]; + const struct ggml_tensor * dst = op->op == GGML_OP_CPY ? op->src[1] : op; + return ggml_cpu_quantized_rows_supported(src) && ggml_cpu_quantized_rows_supported(dst); +} diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 358e402d000f..e6946a53f15d 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -12,6 +12,7 @@ #include "binary-ops.h" #include "vec.h" #include "ops.h" +#include "copy.h" #include "ggml.h" #include "common.h" @@ -3320,6 +3321,16 @@ struct ggml_threadpool * ggml_threadpool_new(struct ggml_threadpool_params * tpp enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { ggml_cpu_init(); + // Check before any worker can write output, including callers that bypass + // backend scheduling or execute a previously created graph plan. + for (int i = 0; i < cgraph->n_nodes; ++i) { + if (!ggml_cpu_copy_layout_supported(cgraph->nodes[i])) { + GGML_LOG_ERROR("%s: unsupported quantized copy layout for %s\n", + __func__, cgraph->nodes[i]->name); + return GGML_STATUS_FAILED; + } + } + GGML_ASSERT(cplan); GGML_ASSERT(cplan->n_threads > 0); GGML_ASSERT(cplan->work_size == 0 || cplan->work_data != NULL); diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 128883b41ce7..8cc1fe6c21ab 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -1,6 +1,7 @@ #include "ggml-backend.h" #include "ggml-backend-impl.h" #include "ggml-cpu.h" +#include "copy.h" #include "repack.h" #include "traits.h" #include "ggml-impl.h" @@ -421,6 +422,9 @@ static ggml_backend_buffer_t ggml_backend_cpu_device_buffer_from_host_ptr(ggml_b } static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + if (!ggml_cpu_copy_layout_supported(op)) { + return false; + } const struct ggml_tensor * src0 = op->src[0]; const struct ggml_tensor * src1 = op->src[1]; diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 372b815a2ea0..9cd2c8c6d2b0 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -1,4 +1,5 @@ #include "ops.h" +#include "copy.h" #include "ggml-cpu.h" #include "ggml-impl.h" @@ -373,7 +374,7 @@ static void ggml_compute_forward_dup_bytes( if (ggml_is_contiguous(dst)) { size_t id = 0; char * dst_ptr = (char *) dst->data; - const size_t rs = ne00 * type_size; + const size_t rs = ggml_row_size(src0->type, ne00); if (nb00 == type_size) { // src0 is contiguous on first dimension, copy by rows @@ -559,6 +560,8 @@ void ggml_compute_forward_dup( const ggml_compute_params * params, ggml_tensor * dst) { + GGML_ASSERT(ggml_cpu_copy_layout_supported(dst)); + const ggml_tensor * src0 = dst->src[0]; if (src0->type == dst->type) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d223898b42ed..e33531eaf8aa 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -243,6 +243,10 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC) endif() llama_build_and_test(test-gguf.cpp) llama_build_and_test(test-backend-ops.cpp) +if (GGML_CPU) + llama_build_and_test(test-cpu-quantized-copy.cpp) + target_link_libraries(test-cpu-quantized-copy PRIVATE ggml-cpu) +endif() llama_build_and_test(test-model-load-cancel.cpp LABEL "model") llama_build_and_test(test-autorelease.cpp LABEL "model") diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index dbf763381f6d..3a7cf3907d63 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8213,6 +8213,10 @@ static std::vector> make_test_cases_eval() { } // CPY - different src/dst shapes (reshaping via CPY) + for (auto type : {GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) { + test_cases.emplace_back(new test_cpy(type, type, {256, 8, 4, 1}, {64, 4, 32, 1}, {0, 2, 1, 3})); + } + // Use permutations of {3, 5, 7, 32}. Total elements: 3*5*7*32 = 3360. // Each src permutation is tested against canonical sorted and reverse dst (skip self). { diff --git a/tests/test-cpu-quantized-copy.cpp b/tests/test-cpu-quantized-copy.cpp new file mode 100644 index 000000000000..62c2491ceb4a --- /dev/null +++ b/tests/test-cpu-quantized-copy.cpp @@ -0,0 +1,86 @@ +#include "ggml.h" +#include "ggml-backend.h" +#include "ggml-cpu.h" + +#include +#include +#include +#include + +static void check_rejection(ggml_backend_t cpu, ggml_type type, ggml_op op, bool transpose_dst, int width = 256) { + auto * ctx = ggml_init({4*1024*1024, nullptr, true}); + const int heads = width == 32 ? 1 : 8; + auto * raw = ggml_new_tensor_4d(ctx, type, width, heads, 256, 1); + auto * perm = ggml_permute(ctx, raw, 0, 2, 1, 3); + auto * trans = ggml_transpose(ctx, perm); + auto * dst = ggml_new_tensor_4d(ctx, type, 256, width, heads, 1); + auto * out = op == GGML_OP_CONT ? ggml_cont(ctx, trans) : + op == GGML_OP_DUP ? ggml_dup(ctx, trans) : + transpose_dst ? ggml_cpy(ctx, dst, trans) : ggml_cpy(ctx, trans, dst); + std::vector source(ggml_nbytes(raw), 0x3C); + std::vector target(ggml_nbytes(out), 0xA5); + raw->data = perm->data = trans->data = transpose_dst ? target.data() : source.data(); + dst->data = transpose_dst ? source.data() : target.data(); + out->data = target.data(); + auto * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, out); + GGML_ASSERT(!ggml_backend_supports_op(cpu, out)); + GGML_ASSERT(ggml_backend_graph_compute(cpu, graph) == GGML_STATUS_FAILED); + auto plan = ggml_backend_graph_plan_create(cpu, graph); + GGML_ASSERT(plan != nullptr); + GGML_ASSERT(ggml_backend_graph_plan_compute(cpu, plan) == GGML_STATUS_FAILED); + ggml_backend_graph_plan_free(cpu, plan); + GGML_ASSERT(std::all_of(target.begin(), target.end(), [](unsigned char x) { return x == 0xA5; })); + ggml_free(ctx); +} + +// Exercise row permutations plus a differently shaped destination. The old +// generic copy multiplied the row length by block bytes instead of row bytes. +static void check_valid(ggml_backend_t cpu, ggml_type type, int threads) { + ggml_backend_cpu_set_n_threads(cpu, threads); + auto * ctx = ggml_init({4*1024*1024, nullptr, true}); + auto * raw = ggml_new_tensor_3d(ctx, type, 256, 8, 4); + auto * perm = ggml_permute(ctx, raw, 0, 2, 1, 3); + auto * dst = ggml_new_tensor_3d(ctx, type, 64, 4, 32); + auto * out = ggml_cpy(ctx, perm, dst); + const size_t bytes = ggml_nbytes(raw); + const size_t row = ggml_row_size(type, 256); + std::vector source(bytes), target(bytes + 64, 0xA5), expected(bytes); + for (size_t i = 0; i < bytes; ++i) { + source[i] = i % 251; + } + for (size_t y = 0; y < 8; ++y) { + for (size_t z = 0; z < 4; ++z) { + std::memcpy(expected.data() + (y*4 + z)*row, source.data() + (z*8 + y)*row, row); + } + } + raw->data = perm->data = source.data(); + dst->data = out->data = target.data(); + auto * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, out); + GGML_ASSERT(ggml_backend_supports_op(cpu, out)); + GGML_ASSERT(ggml_backend_graph_compute(cpu, graph) == GGML_STATUS_SUCCESS); + GGML_ASSERT(std::equal(expected.begin(), expected.end(), target.begin())); + GGML_ASSERT(std::all_of(target.begin() + bytes, target.end(), [](unsigned char x) { return x == 0xA5; })); + ggml_free(ctx); +} + +int main() { + auto cpu = ggml_backend_cpu_init(); + GGML_ASSERT(cpu); + for (auto type : {GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) { + for (auto op : {GGML_OP_CONT, GGML_OP_DUP, GGML_OP_CPY}) { + check_rejection(cpu, type, op, false); + check_rejection(cpu, type, op, false, 32); + } + check_rejection(cpu, type, GGML_OP_CPY, true); + check_rejection(cpu, type, GGML_OP_CPY, true, 32); + } + for (auto type : {GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) { + for (int threads : {1, 4}) { + check_valid(cpu, type, threads); + } + } + ggml_backend_free(cpu); + std::puts("quantized copy rejection and valid reshaped copies: PASS"); +} From 5ea6e1e1b152efccd5cc3114a93e2508976ce63a Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:36:19 +0200 Subject: [PATCH 02/21] fix(vulkan): bound Gemma 4 tuning to validated Lunar Lake workloads --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 25 ++++++++++++++-- src/llama-attention-policy.h | 24 ++++++++++++++++ src/llama-context.cpp | 22 ++++++++++++++ src/llama-cparams.h | 1 + src/llama-graph.cpp | 11 +++---- tests/CMakeLists.txt | 1 + tests/test-gemma4-attention-policy.cpp | 40 ++++++++++++++++++++++++++ 7 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 src/llama-attention-policy.h create mode 100644 tests/test-gemma4-attention-policy.cpp diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index f000b5043393..a30bb6cd1f7d 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -8753,6 +8753,14 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub } // Device tuning +static bool ggml_vk_gemma4_hybrid_device(const vk_device & device) { + // PCI ID observed on the validated Ultra 5 238V. Do not extrapolate the + // Linux measurements to other Xe2 devices or the Windows driver. + return device->vendor_id == VK_VENDOR_ID_INTEL && + device->properties.deviceID == 0x64a0 && + device->driver_id == vk::DriverId::eIntelOpenSourceMESA; +} + static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_t n, uint32_t k, ggml_type src0_type) { if (device->mmvq_mode == 1) { return true; @@ -8806,7 +8814,7 @@ static bool ggml_vk_should_use_mmvq(const vk_device& device, uint32_t m, uint32_ // Lunar Lake Xe2 benefits from integer-dot MMVQ for Q4_0 decode as // well. The generic Intel exclusion below was tuned on Alchemist. if (src0_type == GGML_TYPE_Q2_K || src0_type == GGML_TYPE_Q3_K || - src0_type == GGML_TYPE_Q4_0 || src0_type == GGML_TYPE_Q6_K) { + (src0_type == GGML_TYPE_Q4_0 && ggml_vk_gemma4_hybrid_device(device)) || src0_type == GGML_TYPE_Q6_K) { return true; } } @@ -17806,11 +17814,24 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg, return devices[device]; } +static bool ggml_backend_vk_gemma4_hybrid_device(ggml_backend_dev_t dev) { + auto * ctx = static_cast(dev->context); + return ggml_vk_gemma4_hybrid_device(ggml_vk_get_device(ctx->device)); +} + +static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { + UNUSED(reg); + if (strcmp(name, "ggml_backend_vk_gemma4_hybrid_device") == 0) { + return reinterpret_cast(ggml_backend_vk_gemma4_hybrid_device); + } + return nullptr; +} + static const struct ggml_backend_reg_i ggml_backend_vk_reg_i = { /* .get_name = */ ggml_backend_vk_reg_get_name, /* .get_device_count = */ ggml_backend_vk_reg_get_device_count, /* .get_device = */ ggml_backend_vk_reg_get_device, - /* .get_proc_address = */ NULL, + /* .get_proc_address = */ ggml_backend_vk_reg_get_proc_address, }; ggml_backend_reg_t ggml_backend_vk_reg() { diff --git a/src/llama-attention-policy.h b/src/llama-attention-policy.h new file mode 100644 index 000000000000..c7449affd7f6 --- /dev/null +++ b/src/llama-attention-policy.h @@ -0,0 +1,24 @@ +#pragma once + +#include "ggml.h" + +#include + +static inline bool llama_gemma4_hybrid_requested(const char * value) { + return value != nullptr && std::strcmp(value, "1") == 0; +} + +// Experimental policy: only the short-context, single-stream F16 workload +// measured on Lunar Lake is eligible. Other workloads retain stock attention. +static inline bool llama_gemma4_decompose_attention( + bool enabled, bool probing, bool flash_attn, bool gemma4, + ggml_type type_k, ggml_type type_v, + int64_t n_ctx, int64_t n_seq, int64_t head_dim, int64_t n_query) { + if (!enabled || probing || !flash_attn || !gemma4 || + type_k != GGML_TYPE_F16 || type_v != GGML_TYPE_F16 || + n_ctx > 2048 || n_seq != 1 || n_query < 1) { + return false; + } + return (head_dim == 256 || head_dim == 512) && + (n_query < 32 || (head_dim == 512 && n_query >= 64)); +} diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 8054163792c9..5c124687a722 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2,6 +2,7 @@ #include "ggml.h" #include "llama-arch.h" +#include "llama-attention-policy.h" #include "llama-graph.h" #include "llama-impl.h" #include "llama-batch.h" @@ -324,6 +325,27 @@ llama_context::llama_context( } } + // Resolve the optional device policy once. Partial offload and non-Vulkan + // devices retain stock behavior; no graph-time environment changes. + if (!hparams.vocab_only && llama_gemma4_hybrid_requested(getenv("LLAMA_VK_GEMMA4_HYBRID_FA")) && + (model.arch == LLM_ARCH_GEMMA4 || model.arch == LLM_ARCH_GEMMA4_ASSISTANT) && + cparams.offload_kqv && model.split_mode() != LLAMA_SPLIT_MODE_TENSOR) { + cparams.gemma4_hybrid_fa = true; + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + auto dev = model.dev_layer(il); + auto reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + using device_policy_fn = bool (*)(ggml_backend_dev_t); + auto policy = reg ? reinterpret_cast( + ggml_backend_reg_get_proc_address(reg, "ggml_backend_vk_gemma4_hybrid_device")) : nullptr; + if (!policy || !policy(dev)) { + cparams.gemma4_hybrid_fa = false; + break; + } + } + LLAMA_LOG_INFO("%s: Gemma 4 hybrid device eligibility = %s (experimental F16, single-sequence, <=2048 context)\n", + __func__, cparams.gemma4_hybrid_fa ? "yes" : "no"); + } + // init the memory module if (!hparams.vocab_only) { llama_memory_params params_mem = { diff --git a/src/llama-cparams.h b/src/llama-cparams.h index 288f0d7cd82d..32fb85fd98b1 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -38,6 +38,7 @@ struct llama_cparams { bool offload_kqv; bool flash_attn; bool auto_fa; + bool gemma4_hybrid_fa = false; // validated device; opt-in, immutable for this context bool fused_gdn_ar; // use fused gated delta net (autoregressive) bool fused_gdn_ch; // use fused gated delta net (chunked) bool auto_fgdn; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 2eca6b83692e..397c6361dc8b 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -4,6 +4,7 @@ #include "llama-model.h" #include "llama-batch.h" #include "llama-cparams.h" +#include "llama-attention-policy.h" #include "llama-kv-cache.h" #include "llama-kv-cache-iswa.h" @@ -2192,14 +2193,10 @@ ggml_tensor * llm_graph_context::build_attn_mha( const bool k_is_tbq = k->type == GGML_TYPE_TBQ3_0 || k->type == GGML_TYPE_TBQ4_0; const bool v_is_tbq = v->type == GGML_TYPE_TBQ3_0 || v->type == GGML_TYPE_TBQ4_0; const bool any_tbq = k_is_tbq || v_is_tbq; - // On Intel Xe2, Gemma 4's D=512 global prompt attention and small-batch - // decode are faster as the decomposed matmul/softmax graph, while D=256 - // local prompt layers can still benefit from Flash Attention. - // Keep this opt-in because graph construction is backend-agnostic. const bool gemma4_model = arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT; - const bool gemma4_hybrid_fa = - gemma4_model && getenv("LLAMA_VK_GEMMA4_HYBRID_FA") != nullptr && - (q->ne[2] < 32 || (q->ne[0] >= 512 && q->ne[2] >= 64)); + const bool gemma4_hybrid_fa = llama_gemma4_decompose_attention( + cparams.gemma4_hybrid_fa, cparams.auto_fa, cparams.flash_attn, gemma4_model, + k->type, v->type, cparams.n_ctx_seq, cparams.n_seq_max, q->ne[0], q->ne[2]); const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr && !gemma4_hybrid_fa; const enum ggml_type tbq_attn_type = use_flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e33531eaf8aa..4e7fd15f178a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -243,6 +243,7 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC) endif() llama_build_and_test(test-gguf.cpp) llama_build_and_test(test-backend-ops.cpp) +llama_build_and_test(test-gemma4-attention-policy.cpp) if (GGML_CPU) llama_build_and_test(test-cpu-quantized-copy.cpp) target_link_libraries(test-cpu-quantized-copy PRIVATE ggml-cpu) diff --git a/tests/test-gemma4-attention-policy.cpp b/tests/test-gemma4-attention-policy.cpp new file mode 100644 index 000000000000..cfd5e3097d80 --- /dev/null +++ b/tests/test-gemma4-attention-policy.cpp @@ -0,0 +1,40 @@ +#include "../src/llama-attention-policy.h" + +#include +#include + +int main() { + GGML_ASSERT(llama_gemma4_hybrid_requested("1")); + for (const char * value : std::initializer_list{nullptr, "", "0", "false", "true", "01", "1x"}) { + GGML_ASSERT(!llama_gemma4_hybrid_requested(value)); + } + for (int head : {256, 512}) { + for (int query : {1, 2, 16, 31, 32, 63, 64, 128, 512}) { + const bool expected = query < 32 || (head == 512 && query >= 64); + GGML_ASSERT(llama_gemma4_decompose_attention(true, false, true, true, + GGML_TYPE_F16, GGML_TYPE_F16, 2048, 1, head, query) == expected); + // Probing must retain FA even for the one-token reserve graph. + GGML_ASSERT(!llama_gemma4_decompose_attention(true, true, true, true, + GGML_TYPE_F16, GGML_TYPE_F16, 2048, 1, head, query)); + for (auto type : {GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, GGML_TYPE_F32, GGML_TYPE_TBQ4_0}) { + GGML_ASSERT(!llama_gemma4_decompose_attention(true, false, true, true, + GGML_TYPE_F16, type, 2048, 1, head, query)); + GGML_ASSERT(!llama_gemma4_decompose_attention(true, false, true, true, + type, GGML_TYPE_F16, 2048, 1, head, query)); + } + for (int context : {2049, 8192, 16384, 32768}) { + GGML_ASSERT(!llama_gemma4_decompose_attention(true, false, true, true, + GGML_TYPE_F16, GGML_TYPE_F16, context, 1, head, query)); + } + GGML_ASSERT(!llama_gemma4_decompose_attention(true, false, true, true, + GGML_TYPE_F16, GGML_TYPE_F16, 2048, 4, head, query)); + } + } + GGML_ASSERT(!llama_gemma4_decompose_attention(false, false, true, true, + GGML_TYPE_F16, GGML_TYPE_F16, 2048, 1, 512, 1)); + GGML_ASSERT(!llama_gemma4_decompose_attention(true, false, false, true, + GGML_TYPE_F16, GGML_TYPE_F16, 2048, 1, 512, 1)); + GGML_ASSERT(!llama_gemma4_decompose_attention(true, false, true, false, + GGML_TYPE_F16, GGML_TYPE_F16, 2048, 1, 512, 1)); + std::puts("Gemma 4 attention boundaries and fallback policy: PASS"); +} From cab602c7d29377e7f72b4f5da2b04d05d28962eb Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:36:19 +0200 Subject: [PATCH 03/21] test(xe2): add pinned models and end-to-end validation harness --- README.md | 2 + docs/backend/VULKAN-GEMMA4-INTEL-XE2.md | 41 +++- scripts/xe2/README.md | 67 ++++++ scripts/xe2/fetch-models.py | 86 +++++++ scripts/xe2/models.json | 29 +++ scripts/xe2/summarize.py | 49 ++++ scripts/xe2/validate.py | 286 ++++++++++++++++++++++++ tests/CMakeLists.txt | 1 + tests/test-gemma4-device.cpp | 133 +++++++++++ 9 files changed, 688 insertions(+), 6 deletions(-) create mode 100644 scripts/xe2/README.md create mode 100644 scripts/xe2/fetch-models.py create mode 100644 scripts/xe2/models.json create mode 100644 scripts/xe2/summarize.py create mode 100644 scripts/xe2/validate.py create mode 100644 tests/test-gemma4-device.cpp diff --git a/README.md b/README.md index 92c59571f926..aae5e0e4801b 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ Every entry below carries a **Why** — the reason it exists downstream. That is | DFlash speculative decoding | Block-diffusion drafter integration (`LLM_ARCH_DFLASH`, `--spec-type dflash`, `llama_set_dflash`, CUDA kernels for partial-accept feature extraction). Designed against the [z-lab DFlash](https://github.com/z-lab/dflash) reference for Gemma4 31B targets. | Active product bet: diffusion drafting to lift Gemma4 decode throughput on the 3090/P5200 fleet. Upstream has no diffusion-drafter framework to extend. | No | | Gemma4 MTP speculative | Vendored upstream PR [#23398](https://github.com/ggml-org/llama.cpp/pull/23398) (`gemma4-assistant` arch + `--spec-type draft-mtp`) ahead of upstream merge so the gemma-4-12b-qat-mtp preset can ship on titan. Retires when #23398 merges upstream and flows through a normal master sync. | Ships a measured 1.66× sampled / 3.02× greedy decode speedup on titan months before upstream review completes. Explicitly temporary. | [#23398](https://github.com/ggml-org/llama.cpp/pull/23398) | | D=512 FA vec kernels | CUDA flash-attention vec-kernel instances for head size 512 with matched quantized KV (`q4_0`/`q8_0`), dispatch-gated to `gqa_ratio <= 4`; deployment-shape (Gemma4 MQA-16) correctness + perf cases in `test-backend-ops`. | Low-GQA D=512 shapes skip the per-step F16 dequant staging (up to 2× per-op). The gate keeps Gemma4's MQA-16 global layers on the faster TILE/MMA path — measured on both sm_61 and sm_86. | No | +| Lunar Lake Gemma 4 tuning | Opt-in hybrid F16 attention for single-sequence contexts up to 2K and Q4_0 MMVQ dispatch, restricted to Intel `8086:64a0` with Mesa Vulkan. [Notes](docs/backend/VULKAN-GEMMA4-INTEL-XE2.md), [validation](scripts/xe2/README.md). | Keep device-specific speed experiments bounded to supported cache types and measured shapes; preserve stock selection elsewhere. | No | +| CPU quantized copy safety | Reject unsupported quantized transposes before direct/planned execution; correct row-byte counts for valid reshaped copies. | Hybrid attention exposed a fallback copy that wrote beyond its tensor allocation. | No | | Router-mode robustness | `llama-server` router detects worker crashes via `subprocess_alive` polling; fixes hardcoded proxy timeout | Router mode is the fleet's deployment shape (multi-model boxes); a hung worker must surface as an error, not a stuck request. | [#22003](https://github.com/ggml-org/llama.cpp/pull/22003) | | Tool-calling resilience | Fallback tool-call parser and skip non-`function` tool types so non-conforming models still work | heierchat exposes tools to arbitrary local models; strict parsing turned every malformed call into a hard failure. | No | | Developer-role remap | `--remap-developer-role` flag merges `developer` messages into the system prompt for templates that reject duplicates | OpenAI-client compatibility: clients that emit `developer` roles hit template errors on Gemma-family chat templates. | No | diff --git a/docs/backend/VULKAN-GEMMA4-INTEL-XE2.md b/docs/backend/VULKAN-GEMMA4-INTEL-XE2.md index 56991503161f..ebfd09b84e65 100644 --- a/docs/backend/VULKAN-GEMMA4-INTEL-XE2.md +++ b/docs/backend/VULKAN-GEMMA4-INTEL-XE2.md @@ -1,10 +1,37 @@ # Gemma 4 on Intel Xe2 Vulkan: architecture, Flash Attention, and tuning -Status: engineering notes and local measurements, 2026-08-29. The target used for +Status: historical measurements from 2026-08-29; safety hardening 2026-09-05. +The target used for measurements is a Core Ultra 5 238V / Lunar Lake Arc 130V/140V iGPU, Linux `xe`, Mesa 26.1.5 ANV, Vulkan 1.4, 32-wide subgroups, 48 KiB shared memory, FP16/BF16, integer dot product, KHR cooperative matrix, and unified system memory. +## Safety and validation update (2026-09-05) + +The hybrid experiment now requires the exact value +`LLAMA_VK_GEMMA4_HYBRID_FA=1`; unset, `0`, and other values disable it. +Eligibility is resolved once per context and requires all model layers on the +validated Lunar Lake PCI device `8086:64a0` with Mesa Vulkan and KV offload. +Only F16 K **and** V, a single sequence, and configured context at most 2048 +tokens use the experimental policy. Longer contexts, multiple slots, partial +offload, other backends/devices, and quantized caches retain stock selection. +Flash Attention's `auto` capability probe always runs without hybrid rewriting. + +This restriction fixes a real unsafe combination: hybrid decomposition with +quantized V created a transposed quantized copy unsupported by Vulkan. The CPU +fallback treated blocks as scalar elements and wrote beyond the output tensor. +CPU scheduling and direct/planned graph execution now reject unsupported +quantized copy layouts before writing. Valid reshaped quantized copies also +have a corrected row-byte calculation. The Q4_0 MMVQ exception is restricted +to this Lunar Lake/Mesa device; prior dispatch is preserved elsewhere. + +The benchmark tables below are historical, not fresh acceptance evidence. +Their original Xe2 model hashes were not recorded. The pinned, checksummed +artifacts and repeatable validation commands in [`scripts/xe2`](../../scripts/xe2/README.md) +establish a new baseline without claiming byte-identical reproduction. +Keep hybrid opt-in until model parity, lifecycle tests, depth benchmarks, and +the one-hour soak pass. Do not infer long-context gains from the short runs. + This document is about making each usable Gemma 4 member perform well. It does not recommend replacing one family member with another: the dense, unified, MoE, PLE, multimodal, and MTP variants serve different purposes. @@ -190,7 +217,8 @@ should remain in a deployment A/B; the 26B prompt result clearly favored adaptiv ### Adaptive attention retained in this branch When `LLAMA_VK_GEMMA4_HYBRID_FA=1` is set and `--flash-attn on` is requested, -the graph builder applies the policy to both `gemma4` and `gemma4-assistant`: +the graph builder applies the policy to eligible `gemma4` and +`gemma4-assistant` contexts described in the safety update above: - fewer than 32 query tokens: decomposed attention for decode and speculative verification; @@ -199,9 +227,9 @@ the graph builder applies the policy to both `gemma4` and `gemma4-assistant`: - 64 or more query tokens: FA for D=256 local layers and decomposed attention for D=512 global layers. -The environment gate is intentional: graph construction does not know which -backend will ultimately execute each node, so making this Xe2 result the default -would risk regressing CUDA, Metal, or a future faster Vulkan driver. This is a +The environment gate is intentional: the context checks backend/device +eligibility, but making this experimental result the default would still risk +regression with future drivers or unmeasured workloads. This is a shape-aware scheduler win rather than a new mathematical attention algorithm. Absolute iGPU rates move with package power, temperature, display activity, and @@ -288,7 +316,8 @@ prefill; compare with all-off for 12B): ```bash LLAMA_VK_GEMMA4_HYBRID_FA=1 ./build-vulkan/bin/llama-server \ --model MODEL.gguf --n-gpu-layers all \ - --flash-attn on --ubatch-size 512 + --flash-attn on --ubatch-size 512 --ctx-size 2048 --parallel 1 \ + --cache-type-k f16 --cache-type-v f16 ``` All-off baseline: diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md new file mode 100644 index 000000000000..096cd44772e2 --- /dev/null +++ b/scripts/xe2/README.md @@ -0,0 +1,67 @@ +# Lunar Lake validation + +Target: Ultra 5 238V, Intel `8086:64a0`, Linux/Mesa Vulkan. Hybrid attention +remains experimental and off by default. The exact value `1` enables its +eligibility check; `0` and unset disable it. It only applies with F16 K/V, +one sequence, full GPU layer/KV offload, and context at most 2048 tokens. +Other configurations retain stock attention selection. The Q4_0 MMVQ change +is limited to the validated PCI device and Mesa driver. + +`models.json` pins fresh validation artifacts, including SHA256 and HF revision. +These match filenames in the historical notes, whose original Xe2 hashes were +not recorded. Do not compare new results to those tables as identical artifacts. +GGUFs remain outside the repository under `$GGUFS` or `--models-dir`. + +```bash +python scripts/xe2/fetch-models.py --models-dir "$GGUFS" +cmake --build build-vulkan -j 4 --target llama-server llama-bench \ + test-backend-ops test-cpu-quantized-copy test-gemma4-attention-policy test-gemma4-device +ctest --test-dir build-vulkan --output-on-failure \ + -R 'test-(cpu-quantized-copy|gemma4-attention-policy)$' +./build-vulkan/bin/test-backend-ops test -b Vulkan0 -o CPY,CONT +./build-vulkan/bin/test-backend-ops test -b Vulkan0 -o MUL_MAT -p 'type_a=q4_0' +./build-vulkan/bin/test-backend-ops test -b Vulkan0 -o FLASH_ATTN_EXT \ + -p 'hsk=(256|512),.*type_K=(f16|q8_0|q4_0),type_V=(f16|q8_0|q4_0)' +python scripts/xe2/validate.py parity --output /tmp/xe2-validation +python scripts/xe2/validate.py smoke --output /tmp/xe2-validation +python scripts/xe2/validate.py bench --output /tmp/xe2-validation +python scripts/xe2/summarize.py /tmp/xe2-validation +python scripts/xe2/validate.py soak --output /tmp/xe2-validation +``` + +Run GPU stages sequentially, on AC, with the same power profile and desktop +workload. Benchmarks rotate FA-off, FA-on, and hybrid ordering for five runs +at each of 2K/8K/16K/32K context, for both targets. Above 2K the hybrid command +intentionally follows stock FA; those runs measure fallback behavior, not a +long-context optimization. Retain only gains larger than observed variation. +Each benchmark command measures pp512, tg64, and combined pp512+tg64, starting +at depth `context - 576`. The combined test ends at the named context envelope; +standalone pp/tg finish slightly earlier. Raw JSON records the actual sizes. + +Parity uses full CPU logits at 1/2/16, 31/32, 63/64, 128, and 1152 prompt tokens; +F16 GPU paths must satisfy the backend FA test tolerance (NMSE < 5e-4). +Top-token agreement is recorded separately for review. F16/Q8_0/Q4_0 caches +must remain finite and reproduce logits after dirtying, freeing, and reusing +a suffix beyond the sliding window. CPU reference files belong to the exact +manifest model and are generated afresh by the runner. + +Smoke checks repeated-prefix greedy tokens, the unset/zero switch, quantized +cache fallback, thinking modes, a complete tool-call/result cycle, stream +cancellation and slot reuse, and the matching 12B MTP assistant. The default +soak lasts a total hour: 12B with MTP and one slot, then 26B with four concurrent +slots. Logs retain responses, timing, temperature, power profile, and RSS. +Inspect warm RSS trends and MTP engagement/acceptance in server logs before +accepting the soak; request success alone does not prove bounded memory or +that speculative decoding engaged. + +`--model-ids`, `--depths`, `--repetitions`, and `--soak-seconds` permit targeted +debugging. Shortened runs are not the full acceptance suite. Outputs include +commands, model hashes, Git revision/diff, device descriptions, and subprocess +logs. Any missing model, checksum mismatch, failed assertion, invalid response, +or subprocess failure stops the relevant stage with a nonzero exit. + +For CPU memory-safety validation, configure a separate Debug build with +`GGML_SANITIZE_ADDRESS=ON` and `GGML_SANITIZE_UNDEFINED=ON`, then run +`test-cpu-quantized-copy`. This test checks rejected operations through both +direct and planned execution, and verifies valid reshaped copies with canaries +and multiple threads. diff --git a/scripts/xe2/fetch-models.py b/scripts/xe2/fetch-models.py new file mode 100644 index 000000000000..71fde1cb1e3c --- /dev/null +++ b/scripts/xe2/fetch-models.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Fetch the pinned validation artifacts; publish files only after SHA256 passes.""" +import argparse +import concurrent.futures +import hashlib +import json +import os +from pathlib import Path +import time +import urllib.request + + +def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def fetch(model, root, workers): + path = root / Path(model["file"]).name + if path.exists(): + if path.stat().st_size != model["size"] or digest(path) != model["sha256"]: + raise RuntimeError(f"Existing file does not match manifest: {path}") + print(f"verified {model['id']}: {path}", flush=True) + return + # A separate partial file avoids treating interrupted downloads as models. + partial = path.with_suffix(".download") + url = f"https://huggingface.co/{model['repo']}/resolve/{model['revision']}/{model['file']}" + chunk_size = 32 * 1024 * 1024 + with partial.open("w+b") as stream: + stream.truncate(model["size"]) + fd = stream.fileno() + + def chunk(start): + end = min(model["size"], start + chunk_size) - 1 + for attempt in range(3): + try: + request = urllib.request.Request( + f"{url}?download=true&offset={start}", + headers={"Range": f"bytes={start}-{end}"}, + ) + with urllib.request.urlopen(request, timeout=120) as response: + if response.status != 206 or response.headers.get("Content-Range") != f"bytes {start}-{end}/{model['size']}": + raise RuntimeError("Server did not honor byte range") + position = start + while data := response.read(min(1024 * 1024, end + 1 - position)): + view = memoryview(data) + while view: + written = os.pwrite(fd, view, position) + if written <= 0: + raise OSError("Short disk write") + position += written + view = view[written:] + if position != end + 1: + raise RuntimeError("Truncated download") + return + except Exception: + if attempt == 2: + raise + time.sleep(attempt + 1) + + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: + for i, _ in enumerate(pool.map(chunk, range(0, model["size"], chunk_size)), 1): + if i % 16 == 0: + print(f"{model['id']}: {min(i * chunk_size, model['size'])}/{model['size']} bytes", flush=True) + os.fsync(fd) + if digest(partial) != model["sha256"]: + raise RuntimeError(f"SHA256 mismatch: {partial}") + partial.rename(path) + print(f"verified {model['id']}: {model['sha256']}", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--models-dir", type=Path, default=os.environ.get("GGUFS")) + parser.add_argument("--workers", type=int, default=8) + args = parser.parse_args() + if args.models_dir is None or args.workers < 1: + parser.error("Set GGUFS or --models-dir; --workers must be positive") + args.models_dir.mkdir(parents=True, exist_ok=True) + manifest = json.loads(Path(__file__).with_name("models.json").read_text()) + for model in manifest["models"]: + fetch(model, args.models_dir, args.workers) + + +if __name__ == "__main__": + main() diff --git a/scripts/xe2/models.json b/scripts/xe2/models.json new file mode 100644 index 000000000000..f2a2609c80ee --- /dev/null +++ b/scripts/xe2/models.json @@ -0,0 +1,29 @@ +{ + "provenance": "Fresh validation baseline. Matches documented filenames; original Xe2 artifact hashes were not recorded.", + "models": [ + { + "id": "12b", + "repo": "unsloth/gemma-4-12B-it-qat-GGUF", + "revision": "980b060c40a8539ac159e0501a3e0f66a6365af3", + "file": "gemma-4-12B-it-qat-UD-Q4_K_XL.gguf", + "sha256": "90fd44e29e0d7cffeb0fd00dc73cfdab9ed0b0e95306ecf7821ea634c940c370", + "size": 6716356800 + }, + { + "id": "12b-mtp", + "repo": "unsloth/gemma-4-12B-it-qat-GGUF", + "revision": "980b060c40a8539ac159e0501a3e0f66a6365af3", + "file": "MTP/mtp-gemma-4-12B-it-Q4_0.gguf", + "sha256": "fcb35dea42c71333db904cee11baac525c9ef872818ee3753f6cb156f3c6f4f6", + "size": 253708800 + }, + { + "id": "26b", + "repo": "unsloth/gemma-4-26B-A4B-it-GGUF", + "revision": "c099eb48e663fd284577b04978a94ffccb261841", + "file": "gemma-4-26B-A4B-it-UD-IQ4_XS.gguf", + "sha256": "babd1e389d386352f71600765d37390f7dc993fbfad6725caccf996ffe34aecf", + "size": 13597177568 + } + ] +} diff --git a/scripts/xe2/summarize.py b/scripts/xe2/summarize.py new file mode 100644 index 000000000000..376d615f242c --- /dev/null +++ b/scripts/xe2/summarize.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Summarize benchmark spread and paired hybrid differences without hiding losses.""" +import argparse +from collections import defaultdict +import json +import math +from pathlib import Path +import re +import statistics + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("results", type=Path) + args = parser.parse_args() + groups = defaultdict(dict) + pattern = re.compile(r"(12b|26b)-c(\d+)-(off|on|hybrid)-r(\d+)\.stdout") + for path in args.results.glob("*.stdout"): + match = pattern.fullmatch(path.name) + if not match: + continue + model, context, mode, repeat = match.groups() + for row in json.loads(path.read_text()): + test = f"pp{row['n_prompt']}+tg{row['n_gen']}" + groups[(model, int(context), test, mode)][int(repeat)] = float(row["avg_ts"]) + print("| Model | Context envelope | Test | Mode | Runs | Mean token/s | SD |") + print("|---|---:|---|---|---:|---:|---:|") + for (model, context, test, mode), runs in sorted(groups.items()): + values = list(runs.values()) + deviation = statistics.stdev(values) if len(values) > 1 else 0 + print(f"| {model} | {context} | {test} | {mode} | {len(values)} | {statistics.mean(values):.2f} | {deviation:.2f} |") + print("\nPaired hybrid comparisons (95% t interval for exactly five matching runs):\n") + for (model, context, test, mode), runs in sorted(groups.items()): + if mode != "hybrid": + continue + for baseline in ("off", "on"): + other = groups.get((model, context, test, baseline), {}) + repeats = sorted(runs.keys() & other.keys()) + if len(repeats) != 5: + print(f"- {model} {context} {test} vs {baseline}: incomplete ({len(repeats)}/5 paired runs)") + continue + differences = [100 * (runs[r] / other[r] - 1) for r in repeats] + mean = statistics.mean(differences) + interval = 2.776 * statistics.stdev(differences) / math.sqrt(5) + print(f"- {model} {context} {test} vs {baseline}: {mean:+.2f}% [{mean-interval:+.2f}%, {mean+interval:+.2f}%]") + + +if __name__ == "__main__": + main() diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py new file mode 100644 index 000000000000..2f2599b5e953 --- /dev/null +++ b/scripts/xe2/validate.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +"""Reproducible local Xe2 validation. Results and server logs stay out of Git.""" +import argparse +import concurrent.futures +from contextlib import contextmanager +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import socket +import subprocess +import time +import urllib.error +import urllib.request + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +MANIFEST = json.loads((HERE / "models.json").read_text()) + + +def write_json(path, data): + path.write_text(json.dumps(data, indent=2, allow_nan=False) + "\n") + + +def environment(mode): + env = os.environ.copy() + for name in ("LLAMA_VK_GEMMA4_HYBRID_FA", "GGML_VK_DISABLE_MMVQ", "GGML_VK_FORCE_MMVQ"): + env.pop(name, None) + if mode == "hybrid": + env["LLAMA_VK_GEMMA4_HYBRID_FA"] = "1" + elif mode == "zero": + env["LLAMA_VK_GEMMA4_HYBRID_FA"] = "0" + return env + + +def snapshot(pid=None): + data = {"time": time.time()} + for path in (Path("/sys/class/power_supply/AC/online"), Path("/sys/firmware/acpi/platform_profile")): + if path.exists(): + data[str(path)] = path.read_text().strip() + data["temperatures"] = {str(p): p.read_text().strip() for p in Path("/sys/class/thermal").glob("thermal_zone*/temp")} + if pid: + try: + for line in Path(f"/proc/{pid}/status").read_text().splitlines(): + if line.startswith(("VmRSS:", "VmHWM:")): + key, value = line.split(":", 1) + data[key] = value.strip() + except FileNotFoundError: + pass + return data + + +def run(args, command, name, env=None): + print(name, flush=True) + start = snapshot() + with (args.output / f"{name}.stdout").open("w") as out, (args.output / f"{name}.stderr").open("w") as err: + process = subprocess.Popen([str(x) for x in command], stdout=out, stderr=err, env=env) + samples = [] + try: + while process.poll() is None: + samples.append(snapshot(process.pid)) + time.sleep(1) + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=30) + write_json(args.output / f"{name}.run.json", { + "command": [str(x) for x in command], "start": start, "end": snapshot(), + "returncode": process.returncode, "samples": samples, + "tuning_environment": {k: v for k, v in (env or os.environ).items() if k.startswith(("LLAMA_VK_", "GGML_VK_"))}, + }) + if process.returncode: + raise RuntimeError(f"{name} failed; inspect {args.output / (name + '.stderr')}") + + +def request(port, path, payload=None): + data = None if payload is None else json.dumps(payload).encode() + req = urllib.request.Request(f"http://127.0.0.1:{port}{path}", data=data, headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=600) as response: + result = json.load(response) + + def finite(value): + if isinstance(value, float) and not math.isfinite(value): + raise RuntimeError("Non-finite server output") + if isinstance(value, dict): + for item in value.values(): + finite(item) + if isinstance(value, list): + for item in value: + finite(item) + finite(result) + return result + + +@contextmanager +def server(args, model, mode, name, mtp=False, slots=1, cache="f16", context=2048): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + command = [str(args.bin / "llama-server"), "-m", str(model), "-ngl", "999", "-c", str(context * slots), + "-np", str(slots), "-b", "2048", "-ub", "512", "-t", "4", "-tb", "4", + "-fa", "off" if mode == "off" else "on", "-ctk", cache, "-ctv", cache, + "--host", "127.0.0.1", "--port", str(port), "--jinja"] + if mtp: + command += ["--spec-draft-model", str(args.models["12b-mtp"]), "--spec-type", "draft-mtp", + "--spec-draft-n-max", "16", "--spec-draft-p-min", "0.9", "--n-gpu-layers-draft", "999"] + write_json(args.output / f"{name}.command.json", command) + with (args.output / f"{name}.server.log").open("w") as log: + process = subprocess.Popen(command, stdout=log, stderr=log, env=environment(mode)) + try: + deadline = time.monotonic() + 600 + while True: + if process.poll() is not None: + raise RuntimeError(f"Server exited: {name}") + try: + request(port, "/health") + break + except (urllib.error.URLError, TimeoutError): + if time.monotonic() > deadline: + raise RuntimeError(f"Server startup timed out: {name}") + time.sleep(1) + yield port, process + finally: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def completion(port, prompt, cached=True): + result = request(port, "/completion", {"prompt": prompt, "n_predict": 64, "temperature": 0, + "seed": 1234, "cache_prompt": cached, "return_tokens": True}) + if not result.get("tokens") or result.get("truncated"): + raise RuntimeError("Empty or truncated completion") + return result + + +def cancel_completion(port): + payload = {"prompt": "Write a long story about an observatory.", "n_predict": 512, + "temperature": 0, "stream": True} + req = urllib.request.Request(f"http://127.0.0.1:{port}/completion", data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=600) as response: + if not response.readline(): + raise RuntimeError("Stream closed before cancellation test") + # Closing the response cancels this request. The next completion proves the + # slot can be reused; no process restart masks lifecycle failures. + + +def chat_lifecycle(port): + results = [] + for thinking in (False, True): + result = request(port, "/v1/chat/completions", {"messages": [ + {"role": "system", "content": "Answer briefly."}, + {"role": "user", "content": "What is two plus three?"}], "temperature": 0, "max_tokens": 256, + "chat_template_kwargs": {"enable_thinking": thinking}}) + if not result.get("choices"): + raise RuntimeError("Missing chat result") + results.append(result) + tools = [{"type": "function", "function": {"name": "get_weather", "description": "Get weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}] + messages = [{"role": "user", "content": "Use get_weather to check Oslo's weather."}] + response = request(port, "/v1/chat/completions", {"messages": messages, "tools": tools, + "tool_choice": {"type": "function", "function": {"name": "get_weather"}}, "temperature": 0, + "max_tokens": 256, "chat_template_kwargs": {"enable_thinking": False}}) + assistant = response["choices"][0]["message"] + calls = assistant.get("tool_calls", []) + if not calls or calls[0]["function"]["name"] != "get_weather": + raise RuntimeError("Expected structured weather tool call") + json.loads(calls[0]["function"]["arguments"]) + messages.append(assistant) + for call in calls: + messages.append({"role": "tool", "tool_call_id": call["id"], "content": "Oslo: sunny, 18 C."}) + final = request(port, "/v1/chat/completions", {"messages": messages, "tools": tools, "tool_choice": "none", + "temperature": 0, "max_tokens": 256, "chat_template_kwargs": {"enable_thinking": False}}) + if not final["choices"][0]["message"].get("content"): + raise RuntimeError("Missing response after tool result") + return {"thinking": results, "tool_call": response, "tool_result": final} + + +def parity(args): + for key in args.model_ids: + reference = args.output / f"{key}.cpu-logits.bin" + for backend in ("cpu", "gpu"): + run(args, [args.bin / "test-gemma4-device", args.models[key], reference, backend], + f"{key}-parity-{backend}", environment("off")) + + +def bench(args): + for key in args.model_ids: + for depth in args.depths: + for repeat in range(args.repetitions): + modes = ["off", "on", "hybrid"] + modes = modes[repeat % 3:] + modes[:repeat % 3] + for mode in modes: + name = f"{key}-c{depth}-{mode}-r{repeat}" + run(args, [args.bin / "llama-bench", "-m", args.models[key], "-ngl", "999", + "-p", "512", "-n", "64", "-pg", "512,64", "-d", str(depth - 576), + "-fa", "off" if mode == "off" else "on", "-ctk", "f16", "-ctv", "f16", + "-ub", "512", "-b", "2048", "-t", "4", "-r", "1", "-o", "json"], name, environment(mode)) + + +def smoke(args): + prompt = "The capital of Norway is" + for key in args.model_ids: + baseline = None + for mode, mtp, cache in [("on", False, "f16"), ("zero", False, "f16"), ("hybrid", False, "f16"), + ("hybrid", False, "q8_0"), ("hybrid", False, "q4_0")] + ( + [("hybrid", True, "f16")] if key == "12b" else []): + name = f"{key}-smoke-{mode}-{cache}-mtp{int(mtp)}" + with server(args, args.models[key], mode, name, mtp=mtp, cache=cache) as (port, process): + first = completion(port, prompt, False) + second = completion(port, prompt) + if first["tokens"] != second["tokens"]: + raise RuntimeError(f"Prefix reuse changed greedy tokens: {name}") + if mode == "on": + baseline = first["tokens"] + if mode == "zero" and first["tokens"] != baseline: + raise RuntimeError("Hybrid=0 differs from unset") + chat = chat_lifecycle(port) + cancel_completion(port) + after_cancel = completion(port, prompt) + if after_cancel["tokens"] != first["tokens"]: + raise RuntimeError(f"Cancellation/reuse changed greedy tokens: {name}") + write_json(args.output / f"{name}.json", {"first": first, "cached": second, "chat": chat, + "after_cancel": after_cancel, "memory": snapshot(process.pid)}) + + +def soak(args): + for key in args.model_ids: + name = f"{key}-soak" + slots = 1 if key == "12b" else 4 + with server(args, args.models[key], "hybrid", name, mtp=key == "12b", slots=slots) as (port, process): + started = time.monotonic() + count = 0 + prompts = ["The capital of Norway is", "Write a Python function that adds two integers.\n", + "An observatory studies stars. " * 180 + "\nSummarize in one sentence:"] + with (args.output / f"{name}.jsonl").open("w") as output: + with concurrent.futures.ThreadPoolExecutor(max_workers=slots) as pool: + while time.monotonic() - started < args.soak_seconds / len(args.model_ids): + results = list(pool.map(lambda p: completion(port, p), + [prompts[(count + i) % len(prompts)] for i in range(slots)])) + output.write(json.dumps({"elapsed": time.monotonic() - started, "memory": snapshot(process.pid), + "results": results}, allow_nan=False) + "\n") + output.flush() + count += slots + if process.poll() is not None: + raise RuntimeError("Server died during soak") + write_json(args.output / f"{name}.summary.json", {"requests": count, "elapsed": time.monotonic() - started}) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("stage", choices=["parity", "bench", "smoke", "soak"]) + parser.add_argument("--models-dir", type=Path, default=os.environ.get("GGUFS")) + parser.add_argument("--bin", type=Path, default=REPO / "build-vulkan/bin") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--model-ids", nargs="+", choices=["12b", "26b"], default=["12b", "26b"]) + parser.add_argument("--depths", nargs="+", type=int, default=[2048, 8192, 16384, 32768]) + parser.add_argument("--repetitions", type=int, default=5) + parser.add_argument("--soak-seconds", type=int, default=3600) + args = parser.parse_args() + if args.models_dir is None or min(args.depths) < 576 or args.repetitions < 1 or args.soak_seconds < 1: + parser.error("Set GGUFS or --models-dir; depths >=576 and counts positive") + args.bin = args.bin.resolve() + args.output.mkdir(parents=True, exist_ok=True) + args.models = {m["id"]: args.models_dir / Path(m["file"]).name for m in MANIFEST["models"]} + for m in MANIFEST["models"]: + if m["id"] not in args.model_ids and not (m["id"] == "12b-mtp" and "12b" in args.model_ids and args.stage in ("smoke", "soak")): + continue + with args.models[m["id"]].open("rb") as stream: + if hashlib.file_digest(stream, "sha256").hexdigest() != m["sha256"]: + raise RuntimeError(f"Model checksum mismatch: {m['id']}") + write_json(args.output / f"{args.stage}.metadata.json", {"platform": platform.platform(), "models": MANIFEST, + "git": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip(), + "diff": subprocess.check_output(["git", "diff"], cwd=REPO, text=True), "environment": snapshot(), + "devices": subprocess.check_output([str(args.bin / "llama-bench"), "--list-devices"], text=True, stderr=subprocess.STDOUT)}) + globals()[args.stage](args) + + +if __name__ == "__main__": + main() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4e7fd15f178a..cd8832e57ded 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -244,6 +244,7 @@ endif() llama_build_and_test(test-gguf.cpp) llama_build_and_test(test-backend-ops.cpp) llama_build_and_test(test-gemma4-attention-policy.cpp) +llama_build(test-gemma4-device.cpp) if (GGML_CPU) llama_build_and_test(test-cpu-quantized-copy.cpp) target_link_libraries(test-cpu-quantized-copy PRIVATE ggml-cpu) diff --git a/tests/test-gemma4-device.cpp b/tests/test-gemma4-device.cpp new file mode 100644 index 000000000000..7687d6c078a4 --- /dev/null +++ b/tests/test-gemma4-device.cpp @@ -0,0 +1,133 @@ +#include "common.h" +#include "ggml-backend.h" +#include "llama.h" + +#include +#include +#include +#include +#include +#include +#include + +static const std::vector lengths = {1, 2, 16, 31, 32, 63, 64, 128, 1152}; + +static std::vector logits(llama_context * ctx, int n_vocab) { + const float * data = llama_get_logits_ith(ctx, -1); + GGML_ASSERT(data); + std::vector result(data, data + n_vocab); + for (float x : result) { GGML_ASSERT(std::isfinite(x)); } + return result; +} + +static void compare(const std::vector & actual, const std::vector & reference, const char * label) { + GGML_ASSERT(actual.size() == reference.size()); + double error = 0, scale = 0; + for (size_t i = 0; i < actual.size(); ++i) { + error += std::pow(double(actual[i]) - reference[i], 2); + scale += double(reference[i])*reference[i]; + } + const double nmse = error / std::max(scale, 1e-30); + const bool same_top = std::max_element(actual.begin(), actual.end()) - actual.begin() == + std::max_element(reference.begin(), reference.end()) - reference.begin(); + std::printf("%s: nmse=%.9g top1_equal=%d\n", label, nmse, same_top); + std::fflush(stdout); + // Same tolerance as the FLASH_ATTN_EXT backend comparisons. Record top-1 + // separately because nearly tied logits may swap under FP16 arithmetic. + GGML_ASSERT(nmse < 5e-4); +} + +int main(int argc, char ** argv) { + if (argc != 4 || (std::string(argv[3]) != "cpu" && std::string(argv[3]) != "gpu")) { + std::fprintf(stderr, "usage: %s MODEL CPU_LOGITS_FILE cpu|gpu\n", argv[0]); + return 1; + } + const bool cpu = std::string(argv[3]) == "cpu"; + ggml_backend_load_all(); + llama_backend_init(); + auto mp = llama_model_default_params(); + mp.n_gpu_layers = cpu ? 0 : 999; + auto * model = llama_model_load_from_file(argv[1], mp); + GGML_ASSERT(model); + const auto * vocab = llama_model_get_vocab(model); + const int n_vocab = llama_vocab_n_tokens(vocab); + std::string text; + while (text.size() < 30000) { + text += "The observatory records the stars each night. Explain how the telescope measures their positions.\n"; + } + auto tokens = common_tokenize(vocab, text, true, true); + GGML_ASSERT(tokens.size() > 1200); + + std::vector> reference(lengths.size(), std::vector(n_vocab)); + if (!cpu) { + std::ifstream input(argv[2], std::ios::binary); + GGML_ASSERT(input); + for (auto & row : reference) { + input.read(reinterpret_cast(row.data()), row.size()*sizeof(float)); + GGML_ASSERT(input); + } + GGML_ASSERT(input.peek() == std::char_traits::eof()); + } + struct configuration { const char * name; const char * gate; llama_flash_attn_type fa; ggml_type cache; }; + const std::vector configs = cpu ? std::vector{ + {"cpu", "0", LLAMA_FLASH_ATTN_TYPE_DISABLED, GGML_TYPE_F16}, + } : std::vector{ + {"off", "0", LLAMA_FLASH_ATTN_TYPE_DISABLED, GGML_TYPE_F16}, + {"on", "0", LLAMA_FLASH_ATTN_TYPE_ENABLED, GGML_TYPE_F16}, + {"hybrid", "1", LLAMA_FLASH_ATTN_TYPE_ENABLED, GGML_TYPE_F16}, + {"hybrid-auto", "1", LLAMA_FLASH_ATTN_TYPE_AUTO, GGML_TYPE_F16}, + {"hybrid-q8", "1", LLAMA_FLASH_ATTN_TYPE_ENABLED, GGML_TYPE_Q8_0}, + {"hybrid-q4", "1", LLAMA_FLASH_ATTN_TYPE_ENABLED, GGML_TYPE_Q4_0}, + }; + for (const auto & config : configs) { +#ifdef _WIN32 + _putenv_s("LLAMA_VK_GEMMA4_HYBRID_FA", config.gate); +#else + setenv("LLAMA_VK_GEMMA4_HYBRID_FA", config.gate, 1); +#endif + auto cp = llama_context_default_params(); + cp.n_ctx = 2048; + cp.n_batch = 2048; + cp.n_ubatch = 512; + cp.n_seq_max = 1; + cp.n_threads = cp.n_threads_batch = 4; + cp.op_offload = !cpu; + cp.offload_kqv = !cpu; + cp.flash_attn_type = config.fa; + cp.type_k = cp.type_v = config.cache; + auto * ctx = llama_init_from_model(model, cp); + GGML_ASSERT(ctx); + for (size_t i = 0; i < lengths.size(); ++i) { + llama_memory_clear(llama_get_memory(ctx), true); + GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data(), lengths[i])) == 0); + auto result = logits(ctx, n_vocab); + if (cpu) { + reference[i] = result; + } else if (config.cache == GGML_TYPE_F16) { + compare(result, reference[i], (std::string(config.name) + " n=" + std::to_string(lengths[i])).c_str()); + } + // Dirty then free/reuse cells after SWA has wrapped, simulating a + // rejected speculative suffix while keeping the prefix resident. + if (lengths[i] > 1024) { + auto memory = llama_get_memory(ctx); + const int start = 1100; + GGML_ASSERT(llama_memory_seq_rm(memory, 0, start, -1)); + GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + 20, lengths[i] - start)) == 0); + GGML_ASSERT(llama_memory_seq_rm(memory, 0, start, -1)); + GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + start, lengths[i] - start)) == 0); + compare(logits(ctx, n_vocab), result, (std::string(config.name) + " reused KV").c_str()); + } + } + llama_free(ctx); + } + if (cpu) { + std::ofstream output(argv[2], std::ios::binary); + for (const auto & row : reference) { + output.write(reinterpret_cast(row.data()), row.size()*sizeof(float)); + } + GGML_ASSERT(output); + } + llama_model_free(model); + llama_backend_free(); + std::puts("Gemma 4 device validation: PASS"); +} From 926d541412dd756ec04befda10e05462ca4f5730 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:41:09 +0200 Subject: [PATCH 04/21] test(xe2): retain per-stage provenance and reject invalid benchmark metrics --- scripts/xe2/validate.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index 2f2599b5e953..91fab967f9d2 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -9,6 +9,7 @@ import os from pathlib import Path import platform +import shutil import socket import subprocess import time @@ -202,6 +203,9 @@ def bench(args): "-p", "512", "-n", "64", "-pg", "512,64", "-d", str(depth - 576), "-fa", "off" if mode == "off" else "on", "-ctk", "f16", "-ctv", "f16", "-ub", "512", "-b", "2048", "-t", "4", "-r", "1", "-o", "json"], name, environment(mode)) + rows = json.loads((args.output / f"{name}.stdout").read_text()) + if len(rows) != 3 or any(not math.isfinite(row["avg_ts"]) or row["avg_ts"] <= 0 for row in rows): + raise RuntimeError(f"Invalid benchmark metrics: {name}") def smoke(args): @@ -275,9 +279,14 @@ def main(): with args.models[m["id"]].open("rb") as stream: if hashlib.file_digest(stream, "sha256").hexdigest() != m["sha256"]: raise RuntimeError(f"Model checksum mismatch: {m['id']}") - write_json(args.output / f"{args.stage}.metadata.json", {"platform": platform.platform(), "models": MANIFEST, + packages = subprocess.run(["pacman", "-Q", "mesa", "vulkan-intel", "gcc"], capture_output=True, text=True).stdout if shutil.which("pacman") else "" + write_json(args.output / f"{args.stage}-{'-'.join(args.model_ids)}.metadata.json", { + "platform": platform.platform(), "packages": packages, "models": MANIFEST, + "settings": {"model_ids": args.model_ids, "depths": args.depths, "repetitions": args.repetitions, + "soak_seconds": args.soak_seconds, "models_dir": str(args.models_dir)}, "git": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip(), "diff": subprocess.check_output(["git", "diff"], cwd=REPO, text=True), "environment": snapshot(), + "server_version": subprocess.check_output([str(args.bin / "llama-server"), "--version"], text=True, stderr=subprocess.STDOUT), "devices": subprocess.check_output([str(args.bin / "llama-bench"), "--list-devices"], text=True, stderr=subprocess.STDOUT)}) globals()[args.stage](args) From 899e99441fd7dae65818f2254325cf81a38b1753 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 11:45:39 +0200 Subject: [PATCH 05/21] feat(xe2): add bounded serving presets and soak both profiles --- scripts/xe2/README.md | 21 +++++++++++++++-- scripts/xe2/serve.py | 51 +++++++++++++++++++++++++++++++++++++++++ scripts/xe2/validate.py | 41 ++++++++++++++++++--------------- 3 files changed, 92 insertions(+), 21 deletions(-) create mode 100644 scripts/xe2/serve.py diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index 096cd44772e2..d02ff47f303d 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -12,6 +12,21 @@ These match filenames in the historical notes, whose original Xe2 hashes were not recorded. Do not compare new results to those tables as identical artifacts. GGUFs remain outside the repository under `$GGUFS` or `--models-dir`. +Serving presets make context and cache choices explicit. The baseline uses +FA-off, F16 K/V, one slot, and 8K context; the hybrid experiment uses FA-on and +2K context. Both verify model hashes at startup. MTP is optional and limited +to the matching 12B assistant. No service is installed or deployed by these scripts. + +```bash +python scripts/xe2/serve.py 12b --mtp +python scripts/xe2/serve.py 26b +python scripts/xe2/serve.py 26b --profile hybrid +``` + +Switch back to `--profile baseline` to disable hybrid selection. The baseline +accepts `--ctx-size` up to 32768; hybrid rejects sizes above 2048. The presets +remain candidates until the acceptance runs below have passed. + ```bash python scripts/xe2/fetch-models.py --models-dir "$GGUFS" cmake --build build-vulkan -j 4 --target llama-server llama-bench \ @@ -48,8 +63,10 @@ manifest model and are generated afresh by the runner. Smoke checks repeated-prefix greedy tokens, the unset/zero switch, quantized cache fallback, thinking modes, a complete tool-call/result cycle, stream cancellation and slot reuse, and the matching 12B MTP assistant. The default -soak lasts a total hour: 12B with MTP and one slot, then 26B with four concurrent -slots. Logs retain responses, timing, temperature, power profile, and RSS. +soak lasts a total hour: 15 minutes per target/profile combination. Baseline +uses 8K context and one slot, with long prompts spanning multiple prefill batches; +hybrid uses 2K per slot, one slot for 12B and four concurrent slots for 26B. +Both 12B phases use MTP. Logs retain responses, timing, temperature, power profile, and RSS. Inspect warm RSS trends and MTP engagement/acceptance in server logs before accepting the soak; request success alone does not prove bounded memory or that speculative decoding engaged. diff --git a/scripts/xe2/serve.py b/scripts/xe2/serve.py new file mode 100644 index 000000000000..fe708a5b7e61 --- /dev/null +++ b/scripts/xe2/serve.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Launch a bounded Lunar Lake serving preset with the pinned model artifacts.""" +import argparse +import hashlib +import json +import os +from pathlib import Path + + +def main(): + here = Path(__file__).resolve().parent + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("model", choices=["12b", "26b"]) + parser.add_argument("--profile", choices=["baseline", "hybrid"], default="baseline") + parser.add_argument("--models-dir", type=Path, default=os.environ.get("GGUFS")) + parser.add_argument("--bin", type=Path, default=here.parent.parent / "build-vulkan/bin") + parser.add_argument("--ctx-size", type=int) + parser.add_argument("--mtp", action="store_true") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8080) + args = parser.parse_args() + context = args.ctx_size if args.ctx_size is not None else (2048 if args.profile == "hybrid" else 8192) + if args.models_dir is None or not 1 <= context <= 32768: + parser.error("Set GGUFS or --models-dir; context must be between 1 and 32768") + if args.profile == "hybrid" and context > 2048: + parser.error("The experimental hybrid preset is limited to 2048 tokens") + if args.mtp and args.model != "12b": + parser.error("This preset validates MTP only for the matching 12B target") + manifest = {m["id"]: m for m in json.loads((here / "models.json").read_text())["models"]} + models = {key: args.models_dir / Path(m["file"]).name for key, m in manifest.items()} + for key in [args.model] + (["12b-mtp"] if args.mtp else []): + if not models[key].is_file(): + parser.error(f"Missing {models[key]}; run scripts/xe2/fetch-models.py first") + with models[key].open("rb") as stream: + if hashlib.file_digest(stream, "sha256").hexdigest() != manifest[key]["sha256"]: + parser.error(f"Model checksum mismatch: {models[key]}") + binary = (args.bin / "llama-server").resolve() + command = [str(binary), "-m", str(models[args.model]), "-ngl", "999", "-np", "1", "-c", str(context), + "-b", "2048", "-ub", "512", "-t", "4", "-tb", "4", "-ctk", "f16", "-ctv", "f16", + "-fa", "on" if args.profile == "hybrid" else "off", "--jinja", + "--host", args.host, "--port", str(args.port)] + if args.mtp: + command += ["--spec-draft-model", str(models["12b-mtp"]), "--spec-type", "draft-mtp", + "--spec-draft-n-max", "16", "--spec-draft-p-min", "0.9", "--n-gpu-layers-draft", "999"] + env = os.environ.copy() + env["LLAMA_VK_GEMMA4_HYBRID_FA"] = "1" if args.profile == "hybrid" else "0" + os.execve(binary, command, env) + + +if __name__ == "__main__": + main() diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index 91fab967f9d2..cd8bb530e417 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -236,25 +236,28 @@ def smoke(args): def soak(args): for key in args.model_ids: - name = f"{key}-soak" - slots = 1 if key == "12b" else 4 - with server(args, args.models[key], "hybrid", name, mtp=key == "12b", slots=slots) as (port, process): - started = time.monotonic() - count = 0 - prompts = ["The capital of Norway is", "Write a Python function that adds two integers.\n", - "An observatory studies stars. " * 180 + "\nSummarize in one sentence:"] - with (args.output / f"{name}.jsonl").open("w") as output: - with concurrent.futures.ThreadPoolExecutor(max_workers=slots) as pool: - while time.monotonic() - started < args.soak_seconds / len(args.model_ids): - results = list(pool.map(lambda p: completion(port, p), - [prompts[(count + i) % len(prompts)] for i in range(slots)])) - output.write(json.dumps({"elapsed": time.monotonic() - started, "memory": snapshot(process.pid), - "results": results}, allow_nan=False) + "\n") - output.flush() - count += slots - if process.poll() is not None: - raise RuntimeError("Server died during soak") - write_json(args.output / f"{name}.summary.json", {"requests": count, "elapsed": time.monotonic() - started}) + for profile in ("baseline", "hybrid"): + name = f"{key}-soak-{profile}" + slots = 4 if key == "26b" and profile == "hybrid" else 1 + context = 8192 if profile == "baseline" else 2048 + mode = "off" if profile == "baseline" else "hybrid" + with server(args, args.models[key], mode, name, mtp=key == "12b", slots=slots, context=context) as (port, process): + started = time.monotonic() + count = 0 + prompts = ["The capital of Norway is", "Write a Python function that adds two integers.\n", + "An observatory studies stars. " * (900 if profile == "baseline" else 180) + "\nSummarize in one sentence:"] + with (args.output / f"{name}.jsonl").open("w") as output: + with concurrent.futures.ThreadPoolExecutor(max_workers=slots) as pool: + while time.monotonic() - started < args.soak_seconds / len(args.model_ids) / 2: + results = list(pool.map(lambda p: completion(port, p), + [prompts[(count + i) % len(prompts)] for i in range(slots)])) + output.write(json.dumps({"elapsed": time.monotonic() - started, "memory": snapshot(process.pid), + "results": results}, allow_nan=False) + "\n") + output.flush() + count += slots + if process.poll() is not None: + raise RuntimeError("Server died during soak") + write_json(args.output / f"{name}.summary.json", {"requests": count, "elapsed": time.monotonic() - started}) def main(): From 48d54984f3be71e902ff091d0ff9e7b2e5670665 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:47:44 +0200 Subject: [PATCH 06/21] test(xe2): honor Gemma suppressed-token logits in parity checks --- scripts/xe2/README.md | 2 ++ tests/test-gemma4-device.cpp | 24 ++++++++++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index d02ff47f303d..665503c44f66 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -55,6 +55,8 @@ standalone pp/tg finish slightly earlier. Raw JSON records the actual sizes. Parity uses full CPU logits at 1/2/16, 31/32, 63/64, 128, and 1152 prompt tokens; F16 GPU paths must satisfy the backend FA test tolerance (NMSE < 5e-4). +Declared suppressed tokens must retain their intentional negative-infinity +logits; all other logits must be finite. Suppressed entries are excluded from NMSE. Top-token agreement is recorded separately for review. F16/Q8_0/Q4_0 caches must remain finite and reproduce logits after dirtying, freeing, and reusing a suffix beyond the sliding window. CPU reference files belong to the exact diff --git a/tests/test-gemma4-device.cpp b/tests/test-gemma4-device.cpp index 7687d6c078a4..c0c358ac55fe 100644 --- a/tests/test-gemma4-device.cpp +++ b/tests/test-gemma4-device.cpp @@ -1,6 +1,7 @@ #include "common.h" #include "ggml-backend.h" #include "llama.h" +#include "../src/llama-vocab.h" #include #include @@ -12,11 +13,22 @@ static const std::vector lengths = {1, 2, 16, 31, 32, 63, 64, 128, 1152}; -static std::vector logits(llama_context * ctx, int n_vocab) { +static std::vector logits(llama_context * ctx, const llama_vocab * vocab) { + const int n_vocab = llama_vocab_n_tokens(vocab); const float * data = llama_get_logits_ith(ctx, -1); GGML_ASSERT(data); std::vector result(data, data + n_vocab); - for (float x : result) { GGML_ASSERT(std::isfinite(x)); } + std::vector suppressed(n_vocab, false); + for (llama_token token : vocab->get_suppress_tokens()) { + if (token >= 0 && token < n_vocab) { suppressed[token] = true; } + } + for (int i = 0; i < n_vocab; ++i) { + if (suppressed[i]) { + GGML_ASSERT(std::isinf(result[i]) && result[i] < 0); + } else { + GGML_ASSERT(std::isfinite(result[i])); + } + } return result; } @@ -24,6 +36,10 @@ static void compare(const std::vector & actual, const std::vector GGML_ASSERT(actual.size() == reference.size()); double error = 0, scale = 0; for (size_t i = 0; i < actual.size(); ++i) { + if (std::isinf(reference[i]) && reference[i] < 0) { + GGML_ASSERT(actual[i] == reference[i]); + continue; + } error += std::pow(double(actual[i]) - reference[i], 2); scale += double(reference[i])*reference[i]; } @@ -100,7 +116,7 @@ int main(int argc, char ** argv) { for (size_t i = 0; i < lengths.size(); ++i) { llama_memory_clear(llama_get_memory(ctx), true); GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data(), lengths[i])) == 0); - auto result = logits(ctx, n_vocab); + auto result = logits(ctx, vocab); if (cpu) { reference[i] = result; } else if (config.cache == GGML_TYPE_F16) { @@ -115,7 +131,7 @@ int main(int argc, char ** argv) { GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + 20, lengths[i] - start)) == 0); GGML_ASSERT(llama_memory_seq_rm(memory, 0, start, -1)); GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + start, lengths[i] - start)) == 0); - compare(logits(ctx, n_vocab), result, (std::string(config.name) + " reused KV").c_str()); + compare(logits(ctx, vocab), result, (std::string(config.name) + " reused KV").c_str()); } } llama_free(ctx); From 867b8e825407e5fac3d1728fb04fe7c11247165b Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:49:35 +0200 Subject: [PATCH 07/21] test(xe2): read suppression metadata through portable GGUF API --- tests/test-gemma4-device.cpp | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/test-gemma4-device.cpp b/tests/test-gemma4-device.cpp index c0c358ac55fe..78645d4cc179 100644 --- a/tests/test-gemma4-device.cpp +++ b/tests/test-gemma4-device.cpp @@ -1,7 +1,7 @@ #include "common.h" #include "ggml-backend.h" +#include "gguf.h" #include "llama.h" -#include "../src/llama-vocab.h" #include #include @@ -13,16 +13,27 @@ static const std::vector lengths = {1, 2, 16, 31, 32, 63, 64, 128, 1152}; -static std::vector logits(llama_context * ctx, const llama_vocab * vocab) { - const int n_vocab = llama_vocab_n_tokens(vocab); - const float * data = llama_get_logits_ith(ctx, -1); - GGML_ASSERT(data); - std::vector result(data, data + n_vocab); +static std::vector suppression_mask(const char * path, int n_vocab) { std::vector suppressed(n_vocab, false); - for (llama_token token : vocab->get_suppress_tokens()) { - if (token >= 0 && token < n_vocab) { suppressed[token] = true; } + auto * metadata = gguf_init_from_file(path, {true, nullptr}); + GGML_ASSERT(metadata); + const int64_t key = gguf_find_key(metadata, "tokenizer.ggml.suppress_tokens"); + if (key >= 0) { + GGML_ASSERT(gguf_get_arr_type(metadata, key) == GGUF_TYPE_INT32); + const auto * tokens = static_cast(gguf_get_arr_data(metadata, key)); + for (size_t i = 0; i < gguf_get_arr_n(metadata, key); ++i) { + if (tokens[i] >= 0 && tokens[i] < n_vocab) { suppressed[tokens[i]] = true; } + } } - for (int i = 0; i < n_vocab; ++i) { + gguf_free(metadata); + return suppressed; +} + +static std::vector logits(llama_context * ctx, const std::vector & suppressed) { + const float * data = llama_get_logits_ith(ctx, -1); + GGML_ASSERT(data); + std::vector result(data, data + suppressed.size()); + for (size_t i = 0; i < suppressed.size(); ++i) { if (suppressed[i]) { GGML_ASSERT(std::isinf(result[i]) && result[i] < 0); } else { @@ -67,6 +78,7 @@ int main(int argc, char ** argv) { GGML_ASSERT(model); const auto * vocab = llama_model_get_vocab(model); const int n_vocab = llama_vocab_n_tokens(vocab); + const auto suppressed = suppression_mask(argv[1], n_vocab); std::string text; while (text.size() < 30000) { text += "The observatory records the stars each night. Explain how the telescope measures their positions.\n"; @@ -116,7 +128,7 @@ int main(int argc, char ** argv) { for (size_t i = 0; i < lengths.size(); ++i) { llama_memory_clear(llama_get_memory(ctx), true); GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data(), lengths[i])) == 0); - auto result = logits(ctx, vocab); + auto result = logits(ctx, suppressed); if (cpu) { reference[i] = result; } else if (config.cache == GGML_TYPE_F16) { @@ -131,7 +143,7 @@ int main(int argc, char ** argv) { GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + 20, lengths[i] - start)) == 0); GGML_ASSERT(llama_memory_seq_rm(memory, 0, start, -1)); GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + start, lengths[i] - start)) == 0); - compare(logits(ctx, vocab), result, (std::string(config.name) + " reused KV").c_str()); + compare(logits(ctx, suppressed), result, (std::string(config.name) + " reused KV").c_str()); } } llama_free(ctx); From 7baac2a2f9c98a018acf0c54eef8638f906877db Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:41:18 +0200 Subject: [PATCH 08/21] test(xe2): retain full parity diagnostics without masking failures --- scripts/xe2/README.md | 7 ++++- tests/test-gemma4-device.cpp | 57 +++++++++++++++++++++++++++++++----- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index 665503c44f66..bb25ba959f1a 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -57,7 +57,12 @@ Parity uses full CPU logits at 1/2/16, 31/32, 63/64, 128, and 1152 prompt tokens F16 GPU paths must satisfy the backend FA test tolerance (NMSE < 5e-4). Declared suppressed tokens must retain their intentional negative-infinity logits; all other logits must be finite. Suppressed entries are excluded from NMSE. -Top-token agreement is recorded separately for review. F16/Q8_0/Q4_0 caches +Top-token agreement, KL divergence, and total variation of token probabilities +are recorded separately for review. The test reports all numerical comparisons +before returning failure; it also compares F16 paths against GPU FA-off and +saves raw GPU logits beside the CPU reference. An optional final configuration +argument to `test-gemma4-device` (for example `off`) selects a diagnostic run; +it does not replace the full suite. F16/Q8_0/Q4_0 caches must remain finite and reproduce logits after dirtying, freeing, and reusing a suffix beyond the sliding window. CPU reference files belong to the exact manifest model and are generated afresh by the runner. diff --git a/tests/test-gemma4-device.cpp b/tests/test-gemma4-device.cpp index 78645d4cc179..7dd7c2c997d1 100644 --- a/tests/test-gemma4-device.cpp +++ b/tests/test-gemma4-device.cpp @@ -43,30 +43,45 @@ static std::vector logits(llama_context * ctx, const std::vector & return result; } -static void compare(const std::vector & actual, const std::vector & reference, const char * label) { +static bool compare(const std::vector & actual, const std::vector & reference, const char * label) { GGML_ASSERT(actual.size() == reference.size()); double error = 0, scale = 0; + const double max_actual = *std::max_element(actual.begin(), actual.end()); + const double max_reference = *std::max_element(reference.begin(), reference.end()); + double sum_actual = 0, sum_reference = 0; + for (size_t i = 0; i < actual.size(); ++i) { + sum_actual += std::exp(actual[i] - max_actual); + sum_reference += std::exp(reference[i] - max_reference); + } + const double log_z_actual = max_actual + std::log(sum_actual); + const double log_z_reference = max_reference + std::log(sum_reference); + double kl = 0, tv = 0; for (size_t i = 0; i < actual.size(); ++i) { if (std::isinf(reference[i]) && reference[i] < 0) { GGML_ASSERT(actual[i] == reference[i]); continue; } + const double log_p = reference[i] - log_z_reference; + const double log_q = actual[i] - log_z_actual; + const double p = std::exp(log_p), q = std::exp(log_q); + kl += p * (log_p - log_q); + tv += std::abs(p - q) / 2; error += std::pow(double(actual[i]) - reference[i], 2); scale += double(reference[i])*reference[i]; } const double nmse = error / std::max(scale, 1e-30); const bool same_top = std::max_element(actual.begin(), actual.end()) - actual.begin() == std::max_element(reference.begin(), reference.end()) - reference.begin(); - std::printf("%s: nmse=%.9g top1_equal=%d\n", label, nmse, same_top); + std::printf("%s: nmse=%.9g kl=%.9g tv=%.9g top1_equal=%d\n", label, nmse, kl, tv, same_top); std::fflush(stdout); // Same tolerance as the FLASH_ATTN_EXT backend comparisons. Record top-1 // separately because nearly tied logits may swap under FP16 arithmetic. - GGML_ASSERT(nmse < 5e-4); + return nmse < 5e-4; } int main(int argc, char ** argv) { - if (argc != 4 || (std::string(argv[3]) != "cpu" && std::string(argv[3]) != "gpu")) { - std::fprintf(stderr, "usage: %s MODEL CPU_LOGITS_FILE cpu|gpu\n", argv[0]); + if ((argc != 4 && argc != 5) || (std::string(argv[3]) != "cpu" && std::string(argv[3]) != "gpu")) { + std::fprintf(stderr, "usage: %s MODEL CPU_LOGITS_FILE cpu|gpu [CONFIG]\n", argv[0]); return 1; } const bool cpu = std::string(argv[3]) == "cpu"; @@ -107,7 +122,13 @@ int main(int argc, char ** argv) { {"hybrid-q8", "1", LLAMA_FLASH_ATTN_TYPE_ENABLED, GGML_TYPE_Q8_0}, {"hybrid-q4", "1", LLAMA_FLASH_ATTN_TYPE_ENABLED, GGML_TYPE_Q4_0}, }; + bool passed = true; + auto gpu_reference = reference; + bool have_gpu_reference = false; + int ran = 0; for (const auto & config : configs) { + if (argc == 5 && std::string(argv[4]) != config.name) { continue; } + ++ran; #ifdef _WIN32 _putenv_s("LLAMA_VK_GEMMA4_HYBRID_FA", config.gate); #else @@ -125,14 +146,32 @@ int main(int argc, char ** argv) { cp.type_k = cp.type_v = config.cache; auto * ctx = llama_init_from_model(model, cp); GGML_ASSERT(ctx); + std::ofstream dump; + if (!cpu) { + dump.open(std::string(argv[2]) + "." + config.name, std::ios::binary); + GGML_ASSERT(dump); + } for (size_t i = 0; i < lengths.size(); ++i) { llama_memory_clear(llama_get_memory(ctx), true); GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data(), lengths[i])) == 0); auto result = logits(ctx, suppressed); + if (!cpu) { + dump.write(reinterpret_cast(result.data()), result.size()*sizeof(float)); + GGML_ASSERT(dump); + } if (cpu) { reference[i] = result; } else if (config.cache == GGML_TYPE_F16) { - compare(result, reference[i], (std::string(config.name) + " n=" + std::to_string(lengths[i])).c_str()); + passed &= compare(result, reference[i], (std::string(config.name) + " n=" + std::to_string(lengths[i])).c_str()); + } + if (!cpu && config.cache == GGML_TYPE_F16) { + if (std::string(config.name) == "off") { + gpu_reference[i] = result; + have_gpu_reference = true; + } else if (have_gpu_reference) { + passed &= compare(result, gpu_reference[i], + (std::string(config.name) + " vs GPU-off n=" + std::to_string(lengths[i])).c_str()); + } } // Dirty then free/reuse cells after SWA has wrapped, simulating a // rejected speculative suffix while keeping the prefix resident. @@ -143,11 +182,12 @@ int main(int argc, char ** argv) { GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + 20, lengths[i] - start)) == 0); GGML_ASSERT(llama_memory_seq_rm(memory, 0, start, -1)); GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + start, lengths[i] - start)) == 0); - compare(logits(ctx, suppressed), result, (std::string(config.name) + " reused KV").c_str()); + passed &= compare(logits(ctx, suppressed), result, (std::string(config.name) + " reused KV").c_str()); } } llama_free(ctx); } + GGML_ASSERT(ran > 0); if (cpu) { std::ofstream output(argv[2], std::ios::binary); for (const auto & row : reference) { @@ -157,5 +197,6 @@ int main(int argc, char ** argv) { } llama_model_free(model); llama_backend_free(); - std::puts("Gemma 4 device validation: PASS"); + std::puts(passed ? "Gemma 4 device validation: PASS" : "Gemma 4 device validation: FAIL"); + return passed ? 0 : 1; } From 8896fc8356a0cc5786b7ac157d468c7e648f8170 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:42:44 +0200 Subject: [PATCH 09/21] test(xe2): account for UMA GPU memory and require MTP engagement --- scripts/xe2/README.md | 4 +++- scripts/xe2/validate.py | 26 +++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index bb25ba959f1a..b0add875d102 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -73,7 +73,9 @@ cancellation and slot reuse, and the matching 12B MTP assistant. The default soak lasts a total hour: 15 minutes per target/profile combination. Baseline uses 8K context and one slot, with long prompts spanning multiple prefill batches; hybrid uses 2K per slot, one slot for 12B and four concurrent slots for 26B. -Both 12B phases use MTP. Logs retain responses, timing, temperature, power profile, and RSS. +Both 12B phases use MTP. Logs retain responses, timing, temperature, power profile, RSS, and per-client +DRM memory accounting. GPU allocations on this UMA device are not all reflected +in process RSS. Each MTP soak phase must actually draft and accept tokens. Inspect warm RSS trends and MTP engagement/acceptance in server logs before accepting the soak; request success alone does not prove bounded memory or that speculative decoding engaged. diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index cd8bb530e417..fa886bcdd94f 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -42,12 +42,29 @@ def snapshot(pid=None): if path.exists(): data[str(path)] = path.read_text().strip() data["temperatures"] = {str(p): p.read_text().strip() for p in Path("/sys/class/thermal").glob("thermal_zone*/temp")} + memory_info = Path("/proc/meminfo") + if memory_info.exists(): + data["system_memory"] = {line.split(":", 1)[0]: line.split(":", 1)[1].strip() + for line in memory_info.read_text().splitlines() + if line.startswith(("MemAvailable:", "SwapFree:"))} if pid: try: for line in Path(f"/proc/{pid}/status").read_text().splitlines(): if line.startswith(("VmRSS:", "VmHWM:")): key, value = line.split(":", 1) data[key] = value.strip() + # Xe allocates GPU buffers outside the process RSS on this UMA + # device. Keep per-client DRM accounting and deduplicate dup fds. + clients = {} + for path in Path(f"/proc/{pid}/fdinfo").iterdir(): + try: + fields = dict(line.split(":", 1) for line in path.read_text().splitlines() + if line.startswith("drm-")) + except FileNotFoundError: + continue + if "drm-client-id" in fields: + clients[fields["drm-client-id"].strip()] = {k: v.strip() for k, v in fields.items()} + data["drm_clients"] = clients except FileNotFoundError: pass return data @@ -244,6 +261,7 @@ def soak(args): with server(args, args.models[key], mode, name, mtp=key == "12b", slots=slots, context=context) as (port, process): started = time.monotonic() count = 0 + drafted = accepted = 0 prompts = ["The capital of Norway is", "Write a Python function that adds two integers.\n", "An observatory studies stars. " * (900 if profile == "baseline" else 180) + "\nSummarize in one sentence:"] with (args.output / f"{name}.jsonl").open("w") as output: @@ -251,13 +269,19 @@ def soak(args): while time.monotonic() - started < args.soak_seconds / len(args.model_ids) / 2: results = list(pool.map(lambda p: completion(port, p), [prompts[(count + i) % len(prompts)] for i in range(slots)])) + for result in results: + drafted += result.get("timings", {}).get("draft_n", 0) + accepted += result.get("timings", {}).get("draft_n_accepted", 0) output.write(json.dumps({"elapsed": time.monotonic() - started, "memory": snapshot(process.pid), "results": results}, allow_nan=False) + "\n") output.flush() count += slots if process.poll() is not None: raise RuntimeError("Server died during soak") - write_json(args.output / f"{name}.summary.json", {"requests": count, "elapsed": time.monotonic() - started}) + write_json(args.output / f"{name}.summary.json", {"requests": count, "elapsed": time.monotonic() - started, + "drafted": drafted, "accepted": accepted}) + if key == "12b" and (drafted == 0 or accepted == 0): + raise RuntimeError(f"MTP did not draft and accept tokens: {name}") def main(): From 5eae094aab1eff35d508bc65d44a511309c711ae Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:43:49 +0200 Subject: [PATCH 10/21] test(xe2): compare MTP output with target-only decoding --- scripts/xe2/validate.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index fa886bcdd94f..2a66a32d64fe 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -229,6 +229,7 @@ def smoke(args): prompt = "The capital of Norway is" for key in args.model_ids: baseline = None + hybrid_target = None for mode, mtp, cache in [("on", False, "f16"), ("zero", False, "f16"), ("hybrid", False, "f16"), ("hybrid", False, "q8_0"), ("hybrid", False, "q4_0")] + ( [("hybrid", True, "f16")] if key == "12b" else []): @@ -242,6 +243,13 @@ def smoke(args): baseline = first["tokens"] if mode == "zero" and first["tokens"] != baseline: raise RuntimeError("Hybrid=0 differs from unset") + if mode == "hybrid" and cache == "f16" and not mtp: + hybrid_target = first["tokens"] + if mtp: + if first["tokens"] != hybrid_target: + raise RuntimeError("MTP differs from target-only greedy tokens") + if first.get("timings", {}).get("draft_n_accepted", 0) <= 0: + raise RuntimeError("MTP smoke test accepted no draft tokens") chat = chat_lifecycle(port) cancel_completion(port) after_cancel = completion(port, prompt) From 4651598c24cd000deca72ff53163e94fc0eab4e9 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:51:12 +0200 Subject: [PATCH 11/21] fix(xe2): bound serving prompt cache on the 32GB UMA device --- scripts/xe2/README.md | 3 ++- scripts/xe2/serve.py | 2 +- scripts/xe2/validate.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index b0add875d102..15d45c160455 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -14,7 +14,8 @@ GGUFs remain outside the repository under `$GGUFS` or `--models-dir`. Serving presets make context and cache choices explicit. The baseline uses FA-off, F16 K/V, one slot, and 8K context; the hybrid experiment uses FA-on and -2K context. Both verify model hashes at startup. MTP is optional and limited +2K context. Both cap the RAM prompt cache at 1 GiB and verify model hashes +at startup. MTP is optional and limited to the matching 12B assistant. No service is installed or deployed by these scripts. ```bash diff --git a/scripts/xe2/serve.py b/scripts/xe2/serve.py index fe708a5b7e61..e9803f63429e 100644 --- a/scripts/xe2/serve.py +++ b/scripts/xe2/serve.py @@ -38,7 +38,7 @@ def main(): command = [str(binary), "-m", str(models[args.model]), "-ngl", "999", "-np", "1", "-c", str(context), "-b", "2048", "-ub", "512", "-t", "4", "-tb", "4", "-ctk", "f16", "-ctv", "f16", "-fa", "on" if args.profile == "hybrid" else "off", "--jinja", - "--host", args.host, "--port", str(args.port)] + "--cache-ram", "1024", "--host", args.host, "--port", str(args.port)] if args.mtp: command += ["--spec-draft-model", str(models["12b-mtp"]), "--spec-type", "draft-mtp", "--spec-draft-n-max", "16", "--spec-draft-p-min", "0.9", "--n-gpu-layers-draft", "999"] diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index 2a66a32d64fe..e6f0637e5640 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -120,7 +120,7 @@ def server(args, model, mode, name, mtp=False, slots=1, cache="f16", context=204 command = [str(args.bin / "llama-server"), "-m", str(model), "-ngl", "999", "-c", str(context * slots), "-np", str(slots), "-b", "2048", "-ub", "512", "-t", "4", "-tb", "4", "-fa", "off" if mode == "off" else "on", "-ctk", cache, "-ctv", cache, - "--host", "127.0.0.1", "--port", str(port), "--jinja"] + "--cache-ram", "1024", "--host", "127.0.0.1", "--port", str(port), "--jinja"] if mtp: command += ["--spec-draft-model", str(args.models["12b-mtp"]), "--spec-type", "draft-mtp", "--spec-draft-n-max", "16", "--spec-draft-p-min", "0.9", "--n-gpu-layers-draft", "999"] From d194f441a004c40d31a137a4dc758ff0c38996d3 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:09:16 +0200 Subject: [PATCH 12/21] test(xe2): validate instruction models with their chat template --- scripts/xe2/README.md | 12 +++++--- scripts/xe2/validate.py | 55 ++++++++++++++++++++++++++++-------- tests/test-gemma4-device.cpp | 30 +++++++++++++++----- 3 files changed, 75 insertions(+), 22 deletions(-) diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index 15d45c160455..07a0f809488f 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -54,7 +54,8 @@ Each benchmark command measures pp512, tg64, and combined pp512+tg64, starting at depth `context - 576`. The combined test ends at the named context envelope; standalone pp/tg finish slightly earlier. Raw JSON records the actual sizes. -Parity uses full CPU logits at 1/2/16, 31/32, 63/64, 128, and 1152 prompt tokens; +Parity uses full CPU logits after continuation batches of 1/2/16, 31/32, +63/64, 128, and 1152 tokens following a GGUF-templated chat prefix; F16 GPU paths must satisfy the backend FA test tolerance (NMSE < 5e-4). Declared suppressed tokens must retain their intentional negative-infinity logits; all other logits must be finite. Suppressed entries are excluded from NMSE. @@ -68,7 +69,9 @@ must remain finite and reproduce logits after dirtying, freeing, and reusing a suffix beyond the sliding window. CPU reference files belong to the exact manifest model and are generated afresh by the runner. -Smoke checks repeated-prefix greedy tokens, the unset/zero switch, quantized +Completion and soak requests use `/apply-template` with thinking disabled. +Raw, untemplated text is not a meaningful instruction-model lifecycle check. +Smoke requires correct capital/arithmetic answers and checks repeated-prefix greedy tokens, the unset/zero switch, quantized cache fallback, thinking modes, a complete tool-call/result cycle, stream cancellation and slot reuse, and the matching 12B MTP assistant. The default soak lasts a total hour: 15 minutes per target/profile combination. Baseline @@ -81,10 +84,11 @@ Inspect warm RSS trends and MTP engagement/acceptance in server logs before accepting the soak; request success alone does not prove bounded memory or that speculative decoding engaged. -`--model-ids`, `--depths`, `--repetitions`, and `--soak-seconds` permit targeted +`--model-ids`, `--smoke-configs`, `--depths`, `--repetitions`, and `--soak-seconds` permit targeted debugging. Shortened runs are not the full acceptance suite. Outputs include commands, model hashes, Git revision/diff, device descriptions, and subprocess -logs. Any missing model, checksum mismatch, failed assertion, invalid response, +logs. Failed smoke comparisons retain their response evidence with `passed: false`. +Any missing model, checksum mismatch, failed assertion, invalid response, or subprocess failure stops the relevant stage with a nonzero exit. For CPU memory-safety validation, configure a separate Debug build with diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index e6f0637e5640..244c61ca1919 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -9,6 +9,7 @@ import os from pathlib import Path import platform +import re import shutil import socket import subprocess @@ -149,6 +150,15 @@ def server(args, model, mode, name, mtp=False, slots=1, cache="f16", context=204 process.wait() +def formatted_prompt(port, text): + result = request(port, "/apply-template", {"messages": [{"role": "user", "content": text}], + "chat_template_kwargs": {"enable_thinking": False}}) + prompt = result.get("prompt") + if not isinstance(prompt, str) or not prompt: + raise RuntimeError("Missing formatted prompt") + return prompt + + def completion(port, prompt, cached=True): result = request(port, "/completion", {"prompt": prompt, "n_predict": 64, "temperature": 0, "seed": 1234, "cache_prompt": cached, "return_tokens": True}) @@ -158,7 +168,7 @@ def completion(port, prompt, cached=True): def cancel_completion(port): - payload = {"prompt": "Write a long story about an observatory.", "n_predict": 512, + payload = {"prompt": formatted_prompt(port, "Write a long story about an observatory."), "n_predict": 512, "temperature": 0, "stream": True} req = urllib.request.Request(f"http://127.0.0.1:{port}/completion", data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"}) @@ -178,6 +188,9 @@ def chat_lifecycle(port): "chat_template_kwargs": {"enable_thinking": thinking}}) if not result.get("choices"): raise RuntimeError("Missing chat result") + content = result["choices"][0]["message"].get("content", "") + if not re.search(r"\b(?:five|5)\b", content, re.IGNORECASE): + raise RuntimeError("Chat failed the arithmetic answer check") results.append(result) tools = [{"type": "function", "function": {"name": "get_weather", "description": "Get weather for a city", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}] @@ -189,7 +202,9 @@ def chat_lifecycle(port): calls = assistant.get("tool_calls", []) if not calls or calls[0]["function"]["name"] != "get_weather": raise RuntimeError("Expected structured weather tool call") - json.loads(calls[0]["function"]["arguments"]) + arguments = json.loads(calls[0]["function"]["arguments"]) + if arguments.get("city", "").casefold() != "oslo": + raise RuntimeError("Weather tool call used the wrong city") messages.append(assistant) for call in calls: messages.append({"role": "tool", "tool_call_id": call["id"], "content": "Oslo: sunny, 18 C."}) @@ -202,7 +217,7 @@ def chat_lifecycle(port): def parity(args): for key in args.model_ids: - reference = args.output / f"{key}.cpu-logits.bin" + reference = args.output / f"{key}.chat.cpu-logits.bin" for backend in ("cpu", "gpu"): run(args, [args.bin / "test-gemma4-device", args.models[key], reference, backend], f"{key}-parity-{backend}", environment("off")) @@ -226,17 +241,27 @@ def bench(args): def smoke(args): - prompt = "The capital of Norway is" + query = "What is the capital of Norway? Answer briefly." for key in args.model_ids: baseline = None hybrid_target = None - for mode, mtp, cache in [("on", False, "f16"), ("zero", False, "f16"), ("hybrid", False, "f16"), - ("hybrid", False, "q8_0"), ("hybrid", False, "q4_0")] + ( - [("hybrid", True, "f16")] if key == "12b" else []): + configs = [("on", "on", False, "f16"), ("zero", "zero", False, "f16"), + ("hybrid", "hybrid", False, "f16"), ("q8", "hybrid", False, "q8_0"), + ("q4", "hybrid", False, "q4_0")] + if key == "12b": + configs.append(("mtp", "hybrid", True, "f16")) + for config, mode, mtp, cache in configs: + if config not in args.smoke_configs: + continue name = f"{key}-smoke-{mode}-{cache}-mtp{int(mtp)}" with server(args, args.models[key], mode, name, mtp=mtp, cache=cache) as (port, process): + prompt = formatted_prompt(port, query) first = completion(port, prompt, False) second = completion(port, prompt) + evidence = {"first": first, "cached": second, "passed": False} + write_json(args.output / f"{name}.json", evidence) + if "oslo" not in first.get("content", "").casefold(): + raise RuntimeError(f"Completion failed the capital answer check: {name}") if first["tokens"] != second["tokens"]: raise RuntimeError(f"Prefix reuse changed greedy tokens: {name}") if mode == "on": @@ -253,10 +278,12 @@ def smoke(args): chat = chat_lifecycle(port) cancel_completion(port) after_cancel = completion(port, prompt) + evidence.update(chat=chat, after_cancel=after_cancel, memory=snapshot(process.pid)) + write_json(args.output / f"{name}.json", evidence) if after_cancel["tokens"] != first["tokens"]: raise RuntimeError(f"Cancellation/reuse changed greedy tokens: {name}") - write_json(args.output / f"{name}.json", {"first": first, "cached": second, "chat": chat, - "after_cancel": after_cancel, "memory": snapshot(process.pid)}) + evidence["passed"] = True + write_json(args.output / f"{name}.json", evidence) def soak(args): @@ -270,8 +297,9 @@ def soak(args): started = time.monotonic() count = 0 drafted = accepted = 0 - prompts = ["The capital of Norway is", "Write a Python function that adds two integers.\n", + prompts = ["What is the capital of Norway? Answer briefly.", "Write a Python function that adds two integers.\n", "An observatory studies stars. " * (900 if profile == "baseline" else 180) + "\nSummarize in one sentence:"] + prompts = [formatted_prompt(port, p) for p in prompts] with (args.output / f"{name}.jsonl").open("w") as output: with concurrent.futures.ThreadPoolExecutor(max_workers=slots) as pool: while time.monotonic() - started < args.soak_seconds / len(args.model_ids) / 2: @@ -301,10 +329,15 @@ def main(): parser.add_argument("--model-ids", nargs="+", choices=["12b", "26b"], default=["12b", "26b"]) parser.add_argument("--depths", nargs="+", type=int, default=[2048, 8192, 16384, 32768]) parser.add_argument("--repetitions", type=int, default=5) + parser.add_argument("--smoke-configs", nargs="+", choices=["on", "zero", "hybrid", "q8", "q4", "mtp"], + default=["on", "zero", "hybrid", "q8", "q4", "mtp"]) parser.add_argument("--soak-seconds", type=int, default=3600) args = parser.parse_args() if args.models_dir is None or min(args.depths) < 576 or args.repetitions < 1 or args.soak_seconds < 1: parser.error("Set GGUFS or --models-dir; depths >=576 and counts positive") + if args.stage == "smoke" and (("zero" in args.smoke_configs and "on" not in args.smoke_configs) or + ("mtp" in args.smoke_configs and "hybrid" not in args.smoke_configs)): + parser.error("The zero comparison requires on; the MTP comparison requires hybrid") args.bin = args.bin.resolve() args.output.mkdir(parents=True, exist_ok=True) args.models = {m["id"]: args.models_dir / Path(m["file"]).name for m in MANIFEST["models"]} @@ -318,7 +351,7 @@ def main(): write_json(args.output / f"{args.stage}-{'-'.join(args.model_ids)}.metadata.json", { "platform": platform.platform(), "packages": packages, "models": MANIFEST, "settings": {"model_ids": args.model_ids, "depths": args.depths, "repetitions": args.repetitions, - "soak_seconds": args.soak_seconds, "models_dir": str(args.models_dir)}, + "soak_seconds": args.soak_seconds, "smoke_configs": args.smoke_configs, "models_dir": str(args.models_dir)}, "git": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip(), "diff": subprocess.check_output(["git", "diff"], cwd=REPO, text=True), "environment": snapshot(), "server_version": subprocess.check_output([str(args.bin / "llama-server"), "--version"], text=True, stderr=subprocess.STDOUT), diff --git a/tests/test-gemma4-device.cpp b/tests/test-gemma4-device.cpp index 7dd7c2c997d1..ec4f2861fafd 100644 --- a/tests/test-gemma4-device.cpp +++ b/tests/test-gemma4-device.cpp @@ -1,4 +1,5 @@ #include "common.h" +#include "chat.h" #include "ggml-backend.h" #include "gguf.h" #include "llama.h" @@ -94,12 +95,24 @@ int main(int argc, char ** argv) { const auto * vocab = llama_model_get_vocab(model); const int n_vocab = llama_vocab_n_tokens(vocab); const auto suppressed = suppression_mask(argv[1], n_vocab); - std::string text; - while (text.size() < 30000) { - text += "The observatory records the stars each night. Explain how the telescope measures their positions.\n"; + auto templates = common_chat_templates_init(model, ""); + common_chat_templates_inputs chat; + common_chat_msg message; + message.role = "user"; + message.content = "Write an observing log with numbered entries. Explain calibration, weather, star positions, and uncertainty."; + chat.messages.push_back(message); + chat.enable_thinking = false; + chat.chat_template_kwargs["enable_thinking"] = "false"; + const auto formatted = common_chat_templates_apply(templates.get(), chat); + auto prefix = common_tokenize(vocab, formatted.prompt, true, true); + std::string text = "The observing log separates measurements from interpretation. Each entry records the exposure and the checks used to assess its reliability.\n\n"; + for (int i = 1; i <= 128; ++i) { + text += "Entry " + std::to_string(i) + ": At minute " + std::to_string(i * 3) + + ", the telescope recorded " + std::to_string(1000 + i * 17) + + " counts. The observer checked tracking, background light, and calibration before comparing this frame with the reference exposure.\n"; } - auto tokens = common_tokenize(vocab, text, true, true); - GGML_ASSERT(tokens.size() > 1200); + auto tokens = common_tokenize(vocab, text, false, true); + GGML_ASSERT(tokens.size() > 1200 && prefix.size() + lengths.back() < 2048); std::vector> reference(lengths.size(), std::vector(n_vocab)); if (!cpu) { @@ -153,6 +166,7 @@ int main(int argc, char ** argv) { } for (size_t i = 0; i < lengths.size(); ++i) { llama_memory_clear(llama_get_memory(ctx), true); + GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(prefix.data(), prefix.size())) == 0); GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data(), lengths[i])) == 0); auto result = logits(ctx, suppressed); if (!cpu) { @@ -161,6 +175,8 @@ int main(int argc, char ** argv) { } if (cpu) { reference[i] = result; + std::printf("cpu n=%d: reference recorded\n", lengths[i]); + std::fflush(stdout); } else if (config.cache == GGML_TYPE_F16) { passed &= compare(result, reference[i], (std::string(config.name) + " n=" + std::to_string(lengths[i])).c_str()); } @@ -178,9 +194,9 @@ int main(int argc, char ** argv) { if (lengths[i] > 1024) { auto memory = llama_get_memory(ctx); const int start = 1100; - GGML_ASSERT(llama_memory_seq_rm(memory, 0, start, -1)); + GGML_ASSERT(llama_memory_seq_rm(memory, 0, prefix.size() + start, -1)); GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + 20, lengths[i] - start)) == 0); - GGML_ASSERT(llama_memory_seq_rm(memory, 0, start, -1)); + GGML_ASSERT(llama_memory_seq_rm(memory, 0, prefix.size() + start, -1)); GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + start, lengths[i] - start)) == 0); passed &= compare(logits(ctx, suppressed), result, (std::string(config.name) + " reused KV").c_str()); } From ede5b05fb80c852e9256aa12ca22e108f81e2004 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:12:27 +0200 Subject: [PATCH 13/21] test(xe2): verify speculative decoding on a longer code response --- scripts/xe2/validate.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index 244c61ca1919..916e0fd26a3e 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -245,6 +245,7 @@ def smoke(args): for key in args.model_ids: baseline = None hybrid_target = None + hybrid_verification = None configs = [("on", "on", False, "f16"), ("zero", "zero", False, "f16"), ("hybrid", "hybrid", False, "f16"), ("q8", "hybrid", False, "q8_0"), ("q4", "hybrid", False, "q4_0")] @@ -270,10 +271,18 @@ def smoke(args): raise RuntimeError("Hybrid=0 differs from unset") if mode == "hybrid" and cache == "f16" and not mtp: hybrid_target = first["tokens"] + if mode == "hybrid" and cache == "f16": + verification = completion(port, formatted_prompt(port, + "Write a Python function called add_numbers that returns a + b. Return only the function."), False) + evidence["verification"] = verification + write_json(args.output / f"{name}.json", evidence) + if not mtp: + hybrid_verification = verification["tokens"] if mtp: - if first["tokens"] != hybrid_target: + if first["tokens"] != hybrid_target or verification["tokens"] != hybrid_verification: raise RuntimeError("MTP differs from target-only greedy tokens") - if first.get("timings", {}).get("draft_n_accepted", 0) <= 0: + if sum(r.get("timings", {}).get("draft_n_accepted", 0) + for r in (first, second, verification)) <= 0: raise RuntimeError("MTP smoke test accepted no draft tokens") chat = chat_lifecycle(port) cancel_completion(port) From e1dcac531eb6d8ad00923bc20956bb6f3211f01f Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:19:23 +0200 Subject: [PATCH 14/21] test(xe2): bound full-model token distributions and retain strict KV checks --- scripts/xe2/README.md | 10 ++++++++-- tests/CMakeLists.txt | 1 + tests/test-gemma4-device.cpp | 37 +++++++++++++++++++++++++++++++----- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index 07a0f809488f..35ca131532a8 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -33,7 +33,7 @@ python scripts/xe2/fetch-models.py --models-dir "$GGUFS" cmake --build build-vulkan -j 4 --target llama-server llama-bench \ test-backend-ops test-cpu-quantized-copy test-gemma4-attention-policy test-gemma4-device ctest --test-dir build-vulkan --output-on-failure \ - -R 'test-(cpu-quantized-copy|gemma4-attention-policy)$' + -R 'test-(cpu-quantized-copy|gemma4-attention-policy|gemma4-logit-metrics)$' ./build-vulkan/bin/test-backend-ops test -b Vulkan0 -o CPY,CONT ./build-vulkan/bin/test-backend-ops test -b Vulkan0 -o MUL_MAT -p 'type_a=q4_0' ./build-vulkan/bin/test-backend-ops test -b Vulkan0 -o FLASH_ATTN_EXT \ @@ -56,7 +56,13 @@ standalone pp/tg finish slightly earlier. Raw JSON records the actual sizes. Parity uses full CPU logits after continuation batches of 1/2/16, 31/32, 63/64, 128, and 1152 tokens following a GGUF-templated chat prefix; -F16 GPU paths must satisfy the backend FA test tolerance (NMSE < 5e-4). +F16 GPU paths must stay below 0.005 nats KL divergence and 0.05 total variation +against the CPU token distribution. A single-operation NMSE threshold is not +an appropriate full-model sampling bound: logits have an arbitrary common +offset, and vocabulary tails with negligible probability can dominate NMSE. +Raw NMSE is still reported for every comparison. KV reuse retains NMSE < 5e-4 +in addition to the distribution checks. The metrics have model-free tests for +shift invariance, changed predictions, near ties, and non-finite rejection. Declared suppressed tokens must retain their intentional negative-infinity logits; all other logits must be finite. Suppressed entries are excluded from NMSE. Top-token agreement, KL divergence, and total variation of token probabilities diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cd8832e57ded..47d0b0d1d042 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -245,6 +245,7 @@ llama_build_and_test(test-gguf.cpp) llama_build_and_test(test-backend-ops.cpp) llama_build_and_test(test-gemma4-attention-policy.cpp) llama_build(test-gemma4-device.cpp) +add_test(NAME test-gemma4-logit-metrics COMMAND test-gemma4-device --self-test) if (GGML_CPU) llama_build_and_test(test-cpu-quantized-copy.cpp) target_link_libraries(test-cpu-quantized-copy PRIVATE ggml-cpu) diff --git a/tests/test-gemma4-device.cpp b/tests/test-gemma4-device.cpp index ec4f2861fafd..bba70f1e8432 100644 --- a/tests/test-gemma4-device.cpp +++ b/tests/test-gemma4-device.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -44,11 +45,17 @@ static std::vector logits(llama_context * ctx, const std::vector & return result; } -static bool compare(const std::vector & actual, const std::vector & reference, const char * label) { +static bool compare(const std::vector & actual, const std::vector & reference, const char * label, bool strict = false) { GGML_ASSERT(actual.size() == reference.size()); + if (actual.empty()) { return false; } + for (size_t i = 0; i < actual.size(); ++i) { + if (std::isinf(reference[i]) && reference[i] < 0 && actual[i] == reference[i]) { continue; } + if (!std::isfinite(actual[i]) || !std::isfinite(reference[i])) { return false; } + } double error = 0, scale = 0; const double max_actual = *std::max_element(actual.begin(), actual.end()); const double max_reference = *std::max_element(reference.begin(), reference.end()); + if (!std::isfinite(max_actual) || !std::isfinite(max_reference)) { return false; } double sum_actual = 0, sum_reference = 0; for (size_t i = 0; i < actual.size(); ++i) { sum_actual += std::exp(actual[i] - max_actual); @@ -75,12 +82,32 @@ static bool compare(const std::vector & actual, const std::vector std::max_element(reference.begin(), reference.end()) - reference.begin(); std::printf("%s: nmse=%.9g kl=%.9g tv=%.9g top1_equal=%d\n", label, nmse, kl, tv, same_top); std::fflush(stdout); - // Same tolerance as the FLASH_ATTN_EXT backend comparisons. Record top-1 - // separately because nearly tied logits may swap under FP16 arithmetic. - return nmse < 5e-4; + // Full-model logits have an arbitrary common offset, and low-probability + // vocabulary tails must not dominate the sampling comparison. Bound the + // distribution change to 0.005 nats KL and 5% total variation. Keep raw + // NMSE in every report and retain the operator tolerance for KV reuse. + return std::isfinite(kl) && std::isfinite(tv) && kl < 0.005 && tv < 0.05 && + (!strict || nmse < 5e-4); } int main(int argc, char ** argv) { + if (argc == 2 && std::string(argv[1]) == "--self-test") { + const float inf = std::numeric_limits::infinity(); + const float nan = std::numeric_limits::quiet_NaN(); + GGML_ASSERT(compare({99, 100, 101}, {-1, 0, 1}, "common offset")); + GGML_ASSERT(!compare({99, 100, 101}, {-1, 0, 1}, "strict offset", true)); + GGML_ASSERT(compare({0, 0}, {0, 0}, "identical zero logits", true)); + GGML_ASSERT(compare({-inf, 5, 6}, {-inf, 0, 1}, "declared suppression")); + GGML_ASSERT(!compare({0, 1}, {1, 0}, "changed prediction")); + GGML_ASSERT(!compare({-4, 4}, {4, -4}, "opposite peaked prediction")); + GGML_ASSERT(compare({0, 0.001f}, {0.001f, 0}, "near tie")); + GGML_ASSERT(!compare({nan, 1}, {0, 1}, "NaN")); + GGML_ASSERT(!compare({inf, 1}, {0, 1}, "positive infinity")); + GGML_ASSERT(!compare({-inf, 1}, {0, 1}, "unexpected suppression")); + GGML_ASSERT(!compare({-inf}, {-inf}, "empty distribution")); + std::puts("Logit comparison invariants: PASS"); + return 0; + } if ((argc != 4 && argc != 5) || (std::string(argv[3]) != "cpu" && std::string(argv[3]) != "gpu")) { std::fprintf(stderr, "usage: %s MODEL CPU_LOGITS_FILE cpu|gpu [CONFIG]\n", argv[0]); return 1; @@ -198,7 +225,7 @@ int main(int argc, char ** argv) { GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + 20, lengths[i] - start)) == 0); GGML_ASSERT(llama_memory_seq_rm(memory, 0, prefix.size() + start, -1)); GGML_ASSERT(llama_decode(ctx, llama_batch_get_one(tokens.data() + start, lengths[i] - start)) == 0); - passed &= compare(logits(ctx, suppressed), result, (std::string(config.name) + " reused KV").c_str()); + passed &= compare(logits(ctx, suppressed), result, (std::string(config.name) + " reused KV").c_str(), true); } } llama_free(ctx); From 124251687705710bc236412125640325f0b2360e Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:26:26 +0200 Subject: [PATCH 15/21] test(xe2): cancel streams after an observed generation token --- scripts/xe2/validate.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index 916e0fd26a3e..567cfab804ed 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -173,10 +173,20 @@ def cancel_completion(port): req = urllib.request.Request(f"http://127.0.0.1:{port}/completion", data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"}) with urllib.request.urlopen(req, timeout=600) as response: - if not response.readline(): - raise RuntimeError("Stream closed before cancellation test") + while True: + line = response.readline() + if not line: + raise RuntimeError("Stream closed before cancellation test") + if not line.startswith(b"data: "): + continue + event = json.loads(line[6:]) + if event.get("content"): + break + if event.get("stop"): + raise RuntimeError("Generation ended before a token could be cancelled") # Closing the response cancels this request. The next completion proves the # slot can be reused; no process restart masks lifecycle failures. + return event def chat_lifecycle(port): @@ -285,9 +295,9 @@ def smoke(args): for r in (first, second, verification)) <= 0: raise RuntimeError("MTP smoke test accepted no draft tokens") chat = chat_lifecycle(port) - cancel_completion(port) + cancelled_after = cancel_completion(port) after_cancel = completion(port, prompt) - evidence.update(chat=chat, after_cancel=after_cancel, memory=snapshot(process.pid)) + evidence.update(chat=chat, cancelled_after=cancelled_after, after_cancel=after_cancel, memory=snapshot(process.pid)) write_json(args.output / f"{name}.json", evidence) if after_cancel["tokens"] != first["tokens"]: raise RuntimeError(f"Cancellation/reuse changed greedy tokens: {name}") From 507d3e863fad3849b10c967ebf5a9bdc99591360 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:34:30 +0200 Subject: [PATCH 16/21] fix(xe2): tolerate telemetry faults and preserve interrupted run evidence --- scripts/xe2/README.md | 6 ++- scripts/xe2/test_validate.py | 53 +++++++++++++++++++++++ scripts/xe2/validate.py | 84 +++++++++++++++++++++++------------- 3 files changed, 113 insertions(+), 30 deletions(-) create mode 100644 scripts/xe2/test_validate.py diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index 35ca131532a8..21ae110d38f6 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -29,6 +29,7 @@ accepts `--ctx-size` up to 32768; hybrid rejects sizes above 2048. The presets remain candidates until the acceptance runs below have passed. ```bash +python scripts/xe2/test_validate.py python scripts/xe2/fetch-models.py --models-dir "$GGUFS" cmake --build build-vulkan -j 4 --target llama-server llama-bench \ test-backend-ops test-cpu-quantized-copy test-gemma4-attention-policy test-gemma4-device @@ -90,10 +91,13 @@ Inspect warm RSS trends and MTP engagement/acceptance in server logs before accepting the soak; request success alone does not prove bounded memory or that speculative decoding engaged. -`--model-ids`, `--smoke-configs`, `--depths`, `--repetitions`, and `--soak-seconds` permit targeted +`--model-ids`, `--smoke-configs`, `--depths`, `--repetitions`, `--timeout-seconds`, and `--soak-seconds` permit targeted debugging. Shortened runs are not the full acceptance suite. Outputs include commands, model hashes, Git revision/diff, device descriptions, and subprocess logs. Failed smoke comparisons retain their response evidence with `passed: false`. +Optional telemetry read failures are recorded rather than aborting the model. +Native/benchmark subprocesses have a configurable 30-minute deadline and retain +run metadata on failure or interruption. Any missing model, checksum mismatch, failed assertion, invalid response, or subprocess failure stops the relevant stage with a nonzero exit. diff --git a/scripts/xe2/test_validate.py b/scripts/xe2/test_validate.py new file mode 100644 index 000000000000..29b99ded730f --- /dev/null +++ b/scripts/xe2/test_validate.py @@ -0,0 +1,53 @@ +"""Model-free regression checks for validation process and telemetry failures.""" +import errno +import json +from pathlib import Path +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import validate + + +class ValidationInfrastructureTests(unittest.TestCase): + def test_firmware_read_failure_is_recorded(self): + target = Path("/sys/firmware/acpi/platform_profile") + original_exists, original_read = Path.exists, Path.read_text + + def exists(path): + return path == target or original_exists(path) + + def read(path, *args, **kwargs): + if path == target: + raise OSError(errno.EIO, "firmware read failed") + return original_read(path, *args, **kwargs) + + with patch.object(Path, "exists", exists), patch.object(Path, "read_text", read): + result = validate.snapshot() + self.assertIsNone(result[str(target)]) + self.assertIn(str(target), result["read_errors"]) + + def test_failed_child_keeps_evidence(self): + with tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace(output=Path(directory), timeout_seconds=10) + with self.assertRaises(RuntimeError): + validate.run(args, [sys.executable, "-c", "raise SystemExit(3)"], "failure") + result = json.loads((args.output / "failure.run.json").read_text()) + self.assertEqual(result["returncode"], 3) + self.assertIsNone(result["runner_error"]) + + def test_timeout_stops_child_and_keeps_evidence(self): + with tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace(output=Path(directory), timeout_seconds=0.01) + with self.assertRaises(TimeoutError): + validate.run(args, [sys.executable, "-c", "import time; time.sleep(60)"], "timeout") + result = json.loads((args.output / "timeout.run.json").read_text()) + self.assertIsNotNone(result["returncode"]) + self.assertNotEqual(result["returncode"], 0) + self.assertIn("TimeoutError", result["runner_error"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index 567cfab804ed..e511e0cf5f30 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -39,57 +39,82 @@ def environment(mode): def snapshot(pid=None): data = {"time": time.time()} + errors = {} + + def read(path): + try: + return path.read_text().strip() + except OSError as error: + errors[str(path)] = str(error) + return None + for path in (Path("/sys/class/power_supply/AC/online"), Path("/sys/firmware/acpi/platform_profile")): if path.exists(): - data[str(path)] = path.read_text().strip() - data["temperatures"] = {str(p): p.read_text().strip() for p in Path("/sys/class/thermal").glob("thermal_zone*/temp")} + data[str(path)] = read(path) + data["temperatures"] = {str(p): read(p) for p in Path("/sys/class/thermal").glob("thermal_zone*/temp")} memory_info = Path("/proc/meminfo") if memory_info.exists(): data["system_memory"] = {line.split(":", 1)[0]: line.split(":", 1)[1].strip() - for line in memory_info.read_text().splitlines() + for line in (read(memory_info) or "").splitlines() if line.startswith(("MemAvailable:", "SwapFree:"))} if pid: + for line in (read(Path(f"/proc/{pid}/status")) or "").splitlines(): + if line.startswith(("VmRSS:", "VmHWM:")): + key, value = line.split(":", 1) + data[key] = value.strip() + # Xe buffers are not all reflected in RSS. Deduplicate duplicated fds + # by DRM client id, and tolerate descriptors closing during sampling. + clients = {} + directory = Path(f"/proc/{pid}/fdinfo") try: - for line in Path(f"/proc/{pid}/status").read_text().splitlines(): - if line.startswith(("VmRSS:", "VmHWM:")): - key, value = line.split(":", 1) - data[key] = value.strip() - # Xe allocates GPU buffers outside the process RSS on this UMA - # device. Keep per-client DRM accounting and deduplicate dup fds. - clients = {} - for path in Path(f"/proc/{pid}/fdinfo").iterdir(): - try: - fields = dict(line.split(":", 1) for line in path.read_text().splitlines() - if line.startswith("drm-")) - except FileNotFoundError: - continue + for path in directory.iterdir(): + fields = dict(line.split(":", 1) for line in (read(path) or "").splitlines() + if line.startswith("drm-")) if "drm-client-id" in fields: clients[fields["drm-client-id"].strip()] = {k: v.strip() for k, v in fields.items()} - data["drm_clients"] = clients - except FileNotFoundError: - pass + except OSError as error: + errors[str(directory)] = str(error) + data["drm_clients"] = clients + if errors: + data["read_errors"] = errors return data def run(args, command, name, env=None): print(name, flush=True) start = snapshot() + timeout = getattr(args, "timeout_seconds", 1800) + deadline = time.monotonic() + timeout with (args.output / f"{name}.stdout").open("w") as out, (args.output / f"{name}.stderr").open("w") as err: process = subprocess.Popen([str(x) for x in command], stdout=out, stderr=err, env=env) samples = [] + runner_error = None try: while process.poll() is None: + if time.monotonic() > deadline: + raise TimeoutError(f"{name} exceeded {timeout} seconds") samples.append(snapshot(process.pid)) time.sleep(1) + except BaseException as error: + runner_error = f"{type(error).__name__}: {error}" + raise finally: - if process.poll() is None: - process.terminate() - process.wait(timeout=30) - write_json(args.output / f"{name}.run.json", { - "command": [str(x) for x in command], "start": start, "end": snapshot(), - "returncode": process.returncode, "samples": samples, - "tuning_environment": {k: v for k, v in (env or os.environ).items() if k.startswith(("LLAMA_VK_", "GGML_VK_"))}, - }) + try: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=30) + finally: + write_json(args.output / f"{name}.run.json", { + "command": [str(x) for x in command], "start": start, "end": snapshot(), + "returncode": process.returncode, "samples": samples, "runner_error": runner_error, + "timeout_seconds": timeout, + "tuning_environment": {k: v for k, v in (env or os.environ).items() + if k.startswith(("LLAMA_VK_", "GGML_VK_"))}, + }) if process.returncode: raise RuntimeError(f"{name} failed; inspect {args.output / (name + '.stderr')}") @@ -350,9 +375,10 @@ def main(): parser.add_argument("--repetitions", type=int, default=5) parser.add_argument("--smoke-configs", nargs="+", choices=["on", "zero", "hybrid", "q8", "q4", "mtp"], default=["on", "zero", "hybrid", "q8", "q4", "mtp"]) + parser.add_argument("--timeout-seconds", type=int, default=1800) parser.add_argument("--soak-seconds", type=int, default=3600) args = parser.parse_args() - if args.models_dir is None or min(args.depths) < 576 or args.repetitions < 1 or args.soak_seconds < 1: + if args.models_dir is None or min(args.depths) < 576 or args.repetitions < 1 or args.soak_seconds < 1 or args.timeout_seconds < 1: parser.error("Set GGUFS or --models-dir; depths >=576 and counts positive") if args.stage == "smoke" and (("zero" in args.smoke_configs and "on" not in args.smoke_configs) or ("mtp" in args.smoke_configs and "hybrid" not in args.smoke_configs)): @@ -370,7 +396,7 @@ def main(): write_json(args.output / f"{args.stage}-{'-'.join(args.model_ids)}.metadata.json", { "platform": platform.platform(), "packages": packages, "models": MANIFEST, "settings": {"model_ids": args.model_ids, "depths": args.depths, "repetitions": args.repetitions, - "soak_seconds": args.soak_seconds, "smoke_configs": args.smoke_configs, "models_dir": str(args.models_dir)}, + "soak_seconds": args.soak_seconds, "timeout_seconds": args.timeout_seconds, "smoke_configs": args.smoke_configs, "models_dir": str(args.models_dir)}, "git": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip(), "diff": subprocess.check_output(["git", "diff"], cwd=REPO, text=True), "environment": snapshot(), "server_version": subprocess.check_output([str(args.bin / "llama-server"), "--version"], text=True, stderr=subprocess.STDOUT), From a388e16f2bb2ccfe45ac6e47293a588c7df8af90 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:42:45 +0200 Subject: [PATCH 17/21] docs(xe2): record parity results and the failing FA-on control --- .../VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md | 96 +++++++++++++++++++ docs/backend/VULKAN-GEMMA4-INTEL-XE2.md | 5 +- 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md diff --git a/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md b/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md new file mode 100644 index 000000000000..cf1b73c589f5 --- /dev/null +++ b/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md @@ -0,0 +1,96 @@ +# Lunar Lake acceptance record — 2026-09-05 + +Status: acceptance run in progress. Hybrid remains experimental and opt-in. +No fresh speedup or deployment-readiness claim is made until the results below +are complete. + +## Scope and reproducibility + +Device: Core Ultra 5 238V, 32 GB UMA, Intel PCI `8086:64a0`, Linux `xe`, +Mesa/vulkan-intel 26.1.5, GCC 16.1.1. Runs use AC power and the performance +platform profile. The [model manifest](../../scripts/xe2/models.json) pins the +12B target, matching MTP assistant, and 26B A4B target by revision and SHA256. +These establish a fresh baseline; historical Xe2 artifact hashes were absent. + +Use the [validation commands](../../scripts/xe2/README.md) to reproduce the +checks. Raw outputs include commands, Git revision/diff, binary version, +model hashes, device information, power/temperature samples, and process/DRM +memory accounting. Server presets cap the RAM prompt cache at 1 GiB. + +## Safety and local tests + +- CPU quantized-copy regression tests and ASan/UBSan pass. Unsupported block + transposes are rejected before direct/planned execution writes output. + An oversized-buffer reproduction changed from writing 17,268,736 bytes beyond + the logical destination to returning failure with zero writes beyond it. +- Vulkan comparisons pass: 280 CPY/CONT, 29 Q4_0 MUL_MAT, and 247 Gemma attention + cases with 256/512 heads and F16/Q8_0/Q4_0 K/V. +- Local CPU CI passed Debug 45/45 and Release 47/47 using `GG_BUILD_LOW_PERF=1`. + This selection excludes the large-model/high-performance jobs. + `LLAMA_FATAL_WARNINGS=OFF` was required for a GCC 16 warning in unchanged + vocabulary construction code. The later model-free logit-metric test also passes. + +## Corrections to the validation method + +Initial tests fed untemplated text to instruction models. Some completions were +repetitive or nonsensical even while structured chat answered correctly. The +corrected lifecycle and soak tests apply the GGUF chat template, check capital +and arithmetic answers, verify the requested tool city, and compare MTP against +target-only capital and code responses. Stream cancellation waits for an actual +generated token before closing the connection. + +The native test now prefills a templated conversation, then evaluates continuation +batches of 1, 2, 16, 31, 32, 63, 64, 128, and 1152 tokens. This exercises the +attention thresholds and sliding-window reuse with a valid instruction prefix. + +A single-operation NMSE limit was also an unsuitable full-model sampling test. +Logits have an arbitrary common offset; negligible-probability vocabulary tails +can dominate their squared error. Full-model comparisons now require KL divergence +below 0.005 nats and total variation below 0.05. Raw NMSE and top-token agreement +remain in every report. KV-reuse checks additionally retain NMSE below `5e-4`. +Model-free checks cover shift invariance, changed predictions, near ties, and +non-finite rejection. Declared suppressed tokens must remain negative infinity; +other logits must be finite. + +The original raw-text failures are retained as diagnostic evidence. An isolated +build of the original `ht` baseline (`06d9d42`) produced byte-identical full +logits at all nine lengths for both models with FA-off and hybrid. This establishes +that those numerical differences predate the safety hardening; it does not +establish universal CPU/GPU equivalence. A dequantized F16 CPU reference and +higher-precision Vulkan trial explained much of the short-prompt difference, +but the original long, repetitive raw prompt remained numerically sensitive. + +## End-to-end results + +| Check | 12B | 26B A4B | +|---|---|---| +| FA-off/hybrid distributions and KV reuse | Pass | Pass | +| Plain-FA-on comparison | Pass | **Fail**: KL at 16 tokens | +| Templated lifecycle, fallback, tools, cancellation | Pending | Pending | +| Baseline and hybrid soak, 15 minutes each | Pending | Pending | +| Five interleaved runs per mode at 2K/8K/16K/32K | Pending | Pending | + +The complete 26B parity sweep returns failure: plain FA-on at 16 continuation +tokens has KL `0.00991865776` against the CPU reference, exceeding `0.005`. +Its top token still agrees. This is not waived or converted into a passing test. +Both offered presets pass their comparisons; plain FA-on is a benchmark/control +configuration, not a recommended serving preset. + +| Model/mode | Maximum CPU-reference KL | Maximum total variation | Top-token agreement | +|---|---:|---:|---:| +| 12B FA-off | 0.00115015 | 0.0186292 | 9/9 | +| 12B hybrid | 0.000639504 | 0.0116276 | 9/9 | +| 26B FA-off | 0.00312896 | 0.0339058 | 9/9 | +| 26B hybrid | 0.00457187 | 0.0359762 | 9/9 | +| 26B plain FA-on control | **0.00991866** | 0.0359762 | 9/9 | + +Automatic FA with hybrid enabled produces byte-identical logits to explicit +FA-on with hybrid enabled on both models. Worst hybrid KV-reuse NMSE is +`0.000430299` for 12B and `0.0000268791` for 26B. F16 FA-off/FA-on and Q8/Q4 +reuse comparisons are exact. Quantized-cache checks establish finite outputs +and correct reuse, not quantization-quality equivalence to the F16 reference. + +Above 2K, and with multiple configured sequences or quantized caches, the hybrid +switch intentionally retains stock attention. Benchmarks at those settings +measure fallback behavior. The four-slot 26B soak likewise exercises fallback, +not eligibility for hybrid attention. diff --git a/docs/backend/VULKAN-GEMMA4-INTEL-XE2.md b/docs/backend/VULKAN-GEMMA4-INTEL-XE2.md index ebfd09b84e65..6999dee822b5 100644 --- a/docs/backend/VULKAN-GEMMA4-INTEL-XE2.md +++ b/docs/backend/VULKAN-GEMMA4-INTEL-XE2.md @@ -29,8 +29,9 @@ The benchmark tables below are historical, not fresh acceptance evidence. Their original Xe2 model hashes were not recorded. The pinned, checksummed artifacts and repeatable validation commands in [`scripts/xe2`](../../scripts/xe2/README.md) establish a new baseline without claiming byte-identical reproduction. -Keep hybrid opt-in until model parity, lifecycle tests, depth benchmarks, and -the one-hour soak pass. Do not infer long-context gains from the short runs. +The [acceptance record](VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md) separates passing +preset checks from failed controls and unfinished measurements. Keep hybrid +opt-in until its parity, lifecycle tests, depth benchmarks, and soak pass. Do not infer long-context gains from the short runs. This document is about making each usable Gemma 4 member perform well. It does not recommend replacing one family member with another: the dense, unified, From ab22a3a94f677942ca91f1dfc33fcce3e6621434 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:58:21 +0200 Subject: [PATCH 18/21] fix(xe2): bound per-slot checkpoints and report warm UMA memory trends --- .../VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md | 6 +- scripts/xe2/README.md | 8 +- scripts/xe2/serve.py | 2 +- scripts/xe2/summarize-soak.py | 77 +++++++++++++++++++ scripts/xe2/validate.py | 2 +- 5 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 scripts/xe2/summarize-soak.py diff --git a/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md b/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md index cf1b73c589f5..a0681f7bc262 100644 --- a/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md +++ b/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md @@ -15,7 +15,9 @@ These establish a fresh baseline; historical Xe2 artifact hashes were absent. Use the [validation commands](../../scripts/xe2/README.md) to reproduce the checks. Raw outputs include commands, Git revision/diff, binary version, model hashes, device information, power/temperature samples, and process/DRM -memory accounting. Server presets cap the RAM prompt cache at 1 GiB. +memory accounting. Server presets set a 1 GiB prompt-cache budget and a 512 MiB checkpoint +budget per slot. Cache policies can exceed those nominal budgets by a retained +or newly appended state; measured memory use is reported separately. ## Safety and local tests @@ -66,7 +68,7 @@ but the original long, repetitive raw prompt remained numerically sensitive. |---|---|---| | FA-off/hybrid distributions and KV reuse | Pass | Pass | | Plain-FA-on comparison | Pass | **Fail**: KL at 16 tokens | -| Templated lifecycle, fallback, tools, cancellation | Pending | Pending | +| Templated lifecycle, fallback, tools, cancellation | Pass; rechecking final budgets | Pass; rechecking final budgets | | Baseline and hybrid soak, 15 minutes each | Pending | Pending | | Five interleaved runs per mode at 2K/8K/16K/32K | Pending | Pending | diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index 21ae110d38f6..d35aaee2a27a 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -14,8 +14,11 @@ GGUFs remain outside the repository under `$GGUFS` or `--models-dir`. Serving presets make context and cache choices explicit. The baseline uses FA-off, F16 K/V, one slot, and 8K context; the hybrid experiment uses FA-on and -2K context. Both cap the RAM prompt cache at 1 GiB and verify model hashes -at startup. MTP is optional and limited +2K context. Both set a 1 GiB RAM prompt-cache budget and a 512 MiB checkpoint budget +per slot, and verify model hashes at startup. These are cache-policy budgets, +not strict process-memory caps: the prompt cache retains at least one state, +and checkpoint eviction runs before a new state is appended. Measure actual +RSS and GPU memory, including any budget overshoot. MTP is optional and limited to the matching 12B assistant. No service is installed or deployed by these scripts. ```bash @@ -44,6 +47,7 @@ python scripts/xe2/validate.py smoke --output /tmp/xe2-validation python scripts/xe2/validate.py bench --output /tmp/xe2-validation python scripts/xe2/summarize.py /tmp/xe2-validation python scripts/xe2/validate.py soak --output /tmp/xe2-validation +python scripts/xe2/summarize-soak.py /tmp/xe2-validation ``` Run GPU stages sequentially, on AC, with the same power profile and desktop diff --git a/scripts/xe2/serve.py b/scripts/xe2/serve.py index e9803f63429e..d588d81501db 100644 --- a/scripts/xe2/serve.py +++ b/scripts/xe2/serve.py @@ -38,7 +38,7 @@ def main(): command = [str(binary), "-m", str(models[args.model]), "-ngl", "999", "-np", "1", "-c", str(context), "-b", "2048", "-ub", "512", "-t", "4", "-tb", "4", "-ctk", "f16", "-ctv", "f16", "-fa", "on" if args.profile == "hybrid" else "off", "--jinja", - "--cache-ram", "1024", "--host", args.host, "--port", str(args.port)] + "--cache-ram", "1024", "--ctx-checkpoints-max-mib", "512", "--host", args.host, "--port", str(args.port)] if args.mtp: command += ["--spec-draft-model", str(models["12b-mtp"]), "--spec-type", "draft-mtp", "--spec-draft-n-max", "16", "--spec-draft-p-min", "0.9", "--n-gpu-layers-draft", "999"] diff --git a/scripts/xe2/summarize-soak.py b/scripts/xe2/summarize-soak.py new file mode 100644 index 000000000000..2b125f18aa7c --- /dev/null +++ b/scripts/xe2/summarize-soak.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Report soak completion, MTP engagement, and warm CPU/GPU memory trends.""" +import argparse +import json +from pathlib import Path +import re +import statistics + + +def mib(value): + if value is None: + return None + match = re.fullmatch(r"(\d+)\s*(B|kB|KiB|MiB|GiB)?", value) + if not match: + raise ValueError(f"Unknown memory quantity: {value}") + number, unit = match.groups() + scale = {None: 1, "B": 1, "kB": 1024, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}[unit] + return int(number) * scale / 1024**2 + + +def trend(points): + if len(points) < 2: + return "insufficient samples" + times, values = zip(*points) + mean_time, mean_value = statistics.mean(times), statistics.mean(values) + denominator = sum((t - mean_time)**2 for t in times) + slope = sum((t - mean_time)*(v - mean_value) for t, v in points) / denominator if denominator else 0 + return f"{min(values):.1f}–{max(values):.1f}; {slope * 60:+.3f}/min" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("results", type=Path) + parser.add_argument("--warmup-seconds", type=float, default=300) + args = parser.parse_args() + print("| Model | Profile | State | Minutes | Requests | Warm RSS MiB; slope | Warm GPU resident GTT MiB; slope | MTP accepted/drafted | Telemetry errors |") + print("|---|---|---|---:|---:|---|---|---:|---:|") + complete = True + for model in ("12b", "26b"): + for profile in ("baseline", "hybrid"): + path = args.results / f"{model}-soak-{profile}.jsonl" + summary_path = path.with_suffix(".summary.json") + samples = [] + if path.exists(): + # A live writer may have an incomplete final line. + samples = [json.loads(line) for line in path.read_text().splitlines(keepends=True) if line.endswith("\n")] + summary = json.loads(summary_path.read_text()) if summary_path.exists() else None + elapsed = samples[-1]["elapsed"] if samples else 0 + requests = sum(len(sample["results"]) for sample in samples) + drafted = sum(result.get("timings", {}).get("draft_n", 0) for sample in samples for result in sample["results"]) + accepted = sum(result.get("timings", {}).get("draft_n_accepted", 0) for sample in samples for result in sample["results"]) + done = bool(summary and summary["elapsed"] >= 900 and summary["requests"] == requests and requests > 0 + and (model != "12b" or accepted > 0)) + complete &= done + rss, gpu = [], [] + for sample in samples: + if sample["elapsed"] < args.warmup_seconds: + continue + memory = sample["memory"] + resident = mib(memory.get("VmRSS")) + if resident is not None: + rss.append((sample["elapsed"], resident)) + clients = memory.get("drm_clients", {}) + amounts = [mib(client["drm-resident-gtt"]) for client in clients.values() if "drm-resident-gtt" in client] + if amounts: + gpu.append((sample["elapsed"], sum(amounts))) + errors = sum(bool(sample["memory"].get("read_errors")) for sample in samples) + print(f"| {model} | {profile} | {'complete' if done else 'incomplete'} | {elapsed / 60:.2f} | {requests} | " + f"{trend(rss)} | {trend(gpu)} | {accepted}/{drafted} | {errors} |") + print(f"\nFull four-phase hour completed: {'yes' if complete else 'no'}.") + print(f"Memory trends exclude the first {args.warmup_seconds:g} seconds of each phase. " + "CPU RSS and DRM GTT are separate measurements and must not be added as disjoint pools. " + "A finite soak can reveal growth; it cannot prove the absence of every leak.") + + +if __name__ == "__main__": + main() diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index e511e0cf5f30..50564e1d0396 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -146,7 +146,7 @@ def server(args, model, mode, name, mtp=False, slots=1, cache="f16", context=204 command = [str(args.bin / "llama-server"), "-m", str(model), "-ngl", "999", "-c", str(context * slots), "-np", str(slots), "-b", "2048", "-ub", "512", "-t", "4", "-tb", "4", "-fa", "off" if mode == "off" else "on", "-ctk", cache, "-ctv", cache, - "--cache-ram", "1024", "--host", "127.0.0.1", "--port", str(port), "--jinja"] + "--cache-ram", "1024", "--ctx-checkpoints-max-mib", "512", "--host", "127.0.0.1", "--port", str(port), "--jinja"] if mtp: command += ["--spec-draft-model", str(args.models["12b-mtp"]), "--spec-type", "draft-mtp", "--spec-draft-n-max", "16", "--spec-draft-p-min", "0.9", "--n-gpu-layers-draft", "999"] From dcbff802c160bad8e3767340a415ab658aa0b426 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:59:58 +0200 Subject: [PATCH 19/21] test(xe2): check known answers throughout the server soak --- scripts/xe2/README.md | 3 ++- scripts/xe2/validate.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index d35aaee2a27a..d31fff06bba5 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -90,7 +90,8 @@ uses 8K context and one slot, with long prompts spanning multiple prefill batche hybrid uses 2K per slot, one slot for 12B and four concurrent slots for 26B. Both 12B phases use MTP. Logs retain responses, timing, temperature, power profile, RSS, and per-client DRM memory accounting. GPU allocations on this UMA device are not all reflected -in process RSS. Each MTP soak phase must actually draft and accept tokens. +in process RSS. Each MTP soak phase must actually draft and accept tokens. The soak also +checks the capital answer each time that prompt recurs. Inspect warm RSS trends and MTP engagement/acceptance in server logs before accepting the soak; request success alone does not prove bounded memory or that speculative decoding engaged. diff --git a/scripts/xe2/validate.py b/scripts/xe2/validate.py index 50564e1d0396..5d57740ed0a5 100644 --- a/scripts/xe2/validate.py +++ b/scripts/xe2/validate.py @@ -355,6 +355,9 @@ def soak(args): output.write(json.dumps({"elapsed": time.monotonic() - started, "memory": snapshot(process.pid), "results": results}, allow_nan=False) + "\n") output.flush() + for index, result in enumerate(results): + if (count + index) % len(prompts) == 0 and "oslo" not in result.get("content", "").casefold(): + raise RuntimeError(f"Soak failed the capital answer check: {name}") count += slots if process.poll() is not None: raise RuntimeError("Server died during soak") From decccf697e18c306fb7d02469b9efdc18459cc19 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:14:38 +0200 Subject: [PATCH 20/21] test(xe2): separate cache churn from warm memory growth --- docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md | 5 +++-- scripts/xe2/README.md | 3 ++- scripts/xe2/summarize-soak.py | 13 ++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md b/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md index a0681f7bc262..cbce3745cf55 100644 --- a/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md +++ b/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md @@ -30,7 +30,8 @@ or newly appended state; measured memory use is reported separately. - Local CPU CI passed Debug 45/45 and Release 47/47 using `GG_BUILD_LOW_PERF=1`. This selection excludes the large-model/high-performance jobs. `LLAMA_FATAL_WARNINGS=OFF` was required for a GCC 16 warning in unchanged - vocabulary construction code. The later model-free logit-metric test also passes. + vocabulary construction code. The later model-free logit-metric test passes in Debug and Release, and all + three telemetry/process infrastructure regression tests pass. ## Corrections to the validation method @@ -68,7 +69,7 @@ but the original long, repetitive raw prompt remained numerically sensitive. |---|---|---| | FA-off/hybrid distributions and KV reuse | Pass | Pass | | Plain-FA-on comparison | Pass | **Fail**: KL at 16 tokens | -| Templated lifecycle, fallback, tools, cancellation | Pass; rechecking final budgets | Pass; rechecking final budgets | +| Templated lifecycle, fallback, tools, cancellation | Pass (6 profiles) | Pass (5 profiles) | | Baseline and hybrid soak, 15 minutes each | Pending | Pending | | Five interleaved runs per mode at 2K/8K/16K/32K | Pending | Pending | diff --git a/scripts/xe2/README.md b/scripts/xe2/README.md index d31fff06bba5..e09c9509aece 100644 --- a/scripts/xe2/README.md +++ b/scripts/xe2/README.md @@ -92,7 +92,8 @@ Both 12B phases use MTP. Logs retain responses, timing, temperature, power profi DRM memory accounting. GPU allocations on this UMA device are not all reflected in process RSS. Each MTP soak phase must actually draft and accept tokens. The soak also checks the capital answer each time that prompt recurs. -Inspect warm RSS trends and MTP engagement/acceptance in server logs before +The soak summary reports warm memory ranges and complete-cycle RSS peak trends. +Inspect these trends and MTP engagement/acceptance in server logs before accepting the soak; request success alone does not prove bounded memory or that speculative decoding engaged. diff --git a/scripts/xe2/summarize-soak.py b/scripts/xe2/summarize-soak.py index 2b125f18aa7c..10bb511e5f80 100644 --- a/scripts/xe2/summarize-soak.py +++ b/scripts/xe2/summarize-soak.py @@ -33,7 +33,7 @@ def main(): parser.add_argument("results", type=Path) parser.add_argument("--warmup-seconds", type=float, default=300) args = parser.parse_args() - print("| Model | Profile | State | Minutes | Requests | Warm RSS MiB; slope | Warm GPU resident GTT MiB; slope | MTP accepted/drafted | Telemetry errors |") + print("| Model | Profile | State | Minutes | Requests | Warm RSS MiB; cycle-peak trend | Warm GPU resident GTT MiB; slope | MTP accepted/drafted | Telemetry errors |") print("|---|---|---|---:|---:|---|---|---:|---:|") complete = True for model in ("12b", "26b"): @@ -53,22 +53,29 @@ def main(): and (model != "12b" or accepted > 0)) complete &= done rss, gpu = [], [] - for sample in samples: + cycles = {} + for index, sample in enumerate(samples): if sample["elapsed"] < args.warmup_seconds: continue memory = sample["memory"] resident = mib(memory.get("VmRSS")) if resident is not None: rss.append((sample["elapsed"], resident)) + cycles.setdefault(index // 3, []).append((sample["elapsed"], resident)) clients = memory.get("drm_clients", {}) amounts = [mib(client["drm-resident-gtt"]) for client in clients.values() if "drm-resident-gtt" in client] if amounts: gpu.append((sample["elapsed"], sum(amounts))) + peaks = [(statistics.mean(t for t, _ in points), max(v for _, v in points)) + for points in cycles.values() if len(points) == 3] + rss_report = (f"{min(v for _, v in rss):.1f}–{max(v for _, v in rss):.1f}; peaks {trend(peaks)}" + if rss else "insufficient samples") errors = sum(bool(sample["memory"].get("read_errors")) for sample in samples) print(f"| {model} | {profile} | {'complete' if done else 'incomplete'} | {elapsed / 60:.2f} | {requests} | " - f"{trend(rss)} | {trend(gpu)} | {accepted}/{drafted} | {errors} |") + f"{rss_report} | {trend(gpu)} | {accepted}/{drafted} | {errors} |") print(f"\nFull four-phase hour completed: {'yes' if complete else 'no'}.") print(f"Memory trends exclude the first {args.warmup_seconds:g} seconds of each phase. " + "RSS peak trends use complete three-round workload cycles to distinguish cache churn from growth. " "CPU RSS and DRM GTT are separate measurements and must not be added as disjoint pools. " "A finite soak can reveal growth; it cannot prove the absence of every leak.") From f41b69b6f035476d4f9f1c4b78a0124c3f0116e7 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:06:00 +0200 Subject: [PATCH 21/21] docs(xe2): record completed hour of sustained-load validation --- .../VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md b/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md index cbce3745cf55..58f20acc56eb 100644 --- a/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md +++ b/docs/backend/VULKAN-GEMMA4-INTEL-XE2-VALIDATION.md @@ -70,7 +70,7 @@ but the original long, repetitive raw prompt remained numerically sensitive. | FA-off/hybrid distributions and KV reuse | Pass | Pass | | Plain-FA-on comparison | Pass | **Fail**: KL at 16 tokens | | Templated lifecycle, fallback, tools, cancellation | Pass (6 profiles) | Pass (5 profiles) | -| Baseline and hybrid soak, 15 minutes each | Pending | Pending | +| Baseline and hybrid soak, 15 minutes each | Pass | Pass (hybrid switch with four-slot fallback) | | Five interleaved runs per mode at 2K/8K/16K/32K | Pending | Pending | The complete 26B parity sweep returns failure: plain FA-on at 16 continuation @@ -97,3 +97,24 @@ Above 2K, and with multiple configured sequences or quantized caches, the hybrid switch intentionally retains stock attention. Benchmarks at those settings measure fallback behavior. The four-slot 26B soak likewise exercises fallback, not eligibility for hybrid attention. + +## Sustained-load results + +All four phases completed on the final serving cache budgets, totaling 60.47 +minutes and 887 requests. No answer, process, finite-output, or telemetry check +failed. Both 12B phases used MTP; the 26B hybrid-switch phase used four slots. + +| Model/profile | Minutes | Requests | MTP accepted/drafted | Warm GPU resident GTT (MiB) | Warm CPU RSS cycle peaks (MiB) | +|---|---:|---:|---:|---:|---:| +| 12B baseline | 15.21 | 21 | 448/728 | 8304.6 | 1830.1–2105.6 | +| 12B hybrid | 15.01 | 270 | 5670/8561 | 7548.9 | 1756.5–1761.5 | +| 26B baseline | 15.07 | 84 | — | 13985.1 | 1549.2–1578.9 | +| 26B four-slot fallback | 15.18 | 512 | — | 14653.9 | 1700.0–1714.5 | + +Warm measurements exclude the first five minutes of each phase. GPU residency +was flat to the displayed precision; CPU cycle-peak slopes ranged from -38.89 +to +0.95 MiB/min. CPU peaks use complete three-round prompt cycles to account +for cache churn. RSS and DRM GTT are separate views of UMA memory and must not +be added as disjoint allocations. This finite soak does not prove the absence +of every leak. Request counts are not comparable throughput benchmarks: context +lengths, prompt lengths, and slot counts differ across profiles.