From 7fe18b349a836bdcb86dc9cd38a17ec65ef1c632 Mon Sep 17 00:00:00 2001
From: Simon Iribarren <simon.ig13@gmail.com>
Date: Tue, 25 Aug 2026 09:41:27 +0200
Subject: [PATCH] QVAC-24114 fix: fail the load when the model cannot decode
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A model can survive init and still be unable to decode: on a 24 GiB M4 Pro,
Gemma 4 31B Q4_K_M at ctx 1024 loads (Metal over-commits past
recommendedMaxWorkingSetSize) and then every decode fails with 'failed to
decode next token'. Fabric's warmup would not catch it — common_init_from_params
discards the warmup decode status — and LoadFitNormalization disables warmup
anyway. Run one strict BOS/EOS probe decode after context creation and throw
UnableToLoadModel on failure, leaving no KV trace. A successful probe doubles
as a warmup. Skipped for finetuning, which never serves inference from this
context.
---
 .../addon/src/model-interface/LlamaModel.cpp  | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp
index 574918ab62..a41596b822 100644
--- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp
+++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp
@@ -79,6 +79,72 @@ LlamaModel::LlamaModel(
   setInitLoader(InitLoader::LOADER_TYPE::DELAYED);
 }
 
+namespace {
+
+// A model can survive init and still be unable to decode. Measured on a
+// 24 GiB M4 Pro: Gemma 4 31B Q4_K_M at ctx 1024 loads (Metal over-commits
+// past recommendedMaxWorkingSetSize), then every decode fails with
+// "failed to decode next token". Fabric's own warmup would not catch this
+// either: `common_init_from_params` discards the warmup decode status, and
+// LoadFitNormalization disables warmup outright. So run one strict BOS/EOS
+// decode here and fail the load, instead of handing out a handle that dies
+// on the first user request. The successful probe doubles as a warmup:
+// weights are faulted and Metal pipelines compiled before the first token.
+void probeDecodeOrThrow(
+    llama_context* lctx, llama_model* model, const std::string& error) {
+  const llama_vocab* vocab = llama_model_get_vocab(model);
+  std::vector<llama_token> tokens;
+  const llama_token bos = llama_vocab_bos(vocab);
+  const llama_token eos = llama_vocab_eos(vocab);
+  if (bos != LLAMA_TOKEN_NULL) {
+    tokens.push_back(bos);
+  }
+  if (eos != LLAMA_TOKEN_NULL) {
+    tokens.push_back(eos);
+  }
+  if (tokens.empty()) {
+    tokens.push_back(0);
+  }
+
+  int32_t status = 0;
+  if (llama_model_has_encoder(model)) {
+    status = llama_encode(
+        lctx,
+        llama_batch_get_one(
+            tokens.data(), static_cast<int32_t>(tokens.size())));
+    llama_token decoderStart = llama_model_decoder_start_token(model);
+    if (decoderStart == LLAMA_TOKEN_NULL) {
+      decoderStart = bos;
+    }
+    tokens.assign(1, decoderStart);
+  }
+  if (status == 0 && llama_model_has_decoder(model)) {
+    status = llama_decode(
+        lctx,
+        llama_batch_get_one(
+            tokens.data(), static_cast<int32_t>(tokens.size())));
+  }
+
+  // Leave no trace of the probe regardless of outcome.
+  llama_memory_clear(llama_get_memory(lctx), true);
+  llama_synchronize(lctx);
+  llama_perf_context_reset(lctx);
+
+  if (status != 0) {
+    throw qvac_errors::StatusError(
+        ADDON_ID,
+        error,
+        string_format(
+            "%s: model loaded but cannot decode (probe decode failed with "
+            "status %d); the resolved configuration does not fit this "
+            "device's usable memory\n",
+            __func__,
+            status));
+  }
+}
+
+} // namespace
+
 void LlamaModel::reload(
     std::optional<FinetuneConfigOverrides> newFinetuneOverrides) {
   {
@@ -231,6 +297,15 @@ void LlamaModel::init(bool acquireLock) {
       params,
       std::move(llamaInit));
 
+  // Finetuning drives its own graph and never serves inference from this
+  // context, so the inference probe would only add noise there.
+  if (snap->llmContext_ && !params.training) {
+    probeDecodeOrThrow(
+        snap->llmContext_->getCtx(),
+        snap->llmContext_->getModel(),
+        errorWhenFailed);
+  }
+
   if (snap->configuredNDiscarded_ > 0 && snap->llmContext_) {
     snap->llmContext_->setNDiscarded(snap->configuredNDiscarded_);
   }