Production paged + tiered KV cache for the agent army (MAD-126)

.github/workflows/army-test.yml

@@ -0,0 +1,85 @@
+name: CI (army — paged + tiered KV)
+
+# MAD-137 CI integration. Hosted-runner-only:
+#  - Compile for the CPU backend so the mt:: + paged sources catch any
+#    cross-arch breakage immediately.
+#  - Run the mt:: unit tests + the integration tests with no model
+#    (they self-skip when LLAMACPP_TEST_MODELFILE is unset, but the
+#    binaries still have to build cleanly).
+#
+# Real-hardware testing on the army GPUs (R9700, 6900XT, 1070, RX 480)
+# is run manually via scripts/test/run-army-matrix.sh on the dev boxes.
+# Self-hosted runners were intentionally not wired up — they expose the
+# dev boxes to inbound CI traffic, which we don't want.
+
+on:
+  workflow_dispatch:   # manual trigger
+  push:
+    branches: [master, 'feat/MAD-*']
+    paths:
+      - '.github/workflows/army-test.yml'
+      - 'src/llama-kv-cache-paged.{h,cpp}'
+      - 'src/memory-tier/**'
+      - 'tests/test-mt-*'
+      - 'tests/test-paged-*'
+      - 'tests/stress/**'
+      - 'scripts/test/**'
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - '.github/workflows/army-test.yml'
+      - 'src/llama-kv-cache-paged.{h,cpp}'
+      - 'src/memory-tier/**'
+      - 'tests/test-mt-*'
+      - 'tests/test-paged-*'
+      - 'tests/stress/**'
+      - 'scripts/test/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  GGML_NLOOP: 3
+  GGML_N_THREADS: 1
+  LLAMA_LOG_COLORS: 1
+  LLAMA_LOG_PREFIX: 1
+  LLAMA_LOG_TIMESTAMPS: 1
+
+jobs:
+  hosted-cpu-tests:
+    name: "hosted (cpu) — unit + integration smoke"
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+
+      - name: ccache
+        uses: ggml-org/ccache-action@v1.2.21
+        with:
+          key: army-hosted-cpu
+          evict-old-files: 1d
+
+      - name: Configure
+        run: |
+          cmake -B build \
+            -DGGML_CUDA=OFF -DGGML_HIP=OFF -DGGML_VULKAN=OFF \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DLLAMA_BUILD_TESTS=ON
+
+      - name: "Build (mt:: tests + paged-* tests)"
+        run: |
+          cmake --build build --target \
+            test-mt-quant \
+            test-mt-block-pool \
+            test-mt-block-table \
+            test-mt-block-semantic-index \
+            test-mt-tiered-thin \
+            test-paged-lifecycle \
+            test-paged-semantic \
+            -j $(nproc)
+
+      - name: "ctest — mt:: + paged-* (skips integration without model)"
+        working-directory: build
+        run: |
+          ctest -R 'test-mt-|test-paged-' --output-on-failure

.gitignore

@@ -184,6 +184,7 @@ llama.pc
 ggml/ggml-config.cmake
 ggml/ggml-version.cmake
 tests/libgguf-model-data.a
+tests/results/
 *.a
 tests/cmake_install.cmake
 Makefile

README.md

@@ -523,6 +523,7 @@ To learn more about model quantization, [read this documentation](tools/quantize
 - [completion](tools/completion/README.md)
 - [server](tools/server/README.md)
 - [GBNF grammars](grammars/README.md)
+- [Tiered KV cache](docs/memory-tier/) — paged + tiered + semantic prefetch for long-context / multi-agent serving (see [user guide](docs/memory-tier/USER-GUIDE.md), [operator runbook](docs/memory-tier/OPERATOR-RUNBOOK.md), [architecture](docs/memory-tier/ARCHITECTURE.md))
 
 #### Development documentation
 

common/arg.cpp

@@ -1477,9 +1477,10 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     add_opt(common_arg(
         {"--kv-tier-paged-blocks"},
         {"--no-kv-tier-paged-blocks"},
-        "EXPERIMENTAL: enable mt:: paged attention KV cache (vLLM-style block-indexed layout). Routes attention through mt_paged_attention_kernel + mt_reshape_and_cache scatter on HIP/CUDA. Currently validated only on standard non-hybrid transformer models with per-attention-call ctx ≤ ~16k tokens — the kernel's smem footprint scales with ctx and overflows AMD's 64 KiB LDS limit beyond that (clear error logged at dispatch). Hybrid (DeltaNet/Mamba+attention) models compile through the paged path but produce incorrect attention output — needs further debugging. SWA models fall back to the regular kv cache. For multi-agent serving on hybrid models prefer --kv-tiered without this flag (software-only tier eviction; works at --parallel > 1).",
+        "MAD-134: paged-attention KV cache (vLLM-style block-indexed). When --kv-tiered is also set, this is auto-enabled by default (use --no-kv-tier-paged-blocks to opt out). Validated end-to-end on hybrid models (Qwen3.x family); non-hybrid + ctx > 16k may hit kernel LDS limits.",
         [](common_params & params, bool value) {
             params.kv_tier_paged_blocks = value;
+            params.kv_tier_paged_blocks_explicit = true;  // MAD-134: user said something
         }
     ).set_env("LLAMA_ARG_KV_TIER_PAGED_BLOCKS").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
     add_opt(common_arg(
@@ -1492,6 +1493,39 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.kv_tier_paged_block_size = value;
         }
     ).set_env("LLAMA_ARG_KV_TIER_PAGED_BLOCK_SIZE").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
+    add_opt(common_arg(
+        {"--kv-tier-cold-resume"},
+        {"--no-kv-tier-cold-resume"},
+        "MAD-130: when true, skip O_TRUNC on cold-tier files at startup and load the in-memory index from "
+        "${ssd_path}/paged/instance-${INSTANCE_ID}/index.bin (written by the prior server's clean shutdown or "
+        "/slots/save). Lets the server resume cold-tier contents across a clean restart. Sidecar absent or "
+        "invalid → starts fresh with a warning. Default false (legacy behavior: fresh truncation).",
+        [](common_params & params, bool value) {
+            params.kv_tier_cold_resume = value;
+        }
+    ).set_env("LLAMA_ARG_KV_TIER_COLD_RESUME").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
+    add_opt(common_arg(
+        {"--instance-id"}, "ID",
+        "MAD-131: per-instance ID used to scope the cold-tier subdir (${ssd_path}/paged/instance-${ID}/) "
+        "and the per-instance lockfile. Lets multiple llama-server processes share one --kv-tier-ssd-path "
+        "without colliding. Default = process pid as a string. Use a stable ID (e.g. 'main-r9700') for "
+        "deterministic restarts with --kv-tier-cold-resume.",
+        [](common_params & params, const std::string & value) {
+            params.kv_tier_instance_id = value;
+        }
+    ).set_env("LLAMA_ARG_INSTANCE_ID").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
+    add_opt(common_arg(
+        {"--kv-tier-cold-budget-mb"}, "N",
+        "MAD-131: cap the cold-tier pool to N MiB total (across K+V × all attn layers). 0 = no cap "
+        "(size from --kv-tiered cold percentage). Use this to bound SSD wear per instance — e.g. "
+        "10000 (10 GiB) is reasonable for a consumer NVMe with ~600 TBW lifetime serving an army.",
+        [](common_params & params, int value) {
+            if (value < 0) {
+                throw std::invalid_argument("--kv-tier-cold-budget-mb must be >= 0");
+            }
+            params.kv_tier_cold_budget_mb = value;
+        }
+    ).set_env("LLAMA_ARG_KV_TIER_COLD_BUDGET_MB").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}));
     add_opt(common_arg(
         {"-kvu", "--kv-unified"},
         {"-no-kvu", "--no-kv-unified"},

common/common.cpp

@@ -1547,8 +1547,23 @@ struct llama_context_params common_context_params_to_llama(const common_params &
                                               : params.kv_semantic_index.c_str();
     cparams.kv_tier_semantic_threshold  = params.kv_semantic_threshold;
     cparams.kv_tier_semantic_topk       = params.kv_semantic_top_k;
-    cparams.kv_tier_paged_blocks        = params.kv_tier_paged_blocks;
+    // MAD-134: auto-enable paged-blocks when --kv-tiered is set AND
+    // the user didn't explicitly choose either way. The army-goal use
+    // case (hybrid models + multi-agent serving) wants paged-on; the
+    // explicit-pct path stays opt-in via --no-kv-tier-paged-blocks.
+    bool effective_paged = params.kv_tier_paged_blocks;
+    if (!params.kv_tier_paged_blocks_explicit && params.kv_tiered_enabled && !params.kv_tier_paged_blocks) {
+        effective_paged = true;
+        LOG_INF("%s: auto-enabled --kv-tier-paged-blocks (--kv-tiered set; pass "
+                "--no-kv-tier-paged-blocks to opt out)\n", __func__);
+    }
+    cparams.kv_tier_paged_blocks        = effective_paged;
     cparams.kv_tier_paged_block_size    = params.kv_tier_paged_block_size;
+    cparams.kv_tier_cold_resume         = params.kv_tier_cold_resume;
+    cparams.kv_tier_instance_id         = params.kv_tier_instance_id.empty()
+                                              ? nullptr
+                                              : params.kv_tier_instance_id.c_str();
+    cparams.kv_tier_cold_budget_mb      = params.kv_tier_cold_budget_mb;
 
     return cparams;
 }

common/common.h

@@ -613,7 +613,11 @@ struct common_params {
     float kv_semantic_threshold = 0.65f;  // minimum cosine similarity threshold for prefetch hints
     int   kv_semantic_top_k     = 5;      // number of prefetch hints to return
     bool  kv_tier_paged_blocks    = false;  // enable mt:: paged-attention KV cache (vLLM-style block-indexed); standard + hybrid models supported
+    bool  kv_tier_paged_blocks_explicit = false;  // MAD-134: true when user typed --kv-tier-paged-blocks or --no-...; false → auto-default applies
     int   kv_tier_paged_block_size = 16;    // tokens per block when paged_blocks is enabled (must be a power of 2; 16 matches vLLM)
+    bool  kv_tier_cold_resume     = false;  // MAD-130: skip O_TRUNC on cold-tier files; load index sidecar from prior run
+    std::string kv_tier_instance_id;        // MAD-131: per-instance ID for cold-tier subdir + lockfile (default: pid)
+    int   kv_tier_cold_budget_mb  = 0;      // MAD-131: cap cold-pool size to N MiB (0 = no limit beyond percent-derived)
 
     std::string hostname      = "127.0.0.1";
     std::string public_path   = "";                                                                         // NOLINT