diff --git a/.env.example b/.env.example index 3c5810e..dcb558c 100644 --- a/.env.example +++ b/.env.example @@ -9,8 +9,10 @@ WORKER=you@192.0.2.2 # path is used on both Sparks. Default: $HOME/models/sparkglm #MODEL_ROOT=/srv/models/sparkglm -# 4x512k (default): 4 requests x 512K context. 8x128k: 8 requests x 128K. -#PROFILE=4x512k +# 4x1m (default): 4 requests x 1M context, the model's full window; the +# requests share one KV pool (~2.62M tokens with the shard and the carveout at +# 0.91; a 925K-token needle prompt is found). 4x512k: 4 x 512K. 8x128k: 8 x 128K. +#PROFILE=4x1m # Share of each Spark's unified memory for the engine (0.80-0.95, default 0.88). # On Sparks that run nothing else, 0.91 roughly doubles the KV pool that holds @@ -32,6 +34,13 @@ WORKER=you@192.0.2.2 # /run/lock/sparkglm. Default off; set 0 or delete the line to roll back. #DISPLAY_CARVEOUT=1 +# The KV cache is split between the two Sparks: each stores the latents of half +# the blocks, so the shared pool holds about 1.8x the tokens. With the display +# carveout at 0.91: ~2.62M tokens (1.36M unsplit), lowest free memory 4.5 GB in +# a four-session 204K fill. On by default; the disk prefix cache cannot run +# beside it yet, so setting PREFIX_CACHE_DIR turns it off. 0 turns it off. +#KV_SHARD=0 + # Build the image from source instead of pulling ghcr.io/enntity/atlas-sparkglm. #BUILD=1 # Parallel compile jobs for a source build; lower it if the build runs out of memory. diff --git a/README.md b/README.md index 4ded176..50239eb 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,7 @@ comparison videos. ## The engine Atlas here is [`Enntity/atlas`](https://github.com/Enntity/atlas) branch -`sparkglm/atlas-20261007-strings` (pinned in +`sparkglm/atlas-20261009` (pinned in [`install/atlas-source.json`](install/atlas-source.json)), built from three layers: diff --git a/docs/INSTALL-MANUAL.md b/docs/INSTALL-MANUAL.md index fa482d7..6fe4c5f 100644 --- a/docs/INSTALL-MANUAL.md +++ b/docs/INSTALL-MANUAL.md @@ -5,7 +5,7 @@ time. Run steps 1–4 on **both** Sparks with the same `MODEL_ROOT`. | Component | Pin | |---|---| -| Engine | [`Enntity/atlas`](https://github.com/Enntity/atlas) `sparkglm/atlas-20261007-strings` @ `7838b34c` ([`install/atlas-source.json`](../install/atlas-source.json)) | +| Engine | [`Enntity/atlas`](https://github.com/Enntity/atlas) `sparkglm/atlas-20261009` @ `e034500d` ([`install/atlas-source.json`](../install/atlas-source.json)) | | Model | [`nvidia/GLM-5.3-Flash-NVFP4`](https://huggingface.co/nvidia/GLM-5.3-Flash-NVFP4) @ `423acf37583782c51c142d145aef733d72943d93` (MIT) | | Drafter | [`incoai/GLM-5.3-Flash-DFlash2`](https://huggingface.co/incoai/GLM-5.3-Flash-DFlash2) @ `7d74cdd881ed7e32c31175984a67823127b66cfe` (CC BY-NC-ND 4.0) | | Native dependencies | FlashInfer `8eccd0c1`, CUTLASS `cf064d2e`, FlashKDA ([`install/flash_kda/`](../install/flash_kda/)), Rust 1.93.1, CUDA 13.0 | diff --git a/install/atlas-source.json b/install/atlas-source.json index 6a4c1ef..d9ff7b8 100644 --- a/install/atlas-source.json +++ b/install/atlas-source.json @@ -2,24 +2,24 @@ "schema": "sparkglm.atlas-engine-source/v2", "note": "The Atlas engine is built from the public Enntity/atlas fork at this exact commit; build.sh refuses any other tree.", "repository": "https://github.com/Enntity/atlas.git", - "branch": "sparkglm/atlas-20261007-strings", - "commit": "7838b34c18785ea876a758eb4806d005d3fcda38", - "tree": "d20749a55db35bae54a75ab2d67c5ce3def2021f", + "branch": "sparkglm/atlas-20261009", + "commit": "e034500d858742d131529e56179453e8afb45dce", + "tree": "edc66e7c3ca234e6aac3026c2b8242119dc42183", "layers": { "atlas_inf_main_base": "6a3d24ecb891adac1fac8edeef92de29bb6f3e15", "upstream_candidate": { - "branch": "upstream/glm53-flash-20261007-strings", - "commit": "89f2927b4b729db8da3925449d3e252d321d13db", - "note": "upstream/glm53-flash-20261004-gram (619da6ff: strict structured output, linear-cost grammar compile, masked DFlash verify for strict grammars) plus two tool-use fixes: the GLM (poolside_v1) tool parser keeps argument text and types it from the tool schema, so string parameters are no longer turned into booleans, numbers or lists; and the GLM template omits the reasoning-effort header on the empty think block (thinking off with tools)." + "branch": "upstream/glm53-flash-20261009", + "commit": "576ad34e", + "note": "upstream/glm53-flash-20261007-strings (GLM tool arguments keep their schema types; no effort header on the empty think block) plus copy (prompt-lookup) drafts beside DFlash2 (ATLAS_DFLASH_COPY_DRAFTS), the bare role-literal strip limited to ChatML models (GLM streamed replies lost the words tool/user/assistant), and Poolside tool-call headers held until the call validates." }, "sparkglm_only": { "commit": "72588541a466f53cb9d38afd74c515e14ab601db", - "note": "The unchanged FlashKDA/native sparse-MLA prefill bridges and GB10 display-carveout KV placement from fdd2f965, then the carveout takes latent KV pools only (ATLAS_KV_CARVEOUT_ORDER, default latent): the GPU maps the carveout L2-uncached, and sparse-index buffers there slowed 64K cold prefill by 7% (Enntity/sparkglm#30). Fork-only; used only with DISPLAY_CARVEOUT=1." + "note": "The unchanged FlashKDA/native sparse-MLA prefill bridges and GB10 display-carveout KV placement from fdd2f965, then the carveout takes latent KV pools only (ATLAS_KV_CARVEOUT_ORDER, default latent): the GPU maps the carveout L2-uncached, and sparse-index buffers there slowed 64K cold prefill by 7% (Enntity/sparkglm#30). Fork-only; used only with DISPLAY_CARVEOUT=1. Plus: sharded latent pools (ATLAS_GLM_KV_SHARD, opt-in through SPARKGLM_KV_SHARD) size and place in the display carveout." }, "integration": { - "branch": "fix/poolside-string-args", - "commit": "72588541 + 1a39c032 + 7838b34c", - "note": "This pin is sparkglm/atlas-20261005-cvplace (72588541) plus the two upstream-candidate fixes, cherry-picked unchanged; on the MLPerf edge-agentic BFCL leg the parser fix moves 83.72 to 88.74 overall (Enntity/sparkglm-research experiments/2026-10-06-mlperf-edge-agentic)." + "branch": "sparkglm/atlas-20261009", + "commit": "7838b34c + 9 cherry-picks", + "note": "sparkglm/atlas-20261007-strings plus the kv-shard carveout placement (exp/kvshard-20261008), copy drafts (exp/copy-drafts-20261008), the role-literal fix (fix/role-literal-strip-chatml) and the Poolside header hold (sparkqwen/rc15-fixes 25d14fcb), cherry-picked with -x." } }, "kernel_target": "glm-5.3-flash", diff --git a/install/profiles/4x1m.json b/install/profiles/4x1m.json new file mode 100644 index 0000000..3954a3a --- /dev/null +++ b/install/profiles/4x1m.json @@ -0,0 +1,223 @@ +{ + "schema": "sparkglm.atlas-install-profile/v1", + "status": "candidate_2026-10-09", + "environment": { + "ATLAS_GLM_K3_MLA_QUERY_BATCHM": "1", + "ATLAS_GLM_K3_MLA_QUERY_COMPARE": "0", + "ATLAS_GLM_SPARSE_DECODE_SPLIT": "1", + "ATLAS_GLM_K3_MLA_O_BATCHM": "1", + "ATLAS_GLM_K3_HEAD_BATCHM": "0", + "ATLAS_GLM_C3_GROUPED_MOE": "1", + "ATLAS_GLM_SPARSE_DECODE_TC": "1", + "ATLAS_GLM_SPARSE_NATIVE": "1", + "ATLAS_GLM_SPARSE_NATIVE_LIBRARY": "/opt/atlas/native/libatlas_glm_sparse_native.so", + "ATLAS_GLM_HC_TF32_PREWARM": "1", + "ATLAS_KDA_FLASH_PREFILL": "1", + "ATLAS_KDA_FLASH_LIBRARY": "/opt/atlas/native/libatlas_mango_flash.so", + "ATLAS_EP_PROTOCOL": "v2", + "ATLAS_GLM_MTP_LONG_CONTEXT": "1", + "ATLAS_CUBLAS_GEMM": "1", + "ATLAS_DEBUG_NO_GRAPH": "0", + "ATLAS_EP_GRAPHS": "1", + "ATLAS_GLM_C4_DECODE": "0", + "ATLAS_GLM_C4_SPARSE": "0", + "ATLAS_GLM_INDEPENDENT_DECODE": "0", + "ATLAS_GLM_INDEX_ROW_GROUP": "8", + "ATLAS_GLM_INDEX_WMMA": "1", + "ATLAS_GLM_K5_BATCHED_CONV_SNAPSHOT": "1", + "ATLAS_GLM_K5_BATCHED_RECURRENT_SNAPSHOT": "1", + "ATLAS_GLM_K5_BATCHED_SHARED": "0", + "ATLAS_GLM_K5_COMPACT_MOE": "1", + "ATLAS_GLM_K5_DENSE_EXACT": "1", + "ATLAS_GLM_K5_FUSED_COMPACT_GATE_UP": "1", + "ATLAS_GLM_K5_FUSED_DENSE_PAIRS": "0", + "ATLAS_GLM_K5_FUSED_DENSE_TRIPLE": "1", + "ATLAS_GLM_K5_FUSED_MOE_HC": "1", + "ATLAS_GLM_K5_FUSED_QKV": "1", + "ATLAS_GLM_K5_FUSED_SHARED_GATE_UP": "1", + "ATLAS_GLM_K5_FUSED_TP_HC": "1", + "ATLAS_GLM_K5_GROUPED_MOE": "1", + "ATLAS_GLM_K5_HC_CUBLAS": "1", + "ATLAS_GLM_K5_ROUTER_M5": "1", + "ATLAS_GLM_M5_ROUTER_BN4": "1", + "ATLAS_GLM_M5_ROUTER_BN4_VERIFY": "0", + "ATLAS_GLM_MOE_GATE_UP_M16": "1", + "ATLAS_GLM_MOE_GATE_UP_M16_VERIFY": "0", + "ATLAS_GLM_MTP_ALL_GATHER": "0", + "ATLAS_GLM_MTP_BATCHED_PREFILL": "0", + "ATLAS_GLM_MTP_BF16_DRAFTS": "0", + "ATLAS_GLM_MTP_DISTRIBUTED": "0", + "ATLAS_GLM_MTP_DISTRIBUTED_ARGMAX": "0", + "ATLAS_GLM_MTP_FUSED_EH_CHECK": "0", + "ATLAS_GLM_MTP_FUSED_EH_NORM": "0", + "ATLAS_GLM_MTP_HIDDEN_TRACE": "0", + "ATLAS_GLM_MTP_K5_LEDGER": "0", + "ATLAS_GLM_MTP_KV_REPAIR_VERIFY": "0", + "ATLAS_GLM_MTP_NVFP4_EH": "0", + "ATLAS_GLM_MTP_NVFP4_WO": "0", + "ATLAS_GLM_MTP_PROFILE": "0", + "ATLAS_GLM_MTP_REPAIR": "0", + "ATLAS_GLM_MTP_SERIAL_PREFILL": "0", + "ATLAS_GLM_MULTI_SEQ_SPARSE": "0", + "ATLAS_GLM_MULTI_SEQ_SPARSE_GRAPHS": "0", + "ATLAS_GLM_PAGED_PREFILL_BF16_GEMM": "1", + "ATLAS_GLM_SPARSE_HEAD_GROUP": "8", + "ATLAS_GLM_SSM_NORMALIZE": "0", + "ATLAS_GLM_TARGET_SHARED_FP8": "1", + "ATLAS_GLM_TARGET_SHARED_FP8_VERIFY": "0", + "ATLAS_GLM_MLA_MULTI_SEQ": "1", + "ATLAS_GLM_MLA_BATCH23": "1", + "ATLAS_GLM_MTP1_VERIFY_GRAPH": "0", + "ATLAS_GLM_TP_VERIFY_GRAPH": "0", + "ATLAS_HC_CUBLAS_PREFILL": "1", + "ATLAS_KDA_REGRESIDENT_PREFILL": "1", + "ATLAS_GLM_KV_CAP_TO_CONTEXTS": "0", + "ATLAS_MOE_GROUPED_CUTLASS": "0", + "ATLAS_MOE_PREFILL_EXACT_TILES": "1", + "ATLAS_MOE_PREFILL_MAX_LOAD_FACTOR": "8", + "ATLAS_MOE_SHARED_REDUCE_OVERLAP": "1", + "ATLAS_MTP_DRAFT_CONF": "0", + "ATLAS_MTP_SINGLE_DEPTH_ADAPT": "0", + "ATLAS_MTP_SPEC_THINK": "0", + "ATLAS_NO_DECODE_GRAPHS": "0", + "ATLAS_NO_DECODE_GRAPHS_MULTISEQ": "0", + "ATLAS_NVFP4_FUSED_SILU_QUANT": "1", + "ATLAS_NVFP4_MMQ_MOE": "0", + "ATLAS_NVFP4_PREQUANT_MOE": "1", + "ATLAS_NVFP4_VECSCALE": "1", + "ATLAS_UNIFIED_MOE_LAYOUT": "1", + "CUDA_CACHE_PATH": "/atlas-cuda-cache", + "LD_LIBRARY_PATH": "/opt/atlas/native:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64", + "NCCL_ALGO": "Ring", + "NCCL_BUFFSIZE": "33554432", + "NCCL_CUMEM_ENABLE": "0", + "NCCL_CUMEM_HOST_ENABLE": "0", + "NCCL_DEBUG": "WARN", + "NCCL_DMABUF_ENABLE": "0", + "NCCL_IB_ADDR_FAMILY": "AF_INET", + "NCCL_IB_DISABLE": "0", + "NCCL_CROSS_NIC": "0", + "NCCL_IB_RETRY_CNT": "7", + "NCCL_IB_ROCE_VERSION_NUM": "2", + "NCCL_IB_TIMEOUT": "22", + "NCCL_MAX_NCHANNELS": "2", + "NCCL_MIN_NCHANNELS": "1", + "NCCL_NET": "IB", + "NCCL_NET_GDR_C2C": "0", + "NCCL_NET_GDR_LEVEL": "0", + "NCCL_NVLS_ENABLE": "0", + "NCCL_PROTO": "Simple", + "PATH": "/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "RUST_LOG": "info", + "ATLAS_GLM_PAGED_PREFILL_MLA_GEMM": "1", + "ATLAS_GLM_SPARSE_PREFILL_TC": "1", + "ATLAS_GLM_PREFILL_2048": "0", + "ATLAS_GLM_DENSE_PREFILL_BF16": "2", + "ATLAS_GLM_SPARSE_PREFILL_KV_REUSE": "1", + "ATLAS_GLM_PREFILL_4096": "1", + "ATLAS_GLM_ROUTER_PREFILL_BN32": "1", + "ATLAS_GLM_HC_PREFILL_VEC": "1", + "ATLAS_GLM_KDA_PREFILL_LT_FP8": "1", + "ATLAS_HC_GENERIC_MOE_ARM": "1", + "ATLAS_GLM_LONG_BATCH_VERIFY": "1", + "ATLAS_GLM_MTP_REPAIR_CONTEXT": "65536", + "ATLAS_GLM_HC_FUSED_PREFILL": "1", + "ATLAS_GLM_VERIFY_VOCAB_SPLIT": "1", + "ATLAS_GLM_DFLASH": "1", + "ATLAS_DFLASH_OPTION_B": "1", + "ATLAS_W4A16_TC": "1", + "ATLAS_GLM_ROUTER_PREFILL_CUBLAS": "1", + "ATLAS_GLM_HC_BF16": "1", + "ATLAS_DFLASH_MXFP8": "1", + "ATLAS_BF16_GEMM_CUTLASS": "1", + "ATLAS_GLM_HC_POST_MIX": "1", + "ATLAS_DFLASH_MULTI_DRAFT_CAP": "7", + "ATLAS_RDMA_ALLREDUCE": "1", + "ATLAS_GLM_PREFILL_SP": "1", + "ATLAS_GLM_LM_HEAD_MXFP8": "1", + "ATLAS_GLM_MLA_MXFP8": "1", + "ATLAS_DFLASH_MXFP8_HEAD": "1", + "ATLAS_DFLASH_NVFP4_HEAD": "1", + "ATLAS_MOE_PREQUANT_K128": "1", + "ATLAS_GLM_HC_DECODE_SEAM": "1", + "ATLAS_DFLASH_NVFP4_TC": "1", + "ATLAS_DFLASH_DROP_BF16": "1", + "ATLAS_DFLASH_ADAPTIVE_WIDTH": "1", + "ATLAS_DFLASH_SPEC_THINK": "1", + "ATLAS_GLM_SHARED_TP_SPLIT": "1", + "ATLAS_GLM_EXPERT_TP": "1", + "ATLAS_MARCONI_PREFILL_ONLY": "1", + "ATLAS_PREFIX_SUBBLOCK": "0", + "ATLAS_DFLASH_CTX_NVFP4": "1", + "ATLAS_PDL": "1", + "ATLAS_GLM_SPARSE_PREFILL_PIPE": "1", + "ATLAS_GLM_MOE_PREFILL_PERSIST": "1", + "ATLAS_GLM_MOE_UNPERMUTE_VEC": "1", + "ATLAS_GLM_INDEX_LOGITS_V2": "1", + "ATLAS_GLM_INDEX_SPLIT": "1", + "ATLAS_GLM_ROUTER_PREFILL_CUTLASS": "1", + "ATLAS_GLM_MOE_DECODE_M16": "1", + "ATLAS_GLM_MOE_DOWN_ZSKIP": "1", + "ATLAS_GLM_PC_EVICT": "1", + "ATLAS_GLM_PC_BRANCH": "1", + "ATLAS_GLM_WARM_SKIP_CACHED": "1", + "ATLAS_GLM_WARM_CHUNK_RUN": "1", + "ATLAS_PREFILL_SRPT": "1", + "ATLAS_DFLASH_FIRST_APPEND": "none", + "ATLAS_GLM_CMD_RDMA": "1", + "ATLAS_GLM_MOE_DECODE_STREAM": "1", + "ATLAS_GLM_DRAFT_TP": "1", + "ATLAS_GLM_DECODE_FUSE": "1", + "ATLAS_GLM_DECODE_GEMV_BATCH": "1", + "ATLAS_GLM_MOE_DECODE_L2PF": "1", + "ATLAS_GLM_MOE_STREAM_NOSYNC": "1", + "ATLAS_GLM_DRAFT_TP_BATCH": "1", + "ATLAS_RDMA_ONESHOT": "1", + "ATLAS_RDMA_PAIR_CHAIN": "1", + "ATLAS_GLM_HC_SEAM_ILP": "3", + "ATLAS_GLM_DRAFT_TP_CTX": "1", + "ATLAS_DFLASH_CTX_ASYNC_POS": "1", + "ATLAS_GLM_STRICT_SPEC": "1", + "ATLAS_DFLASH_CONF_WIDTH": "1", + "ATLAS_USE_HF_REF_JSON_DUMPS": "1", + "ATLAS_DFLASH_COPY_DRAFTS": "1", + "ATLAS_DFLASH_COPY_REPLY_MATCH": "16", + "ATLAS_DFLASH_COPY_MISS_MAX": "3" + }, + "server_argv": [ + "serve", + "${model}", + "--kernel-target=glm-5.3-flash", + "--world-size=2", + "--tp-size=2", + "--max-seq-len=1048576", + "--gpu-memory-utilization=0.88", + "--ep-size=2", + "--max-prefill-tokens=8192", + "--max-batch-size=4", + "--max-num-seqs=4", + "--oom-guard-mb=4096", + "--kv-cache-dtype=fp8_g128", + "--ssm-h-dtype=f32", + "--ssm-rollback-mode=records", + "--block-size=16", + "--ssm-cache-slots=16", + "--enable-prefix-caching", + "--ssm-checkpoint-interval=0", + "--gpu-ordinal=0", + "--swap-space-gb=0", + "--no-auto-swap", + "--no-tui", + "--lm-head-dtype=bf16", + "--request-timeout=0", + "--video-allow-ffmpeg", + "--vision-allow-remote-images", + "--tool-call-parser=poolside_v1", + "--video-max-frames=32", + "--dflash", + "--draft-model=${drafter}", + "--dflash-gamma=8", + "--dflash-window-size=2048", + "--mtp-gate=force" + ] +} diff --git a/install/profiles/4x512k.json b/install/profiles/4x512k.json index 1818c9a..2518f77 100644 --- a/install/profiles/4x512k.json +++ b/install/profiles/4x512k.json @@ -179,7 +179,10 @@ "ATLAS_DFLASH_CTX_ASYNC_POS": "1", "ATLAS_GLM_STRICT_SPEC": "1", "ATLAS_DFLASH_CONF_WIDTH": "1", - "ATLAS_USE_HF_REF_JSON_DUMPS": "1" + "ATLAS_USE_HF_REF_JSON_DUMPS": "1", + "ATLAS_DFLASH_COPY_DRAFTS": "1", + "ATLAS_DFLASH_COPY_REPLY_MATCH": "16", + "ATLAS_DFLASH_COPY_MISS_MAX": "3" }, "server_argv": [ "serve", diff --git a/install/profiles/8x128k.json b/install/profiles/8x128k.json index ee80753..7cb52da 100644 --- a/install/profiles/8x128k.json +++ b/install/profiles/8x128k.json @@ -178,7 +178,10 @@ "ATLAS_DFLASH_CTX_ASYNC_POS": "1", "ATLAS_GLM_STRICT_SPEC": "1", "ATLAS_DFLASH_CONF_WIDTH": "1", - "ATLAS_USE_HF_REF_JSON_DUMPS": "1" + "ATLAS_USE_HF_REF_JSON_DUMPS": "1", + "ATLAS_DFLASH_COPY_DRAFTS": "1", + "ATLAS_DFLASH_COPY_REPLY_MATCH": "16", + "ATLAS_DFLASH_COPY_MISS_MAX": "3" }, "server_argv": [ "serve", diff --git a/install/serve.py b/install/serve.py index 15abcff..5c7297d 100644 --- a/install/serve.py +++ b/install/serve.py @@ -87,6 +87,25 @@ def disk_prefix_cache(gb): 'ATLAS_SSM_TIER_SLOTS': '2'} +def kv_shard(environ, disk_tier): + """Engine settings that split GLM's MLA latent KV between the two ranks, + when SPARKGLM_KV_SHARD=1 (default off): each rank stores half the blocks' + latents, so the shared pool holds about 1.8x the tokens for the same + memory (with the display carveout at 0.89: 120,186 blocks against 67,608; + single-request decode unchanged, four at once -3%). The disk prefix-cache + tier spills latents by physical block, which a shard cannot serve, so the + two exclude each other.""" + switch = environ.get('SPARKGLM_KV_SHARD', '0') + if switch not in ('0', '1'): + raise ValueError('SPARKGLM_KV_SHARD must be 0 or 1') + if switch == '0': + return {} + if disk_tier: + raise ValueError('SPARKGLM_KV_SHARD=1 and the disk prefix cache (SPARKGLM_PREFIX_CACHE_GB) ' + 'cannot be combined; switch one off') + return {'ATLAS_GLM_KV_SHARD': '1', 'ATLAS_GLM_KV_SHARD_COMPACT': '1'} + + def launch(environ, profile): rank = environ.get('NODE_RANK', '') if rank not in ('0', '1'): @@ -125,7 +144,9 @@ def launch(environ, profile): raise ValueError('SPARKGLM_GPU_MEMORY_UTILIZATION must be 0.80 to 0.95') args = [f'--gpu-memory-utilization={util}' if a.startswith('--gpu-memory-utilization=') else a for a in args] - env.update(disk_prefix_cache(environ.get('SPARKGLM_PREFIX_CACHE_GB', ''))) + disk_tier = disk_prefix_cache(environ.get('SPARKGLM_PREFIX_CACHE_GB', '')) + env.update(disk_tier) + env.update(kv_shard(environ, disk_tier)) name = environ.get('SERVED_MODEL_NAME', 'glm-5.3-flash-atlas') if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._/-]{0,127}', name): raise ValueError('Invalid SERVED_MODEL_NAME') diff --git a/install/start-node.sh b/install/start-node.sh index 62a4af7..0131b36 100755 --- a/install/start-node.sh +++ b/install/start-node.sh @@ -12,7 +12,7 @@ usage: start-node.sh --rank 0|1 --leader-address IP --model-root DIR --image TAG [--fabric-interface IFACE] [--fabric-hca rocep1s0f0] [--profile 4x512k|8x128k|FILE] [--gpu-memory-utilization 0.80-0.95] [--prefix-cache-dir DIR [--prefix-cache-gb 16-100]] - [--display-carveout] [--no-fabric-siblings] [--cuda-cache DIR] [--name NAME] + [--display-carveout] [--kv-shard] [--no-fabric-siblings] [--cuda-cache DIR] [--name NAME] --leader-address rank 0's IPv4 address on the direct Spark-to-Spark fabric --model-root directory holding nvidia--GLM-5.3-Flash-NVFP4, @@ -36,12 +36,15 @@ usage: start-node.sh --rank 0|1 --leader-address IP --model-root DIR --image TAG (2 GiB more KV cache). The container starts with CAP_SYS_ADMIN to export it, which the server drops before it loads anything; use it on both ranks (default: off) + --kv-shard split the KV cache between the two Sparks, about 1.8x the + pool for the same memory; use it on both ranks, without + --prefix-cache-dir (default: off) --cuda-cache persistent CUDA JIT cache (default: ~/.cache/atlas-cuda) EOF exit 2 } rank="" leader="" iface="" model_root="" image="" hca=rocep1s0f0 -cache="$HOME/.cache/atlas-cuda" name="" profile=4x512k util="" pc_dir="" pc_gb="" carveout="" siblings=1 +cache="$HOME/.cache/atlas-cuda" name="" profile=4x512k util="" pc_dir="" pc_gb="" carveout="" siblings=1 kvshard=0 while [[ $# -gt 0 ]]; do case "$1" in --rank) rank=$2; shift 2 ;; @@ -57,6 +60,7 @@ while [[ $# -gt 0 ]]; do --prefix-cache-dir) pc_dir=$2; shift 2 ;; --prefix-cache-gb) pc_gb=$2; shift 2 ;; --display-carveout) carveout=1; shift ;; + --kv-shard) kvshard=1; shift ;; --no-fabric-siblings) siblings=0; shift ;; *) usage ;; esac @@ -125,7 +129,7 @@ docker run -d --name "$name" --restart no --network host --ipc host \ --mount "type=bind,src=$cache,dst=/atlas-cuda-cache" ${profile_mount[@]+"${profile_mount[@]}"} ${pc_args[@]+"${pc_args[@]}"} \ ${carveout_args[@]+"${carveout_args[@]}"} \ -e NODE_RANK="$rank" -e MASTER_ADDR="$leader" -e MASTER_PORT=29510 \ - -e FABRIC_INTERFACE="$iface" -e FABRIC_HCA="$hca" -e FABRIC_SIBLINGS="$siblings" \ + -e FABRIC_INTERFACE="$iface" -e FABRIC_HCA="$hca" -e FABRIC_SIBLINGS="$siblings" -e SPARKGLM_KV_SHARD="$kvshard" \ -e MODEL_PATH="$overlay" -e DRAFTER_PATH="$drafter" \ -e SERVED_MODEL_NAME=glm-5.3-flash-atlas -e SPARKGLM_PROFILE="$profile" \ ${util:+-e SPARKGLM_GPU_MEMORY_UTILIZATION="$util"} \ diff --git a/install/test_install.py b/install/test_install.py index 72ea7d8..b8df9b8 100644 --- a/install/test_install.py +++ b/install/test_install.py @@ -145,7 +145,7 @@ def test_a_second_physical_port_is_not_a_sibling(self): def test_both_profiles_cache_prompt_prefixes(self): # Multi-turn agents resend the whole conversation every turn; without # prefix caching each turn re-prefills it from scratch. - for name in ('4x512k', '8x128k'): + for name in ('4x512k', '4x1m', '8x128k'): profile = json.loads(serve.profile_path({'SPARKGLM_PROFILE': name}, HERE).read_text()) argv, _ = serve.launch(self.environment, profile) with self.subTest(profile=name): @@ -164,7 +164,7 @@ def test_profiles_enable_only_exact_engine_options(self): # the option off; the lossy and the not-yet-qualified options stay out # of a shipped profile. envs = [json.loads(serve.profile_path({'SPARKGLM_PROFILE': name}, HERE).read_text())['environment'] - for name in ('4x512k', '8x128k')] + for name in ('4x512k', '4x1m', '8x128k')] exact = ('ATLAS_GLM_SPARSE_PREFILL_PIPE', 'ATLAS_GLM_MOE_PREFILL_PERSIST', 'ATLAS_GLM_MOE_UNPERMUTE_VEC', 'ATLAS_GLM_INDEX_LOGITS_V2', 'ATLAS_GLM_INDEX_SPLIT', 'ATLAS_GLM_ROUTER_PREFILL_CUTLASS', 'ATLAS_GLM_MOE_DECODE_M16', @@ -175,11 +175,14 @@ def test_profiles_enable_only_exact_engine_options(self): 'ATLAS_GLM_DRAFT_TP_BATCH', 'ATLAS_RDMA_ONESHOT', 'ATLAS_RDMA_PAIR_CHAIN', 'ATLAS_GLM_DRAFT_TP_CTX', 'ATLAS_DFLASH_CTX_ASYNC_POS', 'ATLAS_GLM_STRICT_SPEC') # Lossless but not bit-for-bit against the option off: the prefill queue - # order, and a verify width taken from the drafter's confidence (verify - # numerics already depend on the width). The first draft of a request no - # longer reads a row the previous request left behind. + # order, a verify width taken from the drafter's confidence (verify + # numerics already depend on the width), and copy (prompt-lookup) drafts, + # which change only how many tokens a step accepts (byte-identical at a + # pinned width). The first draft of a request no longer reads a row the + # previous request left behind. policy = {'ATLAS_PREFILL_SRPT': '1', 'ATLAS_DFLASH_CONF_WIDTH': '1', - 'ATLAS_DFLASH_FIRST_APPEND': 'none'} + 'ATLAS_DFLASH_FIRST_APPEND': 'none', 'ATLAS_DFLASH_COPY_DRAFTS': '1', + 'ATLAS_DFLASH_COPY_REPLY_MATCH': '16', 'ATLAS_DFLASH_COPY_MISS_MAX': '3'} held_back = ('ATLAS_GLM_MLA_KVB_MXFP8', 'ATLAS_GLM_INDEX_MXFP8', 'ATLAS_GLM_KV_SHARD', 'ATLAS_GLM_PC_FINISH_LEAF', 'ATLAS_GLM_VERIFY_GRAPH', 'ATLAS_GLM_STEP_FUSE', 'ATLAS_GLM_LAYER_FORK', 'ATLAS_GLM_L2_AHEAD', @@ -221,6 +224,17 @@ def test_env_example_leaves_the_display_carveout_off(self): self.assertIn('#DISPLAY_CARVEOUT=1', lines) self.assertFalse([l for l in lines if l.startswith('DISPLAY_CARVEOUT')]) + def test_kv_shard_is_opt_in_and_excludes_the_disk_tier(self): + _, env = serve.launch(self.environment, self.profile) + self.assertNotIn('ATLAS_GLM_KV_SHARD', env) + _, env = serve.launch(dict(self.environment, SPARKGLM_KV_SHARD='1'), self.profile) + self.assertEqual((env['ATLAS_GLM_KV_SHARD'], env['ATLAS_GLM_KV_SHARD_COMPACT']), ('1', '1')) + for bad in ('yes', '2', ' 1', ''): + with self.subTest(switch=bad), self.assertRaises(ValueError): + serve.launch(dict(self.environment, SPARKGLM_KV_SHARD=bad), self.profile) + with self.assertRaises(ValueError): + serve.launch(dict(self.environment, SPARKGLM_KV_SHARD='1', SPARKGLM_PREFIX_CACHE_GB='48'), self.profile) + def test_disk_prefix_cache_is_off_unless_sized(self): # Ambient tier variables are dropped like any other ATLAS_* flag. for extra in ({}, {'SPARKGLM_PREFIX_CACHE_GB': ''}, {'SPARKGLM_PREFIX_CACHE_GB': '0'}, @@ -279,6 +293,10 @@ def start(rank, *extra, **fake): run, docker_run = start('1', '--no-fabric-siblings') self.assertEqual(run.returncode, 0, run.stderr) self.assertIn('-e FABRIC_SIBLINGS=0 ', docker_run[0]) + self.assertIn('-e SPARKGLM_KV_SHARD=0 ', docker_run[0]) + run, docker_run = start('0', '--kv-shard') + self.assertEqual(run.returncode, 0, run.stderr) + self.assertIn('-e SPARKGLM_KV_SHARD=1 ', docker_run[0]) cache = root/'nvme' for rank in ('0', '1'): run, docker_run = start(rank, '--prefix-cache-dir', str(cache)) @@ -310,6 +328,15 @@ def test_profiles_are_selected_by_name(self): argv, _ = serve.launch(self.environment, small) self.assertIn('--max-seq-len=131072', argv) self.assertIn('--max-num-seqs=8', argv) + # The model's full window, four requests sharing one pool; otherwise the 4x512k profile. + full = json.loads(serve.profile_path({'SPARKGLM_PROFILE': '4x1m'}, HERE).read_text()) + base = json.loads(serve.profile_path({}, HERE).read_text()) + argv, _ = serve.launch(self.environment, full) + self.assertIn('--max-seq-len=1048576', argv) + self.assertIn('--max-num-seqs=4', argv) + self.assertEqual(full['environment'], base['environment']) + self.assertEqual([a for a in full['server_argv'] if not a.startswith('--max-seq-len=')], + [a for a in base['server_argv'] if not a.startswith('--max-seq-len=')]) for bad in ('../x', '', 'a/b', '.hidden'): with self.subTest(name=bad), self.assertRaises(ValueError): serve.profile_path({'SPARKGLM_PROFILE': bad}, HERE) diff --git a/start.sh b/start.sh index 7f557a0..608fab2 100755 --- a/start.sh +++ b/start.sh @@ -13,7 +13,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" if [[ -f .env ]]; then set -a; . ./.env; set +a; fi : "${WORKER:?set WORKER in .env to the ssh destination of the other Spark (see .env.example)}" MODEL_ROOT=${MODEL_ROOT:-$HOME/models/sparkglm} -PROFILE=${PROFILE:-4x512k} +PROFILE=${PROFILE:-4x1m} FABRIC_HCA=${FABRIC_HCA:-rocep1s0f0} IMAGE=${IMAGE:-$(install/build.sh --tag)} CHECKPOINT=nvidia/GLM-5.3-Flash-NVFP4@423acf37583782c51c142d145aef733d72943d93 @@ -103,6 +103,19 @@ serve() { 1) common+=(--display-carveout) ;; *) echo "DISPLAY_CARVEOUT must be 0 or 1" >&2; exit 2 ;; esac + # The KV shard is the default; the disk prefix cache cannot run beside it yet, + # so setting PREFIX_CACHE_DIR keeps the shard off unless KV_SHARD=1 asks for both. + local shard=${KV_SHARD:-} + if [[ -z $shard ]]; then + shard=1 + [[ -z ${PREFIX_CACHE_DIR:-} ]] || { shard=0; say "PREFIX_CACHE_DIR is set: KV shard off (set KV_SHARD=0 to silence this)"; } + fi + case $shard in + 0) ;; + 1) [[ -z ${PREFIX_CACHE_DIR:-} ]] || { echo "KV_SHARD=1 and PREFIX_CACHE_DIR cannot be combined; unset one" >&2; exit 2; } + common+=(--kv-shard) ;; + *) echo "KV_SHARD must be 0 or 1" >&2; exit 2 ;; + esac case ${FABRIC_SIBLINGS:-1} in 1) ;; 0) common+=(--no-fabric-siblings) ;;