diff --git a/.env.example b/.env.example index dcb558c..1061910 100644 --- a/.env.example +++ b/.env.example @@ -37,14 +37,18 @@ WORKER=you@192.0.2.2 # The KV cache is split between the two Sparks: each stores the latents of half # the blocks, so the shared pool holds about 1.8x the tokens. With the display # carveout at 0.91: ~2.62M tokens (1.36M unsplit), lowest free memory 4.5 GB in -# a four-session 204K fill. On by default; the disk prefix cache cannot run -# beside it yet, so setting PREFIX_CACHE_DIR turns it off. 0 turns it off. +# a four-session 204K fill. On by default, with or without the disk prefix +# cache (each Spark spills only its own half). 0 turns it off. #KV_SHARD=0 # Build the image from source instead of pulling ghcr.io/enntity/atlas-sparkglm. #BUILD=1 -# Parallel compile jobs for a source build; lower it if the build runs out of memory. +# Parallel compile jobs for a source build (default: one per ~10 GiB of free +# memory, at most half the cores); lower it if the build runs out of memory. #ATLAS_BUILD_JOBS=4 +# Skip the in-image release checks (default 1: run them) for an engine commit +# that already passed them; the image itself is the same either way. +#ATLAS_IMAGE_CHECKS=0 # Only for private Hugging Face mirrors; the pinned checkpoints are public. #HF_TOKEN= diff --git a/README.md b/README.md index 50239eb..b0eabfe 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,7 @@ comparison videos. ## The engine Atlas here is [`Enntity/atlas`](https://github.com/Enntity/atlas) branch -`sparkglm/atlas-20261009` (pinned in +`sparkglm/atlas-20261009-final` (pinned in [`install/atlas-source.json`](install/atlas-source.json)), built from three layers: diff --git a/docs/INSTALL-MANUAL.md b/docs/INSTALL-MANUAL.md index 6fe4c5f..954bbee 100644 --- a/docs/INSTALL-MANUAL.md +++ b/docs/INSTALL-MANUAL.md @@ -5,7 +5,7 @@ time. Run steps 1–4 on **both** Sparks with the same `MODEL_ROOT`. | Component | Pin | |---|---| -| Engine | [`Enntity/atlas`](https://github.com/Enntity/atlas) `sparkglm/atlas-20261009` @ `e034500d` ([`install/atlas-source.json`](../install/atlas-source.json)) | +| Engine | [`Enntity/atlas`](https://github.com/Enntity/atlas) `sparkglm/atlas-20261009-final` @ `b85befc6` ([`install/atlas-source.json`](../install/atlas-source.json)) | | Model | [`nvidia/GLM-5.3-Flash-NVFP4`](https://huggingface.co/nvidia/GLM-5.3-Flash-NVFP4) @ `423acf37583782c51c142d145aef733d72943d93` (MIT) | | Drafter | [`incoai/GLM-5.3-Flash-DFlash2`](https://huggingface.co/incoai/GLM-5.3-Flash-DFlash2) @ `7d74cdd881ed7e32c31175984a67823127b66cfe` (CC BY-NC-ND 4.0) | | Native dependencies | FlashInfer `8eccd0c1`, CUTLASS `cf064d2e`, FlashKDA ([`install/flash_kda/`](../install/flash_kda/)), Rust 1.93.1, CUDA 13.0 | @@ -36,9 +36,11 @@ any other tree, and builds: A cold build downloads the CUDA base images and compiles everything; allow 30 to 60 minutes, of which the engine's kernels take about 18. With a warm -BuildKit cache it took about 5 minutes on our Sparks. `ATLAS_BUILD_JOBS` -(default 4) trades speed for memory. Keep models off the Spark while it -builds. +BuildKit cache it takes about 3 minutes on our Sparks. `ATLAS_BUILD_JOBS` +(default: one job per ~10 GiB of free memory, at most half the cores) trades +speed for memory, and `ATLAS_IMAGE_CHECKS=0` skips the in-image release checks +for an engine commit that already passed them. Keep models off the Spark while +it builds. ## 3. Download the pinned checkpoints diff --git a/install/Dockerfile b/install/Dockerfile index 810ce4a..d192c72 100644 --- a/install/Dockerfile +++ b/install/Dockerfile @@ -54,6 +54,12 @@ RUN --mount=type=cache,target=/build/target,id=atlas-cargo-target,sharing=shared # License notices of every third-party crate compiled into the binary. COPY install/rust-notices.py /product/install/rust-notices.py RUN python3 /product/install/rust-notices.py /build /build/dist/notices-rust + +# Release checks on the same Linux/CUDA build as the shipped binary. A stage of +# their own: the runtime image below copies only from `builder`, so BuildKit +# runs these only when build.sh asks for this target (ATLAS_IMAGE_CHECKS=1, the +# default). +FROM builder AS checks # Check the actual compiled GLM module set and the startup vision selector. # CUDA stubs permit these CPU-only tests without GPU build entitlements. RUN mkdir -p /opt/cuda-stubs && \ @@ -135,7 +141,7 @@ RUN --mount=type=cache,target=/build/target,id=atlas-cargo-target,sharing=shared grep -F "test result: ok. ${check##*:} passed;" /tmp/atlas-verify-check.log; \ done -FROM nvidia/cuda:13.0.0-runtime-ubuntu24.04@sha256:95318efecfd68ab3d109da5277863257b06137c84f34a87f38de970d5cd035d3 +FROM nvidia/cuda:13.0.0-runtime-ubuntu24.04@sha256:95318efecfd68ab3d109da5277863257b06137c84f34a87f38de970d5cd035d3 AS runtime RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends --allow-change-held-packages \ python3 ca-certificates ffmpeg libnccl2=2.31.2-1+cuda13.3 libibverbs1 librdmacm1 \ ibverbs-providers libnl-3-200 libnl-route-3-200 && rm -rf /var/lib/apt/lists/* diff --git a/install/atlas-source.json b/install/atlas-source.json index d9ff7b8..4cadc38 100644 --- a/install/atlas-source.json +++ b/install/atlas-source.json @@ -2,9 +2,9 @@ "schema": "sparkglm.atlas-engine-source/v2", "note": "The Atlas engine is built from the public Enntity/atlas fork at this exact commit; build.sh refuses any other tree.", "repository": "https://github.com/Enntity/atlas.git", - "branch": "sparkglm/atlas-20261009", - "commit": "e034500d858742d131529e56179453e8afb45dce", - "tree": "edc66e7c3ca234e6aac3026c2b8242119dc42183", + "branch": "sparkglm/atlas-20261009-final", + "commit": "b85befc684772cf2fda5829b1fe2598883448c26", + "tree": "29305f3eccdebadc215fc02d9cf569d3b4d2fe03", "layers": { "atlas_inf_main_base": "6a3d24ecb891adac1fac8edeef92de29bb6f3e15", "upstream_candidate": { @@ -17,9 +17,9 @@ "note": "The unchanged FlashKDA/native sparse-MLA prefill bridges and GB10 display-carveout KV placement from fdd2f965, then the carveout takes latent KV pools only (ATLAS_KV_CARVEOUT_ORDER, default latent): the GPU maps the carveout L2-uncached, and sparse-index buffers there slowed 64K cold prefill by 7% (Enntity/sparkglm#30). Fork-only; used only with DISPLAY_CARVEOUT=1. Plus: sharded latent pools (ATLAS_GLM_KV_SHARD, opt-in through SPARKGLM_KV_SHARD) size and place in the display carveout." }, "integration": { - "branch": "sparkglm/atlas-20261009", - "commit": "7838b34c + 9 cherry-picks", - "note": "sparkglm/atlas-20261007-strings plus the kv-shard carveout placement (exp/kvshard-20261008), copy drafts (exp/copy-drafts-20261008), the role-literal fix (fix/role-literal-strip-chatml) and the Poolside header hold (sparkqwen/rc15-fixes 25d14fcb), cherry-picked with -x." + "branch": "sparkglm/atlas-20261009-final", + "commit": "e034500d + 3 cherry-picks + 2", + "note": "sparkglm/atlas-20261009 plus the disk prefix-cache tier beside the KV shard (feat/kvshard-nvme: each rank spills and restores only the latents it owns) and the bit-exact shard (feat/kvshard-exact: few-row MLA attention takes one canonical decomposition with the shard on and off, so sharding changes no output bit), cherry-picked with -x; then fix/canonical-history: one split count for every row count, so a row's bits no longer depend on the DFlash verify width beside it." } }, "kernel_target": "glm-5.3-flash", diff --git a/install/build.sh b/install/build.sh index e16caf8..f8dda0c 100755 --- a/install/build.sh +++ b/install/build.sh @@ -31,9 +31,21 @@ GIT_LFS_SKIP_SMUDGE=1 git -C "$context/engine" -c advice.detachedHead=false chec git -C "$product" archive HEAD install | tar -x -C "$context" cp "$here/dockerignore" "$context/.dockerignore" -docker build --build-arg "INSTALL_TREE=$tree" \ - --build-arg "BUILD_JOBS=${ATLAS_BUILD_JOBS:-4}" \ - -f "$context/install/Dockerfile" -t "$image" "$context" >&2 +# Parallel compile jobs: ATLAS_BUILD_JOBS, else one per ~10 GiB of free memory +# (nvcc on CUTLASS units takes several GiB each; GB10 memory is shared with the +# host), at most half the cores and at least 2. +jobs=${ATLAS_BUILD_JOBS:-$(awk -v cores="$(nproc)" '/MemAvailable/ { + j = int($2 / 1048576 / 10); if (j > cores / 2) j = int(cores / 2); if (j < 2) j = 2; print j }' /proc/meminfo)} +args=(--build-arg "INSTALL_TREE=$tree" --build-arg "BUILD_JOBS=$jobs" -f "$context/install/Dockerfile") +echo "building $image with $jobs jobs" >&2 +# The release checks (the `checks` stage) run unless ATLAS_IMAGE_CHECKS=0, for +# builds whose engine commit already passed them elsewhere. +case ${ATLAS_IMAGE_CHECKS:-1} in + 1) docker build "${args[@]}" --target checks "$context" >&2 ;; + 0) ;; + *) echo 'ATLAS_IMAGE_CHECKS must be 0 or 1' >&2; exit 2 ;; +esac +docker build "${args[@]}" --target runtime -t "$image" "$context" >&2 [[ $(docker image inspect -f '{{.Architecture}} {{index .Config.Labels "io.enntity.sparkglm.install-tree"}}' "$image") == "arm64 $tree" ]] || { echo "built image $image has the wrong architecture or install-tree label" >&2; exit 1; } echo "$image" diff --git a/install/serve.py b/install/serve.py index 5c7297d..3c7558d 100644 --- a/install/serve.py +++ b/install/serve.py @@ -87,22 +87,17 @@ def disk_prefix_cache(gb): 'ATLAS_SSM_TIER_SLOTS': '2'} -def kv_shard(environ, disk_tier): +def kv_shard(environ): """Engine settings that split GLM's MLA latent KV between the two ranks, - when SPARKGLM_KV_SHARD=1 (default off): each rank stores half the blocks' - latents, so the shared pool holds about 1.8x the tokens for the same - memory (with the display carveout at 0.89: 120,186 blocks against 67,608; - single-request decode unchanged, four at once -3%). The disk prefix-cache - tier spills latents by physical block, which a shard cannot serve, so the - two exclude each other.""" + when SPARKGLM_KV_SHARD=1: each rank stores the latents of half the blocks, + so the shared pool holds about 1.8x the tokens for the same memory. The + disk prefix-cache tier runs beside it: each rank spills and restores only + the latents it owns.""" switch = environ.get('SPARKGLM_KV_SHARD', '0') if switch not in ('0', '1'): raise ValueError('SPARKGLM_KV_SHARD must be 0 or 1') if switch == '0': return {} - if disk_tier: - raise ValueError('SPARKGLM_KV_SHARD=1 and the disk prefix cache (SPARKGLM_PREFIX_CACHE_GB) ' - 'cannot be combined; switch one off') return {'ATLAS_GLM_KV_SHARD': '1', 'ATLAS_GLM_KV_SHARD_COMPACT': '1'} @@ -144,9 +139,8 @@ def launch(environ, profile): raise ValueError('SPARKGLM_GPU_MEMORY_UTILIZATION must be 0.80 to 0.95') args = [f'--gpu-memory-utilization={util}' if a.startswith('--gpu-memory-utilization=') else a for a in args] - disk_tier = disk_prefix_cache(environ.get('SPARKGLM_PREFIX_CACHE_GB', '')) - env.update(disk_tier) - env.update(kv_shard(environ, disk_tier)) + env.update(disk_prefix_cache(environ.get('SPARKGLM_PREFIX_CACHE_GB', ''))) + env.update(kv_shard(environ)) name = environ.get('SERVED_MODEL_NAME', 'glm-5.3-flash-atlas') if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._/-]{0,127}', name): raise ValueError('Invalid SERVED_MODEL_NAME') diff --git a/install/start-node.sh b/install/start-node.sh index 0131b36..d9a3553 100755 --- a/install/start-node.sh +++ b/install/start-node.sh @@ -37,8 +37,7 @@ usage: start-node.sh --rank 0|1 --leader-address IP --model-root DIR --image TAG CAP_SYS_ADMIN to export it, which the server drops before it loads anything; use it on both ranks (default: off) --kv-shard split the KV cache between the two Sparks, about 1.8x the - pool for the same memory; use it on both ranks, without - --prefix-cache-dir (default: off) + pool for the same memory; use it on both ranks (default: off) --cuda-cache persistent CUDA JIT cache (default: ~/.cache/atlas-cuda) EOF exit 2 diff --git a/install/test_install.py b/install/test_install.py index b8df9b8..8ae042d 100644 --- a/install/test_install.py +++ b/install/test_install.py @@ -224,7 +224,7 @@ def test_env_example_leaves_the_display_carveout_off(self): self.assertIn('#DISPLAY_CARVEOUT=1', lines) self.assertFalse([l for l in lines if l.startswith('DISPLAY_CARVEOUT')]) - def test_kv_shard_is_opt_in_and_excludes_the_disk_tier(self): + def test_kv_shard_is_opt_in_in_the_image_and_runs_beside_the_disk_tier(self): _, env = serve.launch(self.environment, self.profile) self.assertNotIn('ATLAS_GLM_KV_SHARD', env) _, env = serve.launch(dict(self.environment, SPARKGLM_KV_SHARD='1'), self.profile) @@ -232,8 +232,9 @@ def test_kv_shard_is_opt_in_and_excludes_the_disk_tier(self): for bad in ('yes', '2', ' 1', ''): with self.subTest(switch=bad), self.assertRaises(ValueError): serve.launch(dict(self.environment, SPARKGLM_KV_SHARD=bad), self.profile) - with self.assertRaises(ValueError): - serve.launch(dict(self.environment, SPARKGLM_KV_SHARD='1', SPARKGLM_PREFIX_CACHE_GB='48'), self.profile) + _, env = serve.launch(dict(self.environment, SPARKGLM_KV_SHARD='1', SPARKGLM_PREFIX_CACHE_GB='48'), + self.profile) + self.assertEqual((env['ATLAS_GLM_KV_SHARD'], env['ATLAS_KV_NVME_GB']), ('1', '24')) def test_disk_prefix_cache_is_off_unless_sized(self): # Ambient tier variables are dropped like any other ATLAS_* flag. diff --git a/start.sh b/start.sh index 608fab2..743fe71 100755 --- a/start.sh +++ b/start.sh @@ -103,17 +103,9 @@ serve() { 1) common+=(--display-carveout) ;; *) echo "DISPLAY_CARVEOUT must be 0 or 1" >&2; exit 2 ;; esac - # The KV shard is the default; the disk prefix cache cannot run beside it yet, - # so setting PREFIX_CACHE_DIR keeps the shard off unless KV_SHARD=1 asks for both. - local shard=${KV_SHARD:-} - if [[ -z $shard ]]; then - shard=1 - [[ -z ${PREFIX_CACHE_DIR:-} ]] || { shard=0; say "PREFIX_CACHE_DIR is set: KV shard off (set KV_SHARD=0 to silence this)"; } - fi - case $shard in + case ${KV_SHARD:-1} in 0) ;; - 1) [[ -z ${PREFIX_CACHE_DIR:-} ]] || { echo "KV_SHARD=1 and PREFIX_CACHE_DIR cannot be combined; unset one" >&2; exit 2; } - common+=(--kv-shard) ;; + 1) common+=(--kv-shard) ;; *) echo "KV_SHARD must be 0 or 1" >&2; exit 2 ;; esac case ${FABRIC_SIBLINGS:-1} in