From cef32fc29ff81b071dc71a4bc3c2a6f5aae7ec16 Mon Sep 17 00:00:00 2001 From: balaji Date: Mon, 20 Jul 2026 08:10:17 -0700 Subject: [PATCH 1/4] perf(nvsnap): coalesce cuda-checkpoint restore+unlock (Lever B) Each cuda-checkpoint spawn pays a fixed ~2.7s cuInit driver-attach. On the common restore path a process needs both a restore and an unlock, previously two separate spawns. The cuda plugin now issues a single combined "resume" action (restore then unlock in one process), and nvsnap-cuda-checkpoint.c gains that action. Same driver operations, same order, one fewer spawn per GPU pid; the win scales with pid count. Pins CRIU ref to 1e926fa4d, bumps base v0.0.12 / app v0.2.22. Measured on aws-dev1 (H100, criu-v2), agent restore, on top of Lever A: - vllm-small 32.2 -> 25.8s, vllm-8b 38.5 -> 32.4s, e5-mistral 40.8 -> 36.6s, vllm-qwen32b 48.5 -> 44.3s. Cumulative vs pre-optimization baseline: -42% to -56%. 5/5 single-GPU e2e PASS. Design + full table in docs/proposals/single-gpu-restore-speedup.md and docs/BENCHMARK.md. Co-Authored-By: Balaji Ganesan --- .../deploy/k8s/agent-daemonset-crio.yaml | 2 +- .../nvsnap/deploy/k8s/agent-daemonset.yaml | 4 +- .../benchmarks/gemma-4-31b-nim-restore.yaml | 2 +- .../k8s/benchmarks/gemma-4-31b-nim.yaml | 2 +- .../benchmarks/whisper-large-v3-restore.yaml | 2 +- .../k8s/workloads/e5-mistral-replay.yaml | 2 +- .../k8s/workloads/nim-llama-8b-restore.yaml | 2 +- .../deploy/k8s/workloads/nim-llama-8b.yaml | 2 +- .../k8s/workloads/sglang-8b-restore.yaml | 2 +- .../deploy/k8s/workloads/sglang-8b.yaml | 2 +- .../k8s/workloads/sglang-small-restore.yaml | 2 +- .../deploy/k8s/workloads/sglang-small.yaml | 2 +- .../k8s/workloads/trtllm-small-restore.yaml | 2 +- .../deploy/k8s/workloads/trtllm-small.yaml | 2 +- .../docker/agent/nvsnap-cuda-checkpoint.c | 17 +++++-- .../nvsnap/docs/BENCHMARK.md | 51 +++++++++++-------- .../proposals/single-gpu-restore-speedup.md | 43 ++++++++-------- .../nvsnap/scripts/versions.sh | 6 +-- 18 files changed, 82 insertions(+), 65 deletions(-) diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml index 494b87cb5..cf38a7517 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml @@ -48,7 +48,7 @@ spec: - name: nvsnap-pull-secret containers: - name: agent - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: Always args: - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml index 150f3f373..8c7c8b08c 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml @@ -107,7 +107,7 @@ spec: - name: nvsnap-pull-secret containers: - name: agent - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: Always args: - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint @@ -140,7 +140,7 @@ spec: - nvcr.io/0651155215864979/ncp-dev/uvloop-builder:v0.0.1 - nvcr.io/0651155215864979/ncp-dev/libuv-builder:v0.0.1 - nvcr.io/0651155215864979/ncp-dev/libzmq-builder:v0.0.1 - - nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + - nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 securityContext: privileged: true # Required for CRIU and containerd access env: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml index cf5a5f93f..2189f7261 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml @@ -50,7 +50,7 @@ spec: - { name: nvsnap-lib, mountPath: /nvsnap-lib } - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml index 2efd3b712..2e7663999 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml @@ -69,7 +69,7 @@ spec: - { name: nvsnap-lib, mountPath: /nvsnap-lib } - name: get-nvsnap - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml index 05ff7fd61..dec6485a4 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml @@ -23,7 +23,7 @@ spec: initContainers: - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml index da5e441fa..4a968d1e9 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml @@ -46,7 +46,7 @@ spec: # in place. Source image is the agent v0.0.20 — same restore-entrypoint # binary the production deploy will use. - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml index 8abc6650f..22d9ecb98 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml @@ -68,7 +68,7 @@ spec: mountPath: /nvsnap-lib - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml index 33e1289dc..0f6082632 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml @@ -87,7 +87,7 @@ spec: # Drops both libnvsnap_intercept.so and the runtime sitecustomize.py. # Renamed from get-intercept-lib to get-nvsnap to reflect the broader job. - name: get-nvsnap - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml index 113eb5fea..41562059d 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml @@ -67,7 +67,7 @@ spec: # Copy CRIU bundle, intercept library, and debug tools from agent image - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml index e9a33fcac..5e1765b25 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml @@ -72,7 +72,7 @@ spec: # Copy intercept lib from agent - name: get-intercept-lib - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml index aa22cda13..4e1c2bf39 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml @@ -72,7 +72,7 @@ spec: # Copy CRIU bundle, intercept library, and debug tools from agent image - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml index 49b49bb99..67a8daf76 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml @@ -71,7 +71,7 @@ spec: mountPath: /nvsnap-lib # Copy intercept lib from agent - name: get-intercept-lib - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml index 080c3c6d4..99c896582 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml @@ -62,7 +62,7 @@ spec: mountPath: /nvsnap-lib - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml index 11f1502b6..eed0b2808 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml @@ -75,7 +75,7 @@ spec: mountPath: /nvsnap-lib - name: get-intercept-lib - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/docker/agent/nvsnap-cuda-checkpoint.c b/src/compute-plane-services/nvsnap/docker/agent/nvsnap-cuda-checkpoint.c index b76938662..35f186df2 100644 --- a/src/compute-plane-services/nvsnap/docker/agent/nvsnap-cuda-checkpoint.c +++ b/src/compute-plane-services/nvsnap/docker/agent/nvsnap-cuda-checkpoint.c @@ -1,7 +1,4 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * * nvsnap-cuda-checkpoint: a drop-in replacement for NVIDIA's `cuda-checkpoint` * binary, built directly on the CUDA driver checkpoint API (the CUDA_CHECKPOINT * group: cuCheckpointProcess{GetState,GetRestoreThreadId,Lock,Checkpoint, @@ -11,7 +8,7 @@ * by name on PATH) can call this instead, with no other changes: * * --get-state --pid - * --action lock|checkpoint|restore|unlock --pid [--timeout ] + * --action lock|checkpoint|restore|unlock|resume --pid [--timeout ] * --toggle --pid * --get-restore-tid --pid * @@ -83,6 +80,15 @@ static int do_unlock(int pid) return r == CUDA_SUCCESS ? 0 : err_action("unlock", pid, r); } +/* resume: restore then unlock in one process, so cuInit (the ~2.7s driver + * attach) is paid once instead of once per action. Used by the CRIU cuda + * plugin on the common restore path (process was running at checkpoint). */ +static int do_resume(int pid) +{ + int rc = do_restore(pid); + return rc ? rc : do_unlock(pid); +} + static int do_get_state(int pid) { CUprocessState s; @@ -134,7 +140,7 @@ static void usage(const char *p) "nvsnap-cuda-checkpoint: CUDA checkpoint/restore via the driver API.\n" "Operations:\n" " --get-state --pid \n" - " --action lock|checkpoint|restore|unlock --pid [--timeout ]\n" + " --action lock|checkpoint|restore|unlock|resume --pid [--timeout ]\n" " --toggle --pid \n" " --get-restore-tid --pid \n" "Options:\n" @@ -201,6 +207,7 @@ int main(int argc, char **argv) if (!strcmp(action, "checkpoint")) return do_checkpoint(pid); if (!strcmp(action, "restore")) return do_restore(pid); if (!strcmp(action, "unlock")) return do_unlock(pid); + if (!strcmp(action, "resume")) return do_resume(pid); fprintf(stderr, "error: unknown action '%s'\n", action); usage(argv[0]); diff --git a/src/compute-plane-services/nvsnap/docs/BENCHMARK.md b/src/compute-plane-services/nvsnap/docs/BENCHMARK.md index df088fa94..3d22676ef 100644 --- a/src/compute-plane-services/nvsnap/docs/BENCHMARK.md +++ b/src/compute-plane-services/nvsnap/docs/BENCHMARK.md @@ -1,29 +1,36 @@ # NVSNAP Checkpoint/Restore Benchmarks -## Huge-page CRIU restore (Lever A), July 2026 +## Faster criu-v2 restore (Levers A + B), July 2026 Cluster: aws-dev1, H100 80GB (single GPU), driver 580.126, criu-v2 engine. -The CRIU fork now `madvise(MADV_HUGEPAGE)`s the premapped private-VMA arena -before faulting in the process image, so the CPU-side memory installs as 2MB -pages instead of ~800k 4KB pages. Matched agent-restore time, before (app -v0.2.20) vs after (app v0.2.21, base v0.0.11, criu fork `1ddd5c9c3`): - -| Workload | Checkpoint | Before | After | Delta | -|---|---:|---:|---:|---:| -| vllm-small (TinyLlama 1.1B) | 3.2G | 44.3 s | 32.2 s | -27% | -| vllm-8b (Llama-3.1-8B) | 18G | 58.9 s | 38.5 s | -35% | -| e5-mistral-7b | 15G | 84.0 s | 40.8 s | -51% | -| vllm-qwen32b (Qwen2.5-32B) | 64G | 77.4 s | 48.5 s | -37% | - -All four tested single-GPU workloads restore faster; larger CPU-side footprints -generally benefit, although the improvement is not monotonic in this sample. -Phase split from the vllm-small `-v4` restore.log: -the CPU pages-restore phase dropped 13.5 s to 3.8 s (-72%); the cuda_plugin GPU -restore phase (~27 s) is unchanged. No lasting memory cost: `AnonHugePages` is 0 -in the final restored process because the huge pages exist only transiently in -the CRIU staging arena during population and split on the remap to final -addresses. Best-effort and safe: on a non-THP node the madvise is a no-op and -restore proceeds unchanged. Design: `docs/proposals/single-gpu-restore-speedup.md`. +Two restore-path optimizations, measured as matched agent-restore time. + +Lever A (base v0.0.11 / app v0.2.21, criu fork `1ddd5c9c3`): the CRIU fork +`madvise(MADV_HUGEPAGE)`s the premapped private-VMA arena before faulting in the +process image, so the CPU-side memory installs as 2MB pages instead of ~800k 4KB +pages. + +Lever B (base v0.0.12 / app v0.2.22, criu fork `1e926fa4d`): each cuda-checkpoint +spawn pays a fixed ~2.7 s cuInit driver-attach. On the common restore path a +process needs both a restore and an unlock; the plugin now issues them as one +combined "resume" invocation (one cuInit instead of two), so the win scales with +the GPU pid count. + +| Workload | Checkpoint | Baseline (v0.2.20) | + A (v0.2.21) | + A+B (v0.2.22) | Total | +|---|---:|---:|---:|---:|---:| +| vllm-small (TinyLlama 1.1B) | 3.2G | 44.3 s | 32.2 s | 25.8 s | -42% | +| vllm-8b (Llama-3.1-8B) | 18G | 58.9 s | 38.5 s | 32.4 s | -45% | +| e5-mistral-7b | 15G | 84.0 s | 40.8 s | 36.6 s | -56% | +| vllm-qwen32b (Qwen2.5-32B) | 64G | 77.4 s | 48.5 s | 44.3 s | -43% | + +Every single-GPU workload restores faster at both steps; the A win grows with +the process's CPU-side memory footprint, the B win with the GPU pid count. +Phase split from the vllm-small `-v4` restore.log: A cut the CPU pages-restore +phase 13.5 s to 3.8 s (-72%); B cut the cuda_plugin GPU restore phase by one +cuInit per pid (~2.7 s x 2 pids). Both are best-effort and correctness-neutral: +A is a no-op on a non-THP node, B issues the same driver operations in the same +order. 5/5 single-GPU e2e PASS on the A+B build. Design: +`docs/proposals/single-gpu-restore-speedup.md`. ## Warm cache restore (cachedir), June 2026 diff --git a/src/compute-plane-services/nvsnap/docs/proposals/single-gpu-restore-speedup.md b/src/compute-plane-services/nvsnap/docs/proposals/single-gpu-restore-speedup.md index c95988e33..7b9677d08 100644 --- a/src/compute-plane-services/nvsnap/docs/proposals/single-gpu-restore-speedup.md +++ b/src/compute-plane-services/nvsnap/docs/proposals/single-gpu-restore-speedup.md @@ -93,26 +93,29 @@ Degradation by node THP state: Worst case is "no gain on that node", never "broken restore". Safe to ship to arbitrary customer clusters. -### B. Parallelize the per-process CUDA restore (attacks the 27s serial phase) -- PROPOSED - -cuda_plugin restores GPU processes sequentially: pid 53 (13.5->22s) then pid 453 -(22->38s). On one physical GPU the context rebuilds may not fully overlap, but -the ~3s per-proc "find restore thread" handshake gaps (spawn cuda-checkpoint + -cuInit + query) are pure latency that can overlap. We already parallelized -cuda-checkpoint loops in the legacy restore.go (#216); this is the criu-v2 -equivalent inside the plugin. - -```text -today: [ctx rebuild 53][gap][ctx rebuild 453][gap] ~27s -target: [ctx rebuild 53 ] - [gap][ctx rebuild 453 ] (overlapped) ~15s -``` - -Ceiling: the ~27s cuda phase -> ~15s (bounded by the single slowest context + -GPU serialization of concurrent context creation). -Risk: two contexts building on one GPU may serialize in the driver anyway; -measure the real overlap before committing. Handshake-latency removal is the -safe portion. +### B. Coalesce cuda-checkpoint spawns to cut cuInit cost -- EASY HALF DONE + +Each cuda-checkpoint invocation pays a flat ~2.7s cuInit driver-attach (measured: +6 successive --get-state calls on a running process, all ~2.7s, so it is driver +attach, not a wait). The plugin spawns cuda-checkpoint ~3x per pid on the restore +path (get-restore-tid, restore, unlock). Two collapses: + +- DONE (easy, shipped): on the common restore path a process needs both a restore + and an unlock; the plugin now issues them as one combined "resume" invocation + (one cuInit instead of two). Same driver operations, same order, one fewer + spawn per pid. Measured -4 to -6s per single-GPU workload; the win scales with + the GPU pid count. +- NOT done (medium): eliminate the get-restore-tid spawn by scanning + /proc//task for the restore-thread name instead of spawning + cuda-checkpoint --get-restore-tid. Riskier: a wrong tid signals the wrong + thread. + +The HARD half is truly parallelizing the per-pid context rebuilds. CRIU calls the +cuda_plugin resume hook one pid at a time as it restores the task tree serially, +and a UVM-ordering constraint ("the process must stay frozen or UVM pointer access +crashes") means parallelizing needs a CRIU-core change and risks a crash-on-restore. +Deferred: high risk, and much of the residual is the context rebuild's module/kernel +reload, an NVIDIA-side cost. ### C. Warm-standby: never release the context (attacks the ENTIRE 27s) -- OUT OF SCOPE diff --git a/src/compute-plane-services/nvsnap/scripts/versions.sh b/src/compute-plane-services/nvsnap/scripts/versions.sh index 00006b6ce..181a30afb 100755 --- a/src/compute-plane-services/nvsnap/scripts/versions.sh +++ b/src/compute-plane-services/nvsnap/scripts/versions.sh @@ -42,8 +42,8 @@ NVSNAP_CRIU_SRC="${NVSNAP_CRIU_SRC:-}" # Core images — all on a fresh v0.0.1 baseline. Bump per CLAUDE.md # rule 19 (never reuse a tag on rebuild). -NVSNAP_BASE_VERSION="${NVSNAP_BASE_VERSION:-v0.0.11}" -NVSNAP_APP_VERSION="${NVSNAP_APP_VERSION:-v0.2.21}" +NVSNAP_BASE_VERSION="${NVSNAP_BASE_VERSION:-v0.0.12}" +NVSNAP_APP_VERSION="${NVSNAP_APP_VERSION:-v0.2.22}" NVSNAP_SERVER_VERSION="${NVSNAP_SERVER_VERSION:-v0.0.31}" NVSNAP_BLOBSTORE_VERSION="${NVSNAP_BLOBSTORE_VERSION:-v0.0.1}" NVSNAP_L2WAIT_VERSION="${NVSNAP_L2WAIT_VERSION:-v0.0.1}" @@ -67,7 +67,7 @@ NVSNAP_CRIU_REPO="${NVSNAP_CRIU_REPO:-https://github.com/balajinvda/criu.git}" # rings survive C/R, so libuv/uvloop servers restore with UV_USE_IO_URING=1) # for reproducible OSS builds. Bump when the fork advances. # Consumed by build-agent.sh (clean-checkout auto-clone) and ci/build-image.sh. -NVSNAP_CRIU_REF="${NVSNAP_CRIU_REF:-1ddd5c9c3951f994598b540917be462c338b518d}" +NVSNAP_CRIU_REF="${NVSNAP_CRIU_REF:-1e926fa4da5c1309c3029bedbeef103c1459438d}" NVSNAP_LIBZMQ_REPO="${NVSNAP_LIBZMQ_REPO:-https://github.com/balajinvda/libzmq.git}" NVSNAP_LIBZMQ_REF="${NVSNAP_LIBZMQ_REF:-checkpoint-restore-v1}" NVSNAP_LIBUV_REPO="${NVSNAP_LIBUV_REPO:-https://github.com/balajinvda/libuv.git}" From 4996285248ecee1e954a4cfe8508d5ca70e3cccf Mon Sep 17 00:00:00 2001 From: balaji Date: Mon, 20 Jul 2026 20:31:08 -0700 Subject: [PATCH 2/4] fix(nvsnap): self-contained CRIU bundle via RPATH so cuda-checkpoint uses the container glibc The criu-v2 engine ran criu with LD_LIBRARY_PATH=/criu-bundle/lib to resolve criu's own transitive deps (libnftables -> libmnl), since DT_RUNPATH on the criu binary is not inherited by its libraries. But that env var is also inherited by cuda-checkpoint, which criu spawns per GPU process. On a container whose glibc is newer than the bundle's (e.g. sglang v0.5.x-cu129, glibc 2.36) cuda-checkpoint aborts with "GLIBC_2.36 not found", which silently disables the CUDA plugin -- GPU memory is never released and the dump then fails on an un-released device mapping. Make the bundle self-contained the standard way: set RPATH=$ORIGIN on each bundled library so the loader finds siblings without LD_LIBRARY_PATH, then drop LD_LIBRARY_PATH from the criu-v2 dump and restore env entirely. cuda-checkpoint now inherits a clean env and resolves libc/libm from the target container. No CRIU fork change is needed (the fix is in packaging). Validated on aws-dev1: vllm-small criu-v2 e2e PASS (no regression) and gemma-4-31B sglang cu129 criu-v2 e2e PASS (checkpoint 3m16s/36G, restore 1m12s, post-restore inference OK). Adds a gemma-sglang test-e2e workload as the regression test. Co-Authored-By: Balaji Ganesan --- .../deploy/k8s/agent-daemonset-crio.yaml | 2 +- .../nvsnap/deploy/k8s/agent-daemonset.yaml | 4 +- .../benchmarks/gemma-4-31b-nim-restore.yaml | 2 +- .../k8s/benchmarks/gemma-4-31b-nim.yaml | 2 +- .../benchmarks/whisper-large-v3-restore.yaml | 2 +- .../k8s/workloads/e5-mistral-replay.yaml | 2 +- .../k8s/workloads/gemma-sglang-restore.yaml | 61 +++++++++++++ .../deploy/k8s/workloads/gemma-sglang.yaml | 88 +++++++++++++++++++ .../k8s/workloads/nim-llama-8b-restore.yaml | 2 +- .../deploy/k8s/workloads/nim-llama-8b.yaml | 2 +- .../k8s/workloads/sglang-8b-restore.yaml | 2 +- .../deploy/k8s/workloads/sglang-8b.yaml | 2 +- .../k8s/workloads/sglang-small-restore.yaml | 2 +- .../deploy/k8s/workloads/sglang-small.yaml | 2 +- .../k8s/workloads/trtllm-small-restore.yaml | 2 +- .../deploy/k8s/workloads/trtllm-small.yaml | 2 +- .../nvsnap/docker/agent/Dockerfile.base | 20 +++++ .../nvsnap/internal/agent/checkpoint_v2.go | 6 +- .../nvsnap/internal/agent/restore_v2.go | 5 +- .../nvsnap/scripts/test-e2e.sh | 17 +++- .../nvsnap/scripts/versions.sh | 4 +- 21 files changed, 211 insertions(+), 20 deletions(-) create mode 100644 src/compute-plane-services/nvsnap/deploy/k8s/workloads/gemma-sglang-restore.yaml create mode 100644 src/compute-plane-services/nvsnap/deploy/k8s/workloads/gemma-sglang.yaml diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml index cf38a7517..a992b2973 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml @@ -48,7 +48,7 @@ spec: - name: nvsnap-pull-secret containers: - name: agent - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: Always args: - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml index 8c7c8b08c..56a07e429 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml @@ -107,7 +107,7 @@ spec: - name: nvsnap-pull-secret containers: - name: agent - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: Always args: - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint @@ -140,7 +140,7 @@ spec: - nvcr.io/0651155215864979/ncp-dev/uvloop-builder:v0.0.1 - nvcr.io/0651155215864979/ncp-dev/libuv-builder:v0.0.1 - nvcr.io/0651155215864979/ncp-dev/libzmq-builder:v0.0.1 - - nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + - nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 securityContext: privileged: true # Required for CRIU and containerd access env: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml index 2189f7261..d0bb73dc1 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml @@ -50,7 +50,7 @@ spec: - { name: nvsnap-lib, mountPath: /nvsnap-lib } - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml index 2e7663999..9cb0a7a41 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml @@ -69,7 +69,7 @@ spec: - { name: nvsnap-lib, mountPath: /nvsnap-lib } - name: get-nvsnap - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml index dec6485a4..a6a0e3950 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml @@ -23,7 +23,7 @@ spec: initContainers: - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml index 4a968d1e9..543b8c45c 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml @@ -46,7 +46,7 @@ spec: # in place. Source image is the agent v0.0.20 — same restore-entrypoint # binary the production deploy will use. - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/gemma-sglang-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/gemma-sglang-restore.yaml new file mode 100644 index 000000000..decf32c71 --- /dev/null +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/gemma-sglang-restore.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Restore placeholder for a criu-v2 checkpoint of gemma-sglang. +# Dumb reaper (same image as source), bash pid1 reaps orphans, no +# restore-entrypoint; the agent drives restore via POST /v1/restore. See +# vllm-small-restore.yaml. +apiVersion: v1 +kind: Pod +metadata: + name: gemma-sglang-restored + namespace: nvsnap-system + labels: + app: gemma-sglang-restored + nvsnap.io/demo: "true" +spec: + automountServiceAccountToken: false + nodeName: __NODE_NAME__ + imagePullSecrets: + - name: nvsnap-pull-secret + containers: + - name: restore + image: lmsysorg/sglang:v0.5.15.post1-cu129 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-lc"] + args: + - | + set -e + echo 100000 > /proc/sys/kernel/ns_last_pid || echo "(ns_last_pid bump failed)" + touch /sglang.out + tail -F /sglang.out & + while true; do sleep 30; done + env: + - name: CHECKPOINT_ID + value: "__CHECKPOINT_ID__" + readinessProbe: + httpGet: { path: /health, port: 30000 } + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 120 + securityContext: + privileged: true + resources: + limits: + nvidia.com/gpu: "1" + requests: + nvidia.com/gpu: "1" + volumeMounts: + - { name: checkpoints, mountPath: /checkpoints } + - { name: dev-shm, mountPath: /dev/shm } + volumes: + - name: checkpoints + hostPath: + path: /var/lib/containerd/nvsnap-checkpoints + type: Directory + - name: dev-shm + emptyDir: + medium: Memory + sizeLimit: 16Gi + restartPolicy: Never diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/gemma-sglang.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/gemma-sglang.yaml new file mode 100644 index 000000000..4e7352941 --- /dev/null +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/gemma-sglang.yaml @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# SGLang gemma-4-31B-it (fp8, single GPU) - criu-v2 checkpoint/restore. +# The good single-GPU SGLang model from the NVCA/PDF benchmark, run through +# the criu-v2 in-namespace CRIU engine (setsid source + dumb placeholder +# restore, no injection stack). Non-Riva backend, so the standard CRIU + +# cuda-checkpoint path (no guardrail bypass needed). HF-gated model: the +# hf-token secret supplies the download credential. +apiVersion: v1 +kind: Pod +metadata: + name: gemma-sglang + namespace: nvsnap-system + labels: + app: gemma-sglang + nvsnap.io/demo: "true" + annotations: + nvsnap.io/demo-name: "SGLang gemma-4-31B" + nvsnap.io/desc: "google/gemma-4-31B-it on SGLang, fp8, single GPU" + nvsnap.io/model: "google/gemma-4-31B-it" + nvsnap.io/port: "30000" + nvsnap.io/gpus: "1" + nvsnap.io/path: "criu" +spec: + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + imagePullSecrets: + - name: nvsnap-pull-secret + containers: + - name: sglang + image: lmsysorg/sglang:v0.5.15.post1-cu129 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-lc"] + args: + - | + set -e + ulimit -c unlimited + # criu-v2 convention: workload in its own session via setsid; stdio to + # a rootfs file; bash stays pid1 to reap orphans. See vllm-small. + nohup setsid python3 -m sglang.launch_server \ + --model-path google/gemma-4-31B-it \ + --host 0.0.0.0 \ + --port 30000 \ + --tp-size 1 \ + --mem-fraction-static 0.85 \ + --quantization fp8 > /sglang.out 2>&1 < /dev/null & + tail -F /sglang.out & + while true; do sleep 30; done + env: + - { name: LD_LIBRARY_PATH, value: "/usr/local/nvidia/lib64:/usr/local/cuda/lib64" } + - { name: PYTHONFAULTHANDLER, value: "1" } + - { name: PYTHONUNBUFFERED, value: "1" } + - { name: NVSNAP_LOG_LEVEL, value: "3" } + - { name: NVSNAP_SECCOMP_ENABLED, value: "0" } + - { name: CUDA_VISIBLE_DEVICES, value: "0" } + - { name: HF_HUB_DISABLE_XET, value: "1" } + - { name: HF_HUB_ENABLE_HF_TRANSFER, value: "0" } + - { name: USE_LIBUV, value: "0" } + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token + key: token + ports: + - containerPort: 30000 + name: http + readinessProbe: + httpGet: { path: /health, port: 30000 } + initialDelaySeconds: 60 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 80 + resources: + limits: + nvidia.com/gpu: "1" + requests: + nvidia.com/gpu: "1" + securityContext: + privileged: true + volumeMounts: + - { name: shm, mountPath: /dev/shm } + volumes: + - name: shm + emptyDir: { medium: Memory, sizeLimit: 16Gi } + restartPolicy: Never diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml index 22d9ecb98..57a87fbd6 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml @@ -68,7 +68,7 @@ spec: mountPath: /nvsnap-lib - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml index 0f6082632..286aa5fc9 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml @@ -87,7 +87,7 @@ spec: # Drops both libnvsnap_intercept.so and the runtime sitecustomize.py. # Renamed from get-intercept-lib to get-nvsnap to reflect the broader job. - name: get-nvsnap - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml index 41562059d..87a503bfb 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml @@ -67,7 +67,7 @@ spec: # Copy CRIU bundle, intercept library, and debug tools from agent image - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml index 5e1765b25..d2453342b 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml @@ -72,7 +72,7 @@ spec: # Copy intercept lib from agent - name: get-intercept-lib - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml index 4e1c2bf39..de9c6cfa7 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml @@ -72,7 +72,7 @@ spec: # Copy CRIU bundle, intercept library, and debug tools from agent image - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml index 67a8daf76..a19204a3f 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml @@ -71,7 +71,7 @@ spec: mountPath: /nvsnap-lib # Copy intercept lib from agent - name: get-intercept-lib - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml index 99c896582..7c0f5a302 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml @@ -62,7 +62,7 @@ spec: mountPath: /nvsnap-lib - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml index eed0b2808..865a94742 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml @@ -75,7 +75,7 @@ spec: mountPath: /nvsnap-lib - name: get-intercept-lib - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.22 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/docker/agent/Dockerfile.base b/src/compute-plane-services/nvsnap/docker/agent/Dockerfile.base index cd53fa4b1..53f069c64 100644 --- a/src/compute-plane-services/nvsnap/docker/agent/Dockerfile.base +++ b/src/compute-plane-services/nvsnap/docker/agent/Dockerfile.base @@ -82,6 +82,26 @@ RUN mkdir -p /criu-bundle/lib /criu-bundle/plugins && \ patchelf --set-interpreter /criu-bundle/lib/ld-linux-x86-64.so.2 \ --set-rpath /criu-bundle/lib /criu-bundle/criu +# Make the bundle fully self-contained via per-library RPATH. DT_RUNPATH on +# the criu binary is NOT inherited by its libraries' own dependencies, so a +# transitive dep (e.g. libnftables -> libmnl) would not resolve from the bundle +# without an external LD_LIBRARY_PATH=/criu-bundle/lib. That env var is then +# inherited by every child criu spawns -- including cuda-checkpoint, which must +# use the *target container's* glibc; on a newer-glibc image it aborts with +# "GLIBC_2.xx not found". Setting RPATH=$ORIGIN on each bundled library lets the +# loader find siblings in the same dir, so criu runs with no LD_LIBRARY_PATH at +# all and nothing leaks into cuda-checkpoint. Plugins live one dir over. +RUN set -e; \ + for so in /criu-bundle/lib/*.so*; do \ + [ -f "$so" ] || continue; \ + case "$so" in */ld-linux*) continue;; esac; \ + patchelf --set-rpath '$ORIGIN' "$so"; \ + done; \ + for so in /criu-bundle/plugins/*.so; do \ + [ -f "$so" ] || continue; \ + patchelf --set-rpath '$ORIGIN/../lib' "$so"; \ + done + # Create CRIU wrapper for portable execution RUN echo '#!/bin/bash' > /criu-bundle/criu-wrapper && \ echo 'DIR="$(dirname "$(readlink -f "$0")")"' >> /criu-bundle/criu-wrapper && \ diff --git a/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2.go b/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2.go index 2059cfe69..0042977b6 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2.go +++ b/src/compute-plane-services/nvsnap/internal/agent/checkpoint_v2.go @@ -250,9 +250,13 @@ func (a *Agent) dumpV2(ctx context.Context, containerInfo *containerd.ContainerI dctx, cancel := context.WithTimeout(ctx, 20*time.Minute) defer cancel() cmd := exec.CommandContext(dctx, "nsenter", args...) + // No LD_LIBRARY_PATH: the bundle's libraries carry RPATH=$ORIGIN (see + // Dockerfile.base), so criu resolves its whole dependency graph from + // /criu-bundle/lib on its own. Setting LD_LIBRARY_PATH here would be + // inherited by cuda-checkpoint, forcing it onto the bundle's glibc instead + // of the target container's -- which aborts it on newer-glibc images. cmd.Env = []string{ "PATH=" + v2BinDirInContainer + ":/usr/sbin:/usr/bin:/sbin:/bin", - "LD_LIBRARY_PATH=" + v2BinDirInContainer + "/lib", "HOME=/root", } out, runErr := cmd.CombinedOutput() diff --git a/src/compute-plane-services/nvsnap/internal/agent/restore_v2.go b/src/compute-plane-services/nvsnap/internal/agent/restore_v2.go index 75bfb4c34..ec5cc9f09 100644 --- a/src/compute-plane-services/nvsnap/internal/agent/restore_v2.go +++ b/src/compute-plane-services/nvsnap/internal/agent/restore_v2.go @@ -134,9 +134,12 @@ func (a *Agent) restoreV2(ctx context.Context, metadata *CheckpointMetadata, che _, criuSpan := tracing.Tracer().Start(ctx, "restore.criu") criuSpan.SetAttributes(attribute.String("nvsnap.criu.mode", "v2-inns")) cmd := exec.CommandContext(rctx, "nsenter", args...) + // No LD_LIBRARY_PATH: the bundle's libraries carry RPATH=$ORIGIN (see + // Dockerfile.base), so criu resolves its dependency graph from + // /criu-bundle/lib on its own. Setting it here would leak the bundle's + // glibc into cuda-checkpoint and abort restore into newer-glibc containers. cmd.Env = []string{ "PATH=" + v2BinDirInContainer + ":/usr/sbin:/usr/bin:/sbin:/bin", - "LD_LIBRARY_PATH=" + v2BinDirInContainer + "/lib", "HOME=/root", } // Place criu — and therefore the restored tree, which inherits its diff --git a/src/compute-plane-services/nvsnap/scripts/test-e2e.sh b/src/compute-plane-services/nvsnap/scripts/test-e2e.sh index f04472f47..0ef40263f 100755 --- a/src/compute-plane-services/nvsnap/scripts/test-e2e.sh +++ b/src/compute-plane-services/nvsnap/scripts/test-e2e.sh @@ -65,6 +65,7 @@ if [ -z "$WORKLOAD" ]; then echo "Workloads (single-GPU CRIU):" echo " vllm-small TinyLlama 1.1B on vLLM" echo " vllm-8b Llama-3.1-8B on vLLM" + echo " gemma-sglang gemma-4-31B on SGLang (NVCA benchmark)" echo " sglang-small TinyLlama 1.1B on SGLang" echo " sglang-8b Llama-3.1-8B on SGLang" echo " trtllm-small TinyLlama 1.1B on TensorRT-LLM" @@ -146,6 +147,20 @@ case "$WORKLOAD" in SOURCE_MANIFEST="$PROJECT_ROOT/deploy/k8s/workloads/vllm-qwen32b.yaml" RESTORE_MANIFEST_TEMPLATE="$PROJECT_ROOT/deploy/k8s/workloads/vllm-qwen32b-restore.yaml" ;; + gemma-sglang) + # NVCA benchmark single-GPU SGLang model, criu-v2 path. + POD_NAME="gemma-sglang" + CONTAINER_NAME="sglang" + RESTORE_POD_NAME="gemma-sglang-restored" + RESTORE_CONTAINER_NAME="restore" + PORT=30000 + MODEL="google/gemma-4-31B-it" + INFER_ENDPOINT="/v1/completions" + INFER_DATA='{"model":"google/gemma-4-31B-it","prompt":"Hello","max_tokens":5}' + POST_INFER_DATA='{"model":"google/gemma-4-31B-it","prompt":"The meaning of life is","max_tokens":10}' + SOURCE_MANIFEST="$PROJECT_ROOT/deploy/k8s/workloads/gemma-sglang.yaml" + RESTORE_MANIFEST_TEMPLATE="$PROJECT_ROOT/deploy/k8s/workloads/gemma-sglang-restore.yaml" + ;; sglang-small) POD_NAME="sglang-small" CONTAINER_NAME="sglang" @@ -250,7 +265,7 @@ case "$WORKLOAD" in ;; *) echo "Unknown workload: $WORKLOAD" - echo "Supported: vllm-small, vllm-8b, sglang-small, sglang-8b, trtllm-small, nim-llama-8b, vllm-70b, nim-qwen3-32b" + echo "Supported: vllm-small, vllm-8b, gemma-sglang, sglang-small, sglang-8b, trtllm-small, nim-llama-8b, vllm-70b, nim-qwen3-32b" exit 1 ;; esac diff --git a/src/compute-plane-services/nvsnap/scripts/versions.sh b/src/compute-plane-services/nvsnap/scripts/versions.sh index 181a30afb..ddff59c94 100755 --- a/src/compute-plane-services/nvsnap/scripts/versions.sh +++ b/src/compute-plane-services/nvsnap/scripts/versions.sh @@ -42,8 +42,8 @@ NVSNAP_CRIU_SRC="${NVSNAP_CRIU_SRC:-}" # Core images — all on a fresh v0.0.1 baseline. Bump per CLAUDE.md # rule 19 (never reuse a tag on rebuild). -NVSNAP_BASE_VERSION="${NVSNAP_BASE_VERSION:-v0.0.12}" -NVSNAP_APP_VERSION="${NVSNAP_APP_VERSION:-v0.2.22}" +NVSNAP_BASE_VERSION="${NVSNAP_BASE_VERSION:-v0.0.14}" +NVSNAP_APP_VERSION="${NVSNAP_APP_VERSION:-v0.2.25}" NVSNAP_SERVER_VERSION="${NVSNAP_SERVER_VERSION:-v0.0.31}" NVSNAP_BLOBSTORE_VERSION="${NVSNAP_BLOBSTORE_VERSION:-v0.0.1}" NVSNAP_L2WAIT_VERSION="${NVSNAP_L2WAIT_VERSION:-v0.0.1}" From 63fd39d4938e0a3bffbe0a0752e2ecb6f7230855 Mon Sep 17 00:00:00 2001 From: balaji Date: Mon, 20 Jul 2026 21:34:03 -0700 Subject: [PATCH 3/4] fix(nvsnap): feature-detect cuda-checkpoint resume, fall back to restore+unlock Harden the Lever B resume coalesce so the CRIU cuda plugin behaves correctly against a cuda-checkpoint build without the combined `resume` action: detect support once via the help output and fall back to separate restore then unlock when unavailable. Also fixes a latent bug where the pre-resume else-if chain would have skipped unlock for a process needing both restore and unlock. Bumps CRIU ref to 31d90a8a8, base v0.0.15 / app v0.2.26 (+ manifest tag sync). Validated on aws-dev1: vllm-small (resume path exercised) and gemma-4-31B sglang cu129 criu-v2 e2e both PASS. Co-Authored-By: Balaji Ganesan --- .../nvsnap/deploy/k8s/agent-daemonset-crio.yaml | 2 +- .../nvsnap/deploy/k8s/agent-daemonset.yaml | 4 ++-- .../deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml | 2 +- .../nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml | 2 +- .../deploy/k8s/benchmarks/whisper-large-v3-restore.yaml | 2 +- .../nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml | 2 +- .../nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml | 2 +- .../nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml | 2 +- .../nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml | 2 +- .../nvsnap/deploy/k8s/workloads/sglang-8b.yaml | 2 +- .../nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml | 2 +- .../nvsnap/deploy/k8s/workloads/sglang-small.yaml | 2 +- .../nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml | 2 +- .../nvsnap/deploy/k8s/workloads/trtllm-small.yaml | 2 +- src/compute-plane-services/nvsnap/scripts/versions.sh | 6 +++--- 15 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml index a992b2973..0c339c00d 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml @@ -48,7 +48,7 @@ spec: - name: nvsnap-pull-secret containers: - name: agent - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: Always args: - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml index 56a07e429..321473a17 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml @@ -107,7 +107,7 @@ spec: - name: nvsnap-pull-secret containers: - name: agent - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: Always args: - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint @@ -140,7 +140,7 @@ spec: - nvcr.io/0651155215864979/ncp-dev/uvloop-builder:v0.0.1 - nvcr.io/0651155215864979/ncp-dev/libuv-builder:v0.0.1 - nvcr.io/0651155215864979/ncp-dev/libzmq-builder:v0.0.1 - - nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + - nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 securityContext: privileged: true # Required for CRIU and containerd access env: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml index d0bb73dc1..1b68cffce 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml @@ -50,7 +50,7 @@ spec: - { name: nvsnap-lib, mountPath: /nvsnap-lib } - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml index 9cb0a7a41..54800f9f5 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml @@ -69,7 +69,7 @@ spec: - { name: nvsnap-lib, mountPath: /nvsnap-lib } - name: get-nvsnap - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml index a6a0e3950..b9f117114 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml @@ -23,7 +23,7 @@ spec: initContainers: - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml index 543b8c45c..a23ec5acf 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml @@ -46,7 +46,7 @@ spec: # in place. Source image is the agent v0.0.20 — same restore-entrypoint # binary the production deploy will use. - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml index 57a87fbd6..cab5fcf62 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml @@ -68,7 +68,7 @@ spec: mountPath: /nvsnap-lib - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml index 286aa5fc9..7835ad7d4 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml @@ -87,7 +87,7 @@ spec: # Drops both libnvsnap_intercept.so and the runtime sitecustomize.py. # Renamed from get-intercept-lib to get-nvsnap to reflect the broader job. - name: get-nvsnap - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml index 87a503bfb..ab599d26b 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml @@ -67,7 +67,7 @@ spec: # Copy CRIU bundle, intercept library, and debug tools from agent image - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml index d2453342b..b1a1ff120 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml @@ -72,7 +72,7 @@ spec: # Copy intercept lib from agent - name: get-intercept-lib - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml index de9c6cfa7..f29866e4a 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml @@ -72,7 +72,7 @@ spec: # Copy CRIU bundle, intercept library, and debug tools from agent image - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml index a19204a3f..90b94d8c9 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml @@ -71,7 +71,7 @@ spec: mountPath: /nvsnap-lib # Copy intercept lib from agent - name: get-intercept-lib - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml index 7c0f5a302..c1392b06b 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml @@ -62,7 +62,7 @@ spec: mountPath: /nvsnap-lib - name: get-criu - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml index 865a94742..2fee9e3fc 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml @@ -75,7 +75,7 @@ spec: mountPath: /nvsnap-lib - name: get-intercept-lib - image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.25 + image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] args: diff --git a/src/compute-plane-services/nvsnap/scripts/versions.sh b/src/compute-plane-services/nvsnap/scripts/versions.sh index ddff59c94..407a5d4b7 100755 --- a/src/compute-plane-services/nvsnap/scripts/versions.sh +++ b/src/compute-plane-services/nvsnap/scripts/versions.sh @@ -42,8 +42,8 @@ NVSNAP_CRIU_SRC="${NVSNAP_CRIU_SRC:-}" # Core images — all on a fresh v0.0.1 baseline. Bump per CLAUDE.md # rule 19 (never reuse a tag on rebuild). -NVSNAP_BASE_VERSION="${NVSNAP_BASE_VERSION:-v0.0.14}" -NVSNAP_APP_VERSION="${NVSNAP_APP_VERSION:-v0.2.25}" +NVSNAP_BASE_VERSION="${NVSNAP_BASE_VERSION:-v0.0.15}" +NVSNAP_APP_VERSION="${NVSNAP_APP_VERSION:-v0.2.26}" NVSNAP_SERVER_VERSION="${NVSNAP_SERVER_VERSION:-v0.0.31}" NVSNAP_BLOBSTORE_VERSION="${NVSNAP_BLOBSTORE_VERSION:-v0.0.1}" NVSNAP_L2WAIT_VERSION="${NVSNAP_L2WAIT_VERSION:-v0.0.1}" @@ -67,7 +67,7 @@ NVSNAP_CRIU_REPO="${NVSNAP_CRIU_REPO:-https://github.com/balajinvda/criu.git}" # rings survive C/R, so libuv/uvloop servers restore with UV_USE_IO_URING=1) # for reproducible OSS builds. Bump when the fork advances. # Consumed by build-agent.sh (clean-checkout auto-clone) and ci/build-image.sh. -NVSNAP_CRIU_REF="${NVSNAP_CRIU_REF:-1e926fa4da5c1309c3029bedbeef103c1459438d}" +NVSNAP_CRIU_REF="${NVSNAP_CRIU_REF:-31d90a8a847219812185232fd93e8bee55c2a443}" NVSNAP_LIBZMQ_REPO="${NVSNAP_LIBZMQ_REPO:-https://github.com/balajinvda/libzmq.git}" NVSNAP_LIBZMQ_REF="${NVSNAP_LIBZMQ_REF:-checkpoint-restore-v1}" NVSNAP_LIBUV_REPO="${NVSNAP_LIBUV_REPO:-https://github.com/balajinvda/libuv.git}" From 8968ceec45c398bcd78e4c1daf149adc61ca4517 Mon Sep 17 00:00:00 2001 From: balaji Date: Mon, 20 Jul 2026 21:46:33 -0700 Subject: [PATCH 4/4] docs(nvsnap): sync stale version/status comments (CodeRabbit review) - versions.sh: baseline comment reflected the historical v0.0.1 reset as if current; reword to say bumped-from-there. - e5-mistral-replay.yaml: init-container comment hardcoded a stale v0.0.20 agent; point at the synced nvsnap-agent tag instead. - single-gpu-restore-speedup.md: status/phasing said Lever B was unimplemented while the body describes the resume coalesce as shipped; sync both. Co-Authored-By: Balaji Ganesan --- .../deploy/k8s/workloads/e5-mistral-replay.yaml | 9 +++++---- .../docs/proposals/single-gpu-restore-speedup.md | 12 ++++++++---- .../nvsnap/scripts/versions.sh | 4 ++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml index a23ec5acf..466348602 100644 --- a/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml +++ b/src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml @@ -41,10 +41,11 @@ spec: - name: nvcr-regcred initContainers: - # Bundle staging via init container (legacy pattern) so this - # manifest doesn't depend on the v0.0.20 DaemonSet hostPath being - # in place. Source image is the agent v0.0.20 — same restore-entrypoint - # binary the production deploy will use. + # Bundle staging via init container (legacy pattern) so this manifest + # doesn't depend on the agent DaemonSet hostPath being in place. Source + # image is the nvsnap-agent tag below (kept current by + # scripts/sync-versions.sh) — same restore-entrypoint binary the + # production deploy uses. - name: get-criu image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26 imagePullPolicy: IfNotPresent diff --git a/src/compute-plane-services/nvsnap/docs/proposals/single-gpu-restore-speedup.md b/src/compute-plane-services/nvsnap/docs/proposals/single-gpu-restore-speedup.md index 7b9677d08..708587e24 100644 --- a/src/compute-plane-services/nvsnap/docs/proposals/single-gpu-restore-speedup.md +++ b/src/compute-plane-services/nvsnap/docs/proposals/single-gpu-restore-speedup.md @@ -5,7 +5,10 @@ SPDX-License-Identifier: Apache-2.0 # Single-GPU restore speedup (target: -80%) Status: Lever A is implemented, measured, and merged (see MEASURED below). -Levers B and C are proposed, not yet implemented. The decision to build A+B and +Lever B's combined "resume" invocation (restore+unlock in one cuda-checkpoint +spawn) is implemented and measured; eliminating get-restore-tid via a /proc scan +is remaining medium work, and per-pid context parallelization is deferred hard +work. Lever C is proposed, not implemented. The decision to build A+B and defer C is locked (see Decision). Numbers are measured on aws-dev1 (driver 580.126, p5 H100, criu-v2 engine). @@ -155,9 +158,10 @@ investigation (see open questions) rather than block A+B on it. 1. Instrument (planned): OTel spans for pages-restore vs per-pid cuda-restore so the split is a dashboard number, not a one-off log parse (extends #114-#116). 2. Lever A (madvise huge pages) in the CRIU fork -- DONE, merged, regression-gated. -3. Lever B (parallelize + de-latency the plugin) -- planned; measure real GPU - overlap first; keep the handshake-latency removal even if context creation - serializes. +3. Lever B (parallelize + de-latency the plugin) -- resume-coalesce (restore + + unlock in one spawn) DONE and shipped; get-restore-tid /proc scan and per-pid + context parallelization still remaining. Measure real GPU overlap first; keep + the handshake-latency removal even if context creation serializes. 4. Lever C only if the target is same-node -- deferred; larger, touches agent lifecycle (donor freeze/thaw, standby accounting) and NVCA (warm-pool semantics). diff --git a/src/compute-plane-services/nvsnap/scripts/versions.sh b/src/compute-plane-services/nvsnap/scripts/versions.sh index 407a5d4b7..ebcad8135 100755 --- a/src/compute-plane-services/nvsnap/scripts/versions.sh +++ b/src/compute-plane-services/nvsnap/scripts/versions.sh @@ -40,8 +40,8 @@ if [ -z "${NVSNAP_CRIU_SRC:-}" ]; then fi NVSNAP_CRIU_SRC="${NVSNAP_CRIU_SRC:-}" -# Core images — all on a fresh v0.0.1 baseline. Bump per CLAUDE.md -# rule 19 (never reuse a tag on rebuild). +# Core images — reset to a v0.0.1 baseline on 2026-05-21 and bumped from +# there per CLAUDE.md rule 19 (never reuse a tag on rebuild). NVSNAP_BASE_VERSION="${NVSNAP_BASE_VERSION:-v0.0.15}" NVSNAP_APP_VERSION="${NVSNAP_APP_VERSION:-v0.2.26}" NVSNAP_SERVER_VERSION="${NVSNAP_SERVER_VERSION:-v0.0.31}"