perf(nvsnap): coalesce cuda-checkpoint restore+unlock (Lever B)

src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset-crio.yaml

@@ -48,7 +48,7 @@ spec:
         - name: nvsnap-pull-secret
       containers:
         - name: agent
-          image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+          image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
           imagePullPolicy: Always
           args:
             - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint

src/compute-plane-services/nvsnap/deploy/k8s/agent-daemonset.yaml

@@ -107,7 +107,7 @@ spec:
         - name: nvsnap-pull-secret
       containers:
         - name: agent
-          image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+          image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
           imagePullPolicy: Always
           args:
             - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint
@@ -140,7 +140,7 @@ spec:
             - nvcr.io/0651155215864979/ncp-dev/uvloop-builder:v0.0.1
             - nvcr.io/0651155215864979/ncp-dev/libuv-builder:v0.0.1
             - nvcr.io/0651155215864979/ncp-dev/libzmq-builder:v0.0.1
-            - nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+            - nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
           securityContext:
             privileged: true  # Required for CRIU and containerd access
           env:

src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim-restore.yaml

@@ -50,7 +50,7 @@ spec:
         - { name: nvsnap-lib, mountPath: /nvsnap-lib }
 
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gemma-4-31b-nim.yaml

@@ -69,7 +69,7 @@ spec:
         - { name: nvsnap-lib, mountPath: /nvsnap-lib }
 
     - name: get-nvsnap
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/whisper-large-v3-restore.yaml

@@ -23,7 +23,7 @@ spec:
 
   initContainers:
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/e5-mistral-replay.yaml

@@ -41,12 +41,13 @@ spec:
     - name: nvcr-regcred
 
   initContainers:
-    # Bundle staging via init container (legacy pattern) so this
-    # manifest doesn't depend on the v0.0.20 DaemonSet hostPath being
-    # in place. Source image is the agent v0.0.20 — same restore-entrypoint
-    # binary the production deploy will use.
+    # Bundle staging via init container (legacy pattern) so this manifest
+    # doesn't depend on the agent DaemonSet hostPath being in place. Source
+    # image is the nvsnap-agent tag below (kept current by
+    # scripts/sync-versions.sh) — same restore-entrypoint binary the
+    # production deploy uses.
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/gemma-sglang-restore.yaml

@@ -0,0 +1,61 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Restore placeholder for a criu-v2 checkpoint of gemma-sglang.
+# Dumb reaper (same image as source), bash pid1 reaps orphans, no
+# restore-entrypoint; the agent drives restore via POST /v1/restore. See
+# vllm-small-restore.yaml.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: gemma-sglang-restored
+  namespace: nvsnap-system
+  labels:
+    app: gemma-sglang-restored
+    nvsnap.io/demo: "true"
+spec:
+  automountServiceAccountToken: false
+  nodeName: __NODE_NAME__
+  imagePullSecrets:
+    - name: nvsnap-pull-secret
+  containers:
+    - name: restore
+      image: lmsysorg/sglang:v0.5.15.post1-cu129
+      imagePullPolicy: IfNotPresent
+      command: ["/bin/bash", "-lc"]
+      args:
+        - |
+          set -e
+          echo 100000 > /proc/sys/kernel/ns_last_pid || echo "(ns_last_pid bump failed)"
+          touch /sglang.out
+          tail -F /sglang.out &
+          while true; do sleep 30; done
+      env:
+        - name: CHECKPOINT_ID
+          value: "__CHECKPOINT_ID__"
+      readinessProbe:
+        httpGet: { path: /health, port: 30000 }
+        initialDelaySeconds: 5
+        periodSeconds: 5
+        timeoutSeconds: 5
+        failureThreshold: 120
+      securityContext:
+        privileged: true
+      resources:
+        limits:
+          nvidia.com/gpu: "1"
+        requests:
+          nvidia.com/gpu: "1"
+      volumeMounts:
+        - { name: checkpoints, mountPath: /checkpoints }
+        - { name: dev-shm, mountPath: /dev/shm }
+  volumes:
+    - name: checkpoints
+      hostPath:
+        path: /var/lib/containerd/nvsnap-checkpoints
+        type: Directory
+    - name: dev-shm
+      emptyDir:
+        medium: Memory
+        sizeLimit: 16Gi
+  restartPolicy: Never

src/compute-plane-services/nvsnap/deploy/k8s/workloads/gemma-sglang.yaml

@@ -0,0 +1,88 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# SGLang gemma-4-31B-it (fp8, single GPU) - criu-v2 checkpoint/restore.
+# The good single-GPU SGLang model from the NVCA/PDF benchmark, run through
+# the criu-v2 in-namespace CRIU engine (setsid source + dumb placeholder
+# restore, no injection stack). Non-Riva backend, so the standard CRIU +
+# cuda-checkpoint path (no guardrail bypass needed). HF-gated model: the
+# hf-token secret supplies the download credential.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: gemma-sglang
+  namespace: nvsnap-system
+  labels:
+    app: gemma-sglang
+    nvsnap.io/demo: "true"
+  annotations:
+    nvsnap.io/demo-name: "SGLang gemma-4-31B"
+    nvsnap.io/desc: "google/gemma-4-31B-it on SGLang, fp8, single GPU"
+    nvsnap.io/model: "google/gemma-4-31B-it"
+    nvsnap.io/port: "30000"
+    nvsnap.io/gpus: "1"
+    nvsnap.io/path: "criu"
+spec:
+  tolerations:
+    - key: "nvidia.com/gpu"
+      operator: "Exists"
+      effect: "NoSchedule"
+  imagePullSecrets:
+    - name: nvsnap-pull-secret
+  containers:
+    - name: sglang
+      image: lmsysorg/sglang:v0.5.15.post1-cu129
+      imagePullPolicy: IfNotPresent
+      command: ["/bin/bash", "-lc"]
+      args:
+        - |
+          set -e
+          ulimit -c unlimited
+          # criu-v2 convention: workload in its own session via setsid; stdio to
+          # a rootfs file; bash stays pid1 to reap orphans. See vllm-small.
+          nohup setsid python3 -m sglang.launch_server \
+            --model-path google/gemma-4-31B-it \
+            --host 0.0.0.0 \
+            --port 30000 \
+            --tp-size 1 \
+            --mem-fraction-static 0.85 \
+            --quantization fp8 > /sglang.out 2>&1 < /dev/null &
+          tail -F /sglang.out &
+          while true; do sleep 30; done
+      env:
+        - { name: LD_LIBRARY_PATH, value: "/usr/local/nvidia/lib64:/usr/local/cuda/lib64" }
+        - { name: PYTHONFAULTHANDLER, value: "1" }
+        - { name: PYTHONUNBUFFERED,   value: "1" }
+        - { name: NVSNAP_LOG_LEVEL,     value: "3" }
+        - { name: NVSNAP_SECCOMP_ENABLED, value: "0" }
+        - { name: CUDA_VISIBLE_DEVICES, value: "0" }
+        - { name: HF_HUB_DISABLE_XET,        value: "1" }
+        - { name: HF_HUB_ENABLE_HF_TRANSFER, value: "0" }
+        - { name: USE_LIBUV,                 value: "0" }
+        - name: HF_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: hf-token
+              key: token
+      ports:
+        - containerPort: 30000
+          name: http
+      readinessProbe:
+        httpGet: { path: /health, port: 30000 }
+        initialDelaySeconds: 60
+        periodSeconds: 15
+        timeoutSeconds: 5
+        failureThreshold: 80
+      resources:
+        limits:
+          nvidia.com/gpu: "1"
+        requests:
+          nvidia.com/gpu: "1"
+      securityContext:
+        privileged: true
+      volumeMounts:
+        - { name: shm, mountPath: /dev/shm }
+  volumes:
+    - name: shm
+      emptyDir: { medium: Memory, sizeLimit: 16Gi }
+  restartPolicy: Never

src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b-restore.yaml

@@ -68,7 +68,7 @@ spec:
           mountPath: /nvsnap-lib
 
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-llama-8b.yaml

@@ -87,7 +87,7 @@ spec:
     # Drops both libnvsnap_intercept.so and the runtime sitecustomize.py.
     # Renamed from get-intercept-lib to get-nvsnap to reflect the broader job.
     - name: get-nvsnap
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b-restore.yaml

@@ -67,7 +67,7 @@ spec:
 
   # Copy CRIU bundle, intercept library, and debug tools from agent image
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-8b.yaml

@@ -72,7 +72,7 @@ spec:
 
   # Copy intercept lib from agent
     - name: get-intercept-lib
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small-restore.yaml

@@ -72,7 +72,7 @@ spec:
 
   # Copy CRIU bundle, intercept library, and debug tools from agent image
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/sglang-small.yaml

@@ -71,7 +71,7 @@ spec:
           mountPath: /nvsnap-lib
   # Copy intercept lib from agent
     - name: get-intercept-lib
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small-restore.yaml

@@ -62,7 +62,7 @@ spec:
           mountPath: /nvsnap-lib
 
     - name: get-criu
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/trtllm-small.yaml

@@ -75,7 +75,7 @@ spec:
           mountPath: /nvsnap-lib
 
     - name: get-intercept-lib
-      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.21
+      image: nvcr.io/0651155215864979/ncp-dev/nvsnap-agent:v0.2.26
       imagePullPolicy: IfNotPresent
       command: ["/bin/sh", "-c"]
       args:

src/compute-plane-services/nvsnap/docker/agent/Dockerfile.base

@@ -82,6 +82,26 @@ RUN mkdir -p /criu-bundle/lib /criu-bundle/plugins && \
     patchelf --set-interpreter /criu-bundle/lib/ld-linux-x86-64.so.2 \
              --set-rpath /criu-bundle/lib /criu-bundle/criu
 
+# Make the bundle fully self-contained via per-library RPATH. DT_RUNPATH on
+# the criu binary is NOT inherited by its libraries' own dependencies, so a
+# transitive dep (e.g. libnftables -> libmnl) would not resolve from the bundle
+# without an external LD_LIBRARY_PATH=/criu-bundle/lib. That env var is then
+# inherited by every child criu spawns -- including cuda-checkpoint, which must
+# use the *target container's* glibc; on a newer-glibc image it aborts with
+# "GLIBC_2.xx not found". Setting RPATH=$ORIGIN on each bundled library lets the
+# loader find siblings in the same dir, so criu runs with no LD_LIBRARY_PATH at
+# all and nothing leaks into cuda-checkpoint. Plugins live one dir over.
+RUN set -e; \
+    for so in /criu-bundle/lib/*.so*; do \
+        [ -f "$so" ] || continue; \
+        case "$so" in */ld-linux*) continue;; esac; \
+        patchelf --set-rpath '$ORIGIN' "$so"; \
+    done; \
+    for so in /criu-bundle/plugins/*.so; do \
+        [ -f "$so" ] || continue; \
+        patchelf --set-rpath '$ORIGIN/../lib' "$so"; \
+    done
+
 # Create CRIU wrapper for portable execution
 RUN echo '#!/bin/bash' > /criu-bundle/criu-wrapper && \
     echo 'DIR="$(dirname "$(readlink -f "$0")")"' >> /criu-bundle/criu-wrapper && \

src/compute-plane-services/nvsnap/docker/agent/nvsnap-cuda-checkpoint.c

@@ -1,7 +1,4 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
- * SPDX-License-Identifier: Apache-2.0
- *
  * nvsnap-cuda-checkpoint: a drop-in replacement for NVIDIA's `cuda-checkpoint`
  * binary, built directly on the CUDA driver checkpoint API (the CUDA_CHECKPOINT
  * group: cuCheckpointProcess{GetState,GetRestoreThreadId,Lock,Checkpoint,
@@ -11,7 +8,7 @@
  * by name on PATH) can call this instead, with no other changes:
  *
  *   --get-state       --pid <pid>
- *   --action lock|checkpoint|restore|unlock --pid <pid> [--timeout <ms>]
+ *   --action lock|checkpoint|restore|unlock|resume --pid <pid> [--timeout <ms>]
  *   --toggle          --pid <pid>
  *   --get-restore-tid --pid <pid>
  *
@@ -83,6 +80,15 @@ static int do_unlock(int pid)
     return r == CUDA_SUCCESS ? 0 : err_action("unlock", pid, r);
 }
 
+/* resume: restore then unlock in one process, so cuInit (the ~2.7s driver
+ * attach) is paid once instead of once per action. Used by the CRIU cuda
+ * plugin on the common restore path (process was running at checkpoint). */
+static int do_resume(int pid)
+{
+    int rc = do_restore(pid);
+    return rc ? rc : do_unlock(pid);
+}
+
 static int do_get_state(int pid)
 {
     CUprocessState s;
@@ -134,7 +140,7 @@ static void usage(const char *p)
         "nvsnap-cuda-checkpoint: CUDA checkpoint/restore via the driver API.\n"
         "Operations:\n"
         "  --get-state       --pid <pid>\n"
-        "  --action lock|checkpoint|restore|unlock --pid <pid> [--timeout <ms>]\n"
+        "  --action lock|checkpoint|restore|unlock|resume --pid <pid> [--timeout <ms>]\n"
         "  --toggle          --pid <pid>\n"
         "  --get-restore-tid --pid <pid>\n"
         "Options:\n"
@@ -201,6 +207,7 @@ int main(int argc, char **argv)
     if (!strcmp(action, "checkpoint")) return do_checkpoint(pid);
     if (!strcmp(action, "restore"))    return do_restore(pid);
     if (!strcmp(action, "unlock"))     return do_unlock(pid);
+    if (!strcmp(action, "resume"))     return do_resume(pid);
 
     fprintf(stderr, "error: unknown action '%s'\n", action);
     usage(argv[0]);