From 154d0c876f72f264272586df6dbb0f758819e810 Mon Sep 17 00:00:00 2001 From: Kaustubh Phatak Date: Mon, 24 Aug 2026 12:02:52 -0700 Subject: [PATCH] Fixing cmk nccl image and initcontainers --- cmk-nccltests/nccl-b200.yaml | 6 +- cmk-nccltests/nccl-b300.yaml | 143 ++++++++++++++++++++++++++++++++++ cmk-nccltests/nccl-gb200.yaml | 6 +- cmk-nccltests/nccl-h100.yaml | 6 +- cmk-nccltests/nccl-h200.yaml | 6 +- 5 files changed, 147 insertions(+), 20 deletions(-) create mode 100644 cmk-nccltests/nccl-b300.yaml diff --git a/cmk-nccltests/nccl-b200.yaml b/cmk-nccltests/nccl-b200.yaml index 07e39a3..61fe81c 100644 --- a/cmk-nccltests/nccl-b200.yaml +++ b/cmk-nccltests/nccl-b200.yaml @@ -3,6 +3,7 @@ kind: MPIJob metadata: name: nccl-test-b200 spec: + launcherCreationPolicy: WaitForWorkersReady slotsPerWorker: 8 runPolicy: cleanPodPolicy: Running @@ -12,11 +13,6 @@ spec: template: spec: restartPolicy: OnFailure - initContainers: - - image: ghcr.io/crusoecloud/nccl-tests:13.0.1-ubuntu24.04-nccl-2.29.2-1 - imagePullPolicy: IfNotPresent - name: init - command: ["sh", "-c", "sleep 5"] volumes: - name: nccl-topo hostPath: diff --git a/cmk-nccltests/nccl-b300.yaml b/cmk-nccltests/nccl-b300.yaml new file mode 100644 index 0000000..b64eab7 --- /dev/null +++ b/cmk-nccltests/nccl-b300.yaml @@ -0,0 +1,143 @@ +apiVersion: kubeflow.org/v2beta1 +kind: MPIJob +metadata: + name: nccl-test-b300 +spec: + launcherCreationPolicy: WaitForWorkersReady + slotsPerWorker: 8 + runPolicy: + cleanPodPolicy: Running + mpiReplicaSpecs: + Launcher: + replicas: 1 + template: + spec: + restartPolicy: OnFailure + volumes: + - name: nccl-topo + hostPath: + path: /etc/crusoe/nccl_topo + type: Directory + containers: + - image: ghcr.io/crusoecloud/nccl-tests:13.0.1-ubuntu24.04-nccl-2.29.2-1 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: nccl-topo + mountPath: /opt/nccl_topo + name: nccl-test-launcher + securityContext: + capabilities: + add: ["IPC_LOCK"] + env: + - name: NCCL_TOPO_FILE + value: /opt/nccl_topo/b300-288gb-sxm-ib-cloud-hypervisor.xml + - name: UCX_RNDV_SCHEME + value: "get_zcopy" # UCX memory setting + - name: UCX_TLS + value: "self,sm,cuda_copy" # UCX memory setting + command: + - mpirun + - --allow-run-as-root + - --tag-output + - -np + - "16" # Total number of processes (8 GPUs per node, 2 nodes = 16 total if using 2 Worker Replicas) + - -N + - "8" + - -bind-to + - none + - -map-by + - slot + - -mca + - coll_hcoll_enable + - "0" + - -x + - NCCL_IB_PCI_RELAXED_ORDERING=1 + - -x + - NCCL_IB_SPLIT_DATA_ON_QPS=0 + - -x + - NCCL_IB_QPS_PER_CONNECTION=2 + - -x + - NCCL_IB_MERGE_VFS=0 + - -x + - NCCL_IB_HCA=mlx5_5:1,mlx5_6:1,mlx5_7:1,mlx5_8:1,mlx5_9:1,mlx5_10:1,mlx5_11:1,mlx5_12:1 + - -x + - NCCL_TOPO_FILE + - -x + - PATH + - -x + - LD_LIBRARY_PATH + - -x + - NCCL_DEBUG=INFO + - -x + - NCCL_NVLS_ENABLE=1 + - -x + - NCCL_IB_SL=1 + - -x + - NCCL_IBEXT_DISABLE=1 + - /opt/nccl-tests/build/all_reduce_perf + - -b + - "2G" + - -e + - "32G" + - -f + - "2" + - -t + - "1" + - -g + - "1" + - -c + - "1" + - -w # Warmup iterations - typically 1, but should be increased for larger clusters + - "50" + - -n + - "100" + Worker: + replicas: 2 # Specify how many worker nodes you have running in the Instances tab + template: + spec: + restartPolicy: OnFailure + runtimeClassName: nvidia + # In case you have GPU taints + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 64Gi + - name: nccl-topo + hostPath: + path: /etc/crusoe/nccl_topo + type: Directory + containers: + - image: ghcr.io/crusoecloud/nccl-tests:13.0.1-ubuntu24.04-nccl-2.29.2-1 + imagePullPolicy: IfNotPresent + name: nccl-worker + securityContext: + capabilities: + add: ["IPC_LOCK"] + env: + - name: NCCL_TOPO_FILE + value: /opt/nccl_topo/b300-288gb-sxm-ib-cloud-hypervisor.xml + - name: NCCL_DEBUG + value: TRACE + - name: UCX_RNDV_SCHEME + value: "get_zcopy" # UCX memory setting + - name: UCX_TLS + value: "self,sm,cuda_copy" # UCX memory setting + volumeMounts: + - mountPath: /dev/shm + name: dshm + - name: nccl-topo + mountPath: /opt/nccl_topo + resources: + limits: + nvidia.com/gpu: 8 # 8 GPUs per node + nvidia.com/hostdev: 8 + memory: 128000Mi + requests: + nvidia.com/gpu: 8 # 8 GPUs per node + nvidia.com/hostdev: 8 + memory: 128000Mi \ No newline at end of file diff --git a/cmk-nccltests/nccl-gb200.yaml b/cmk-nccltests/nccl-gb200.yaml index 0143838..12d3e48 100644 --- a/cmk-nccltests/nccl-gb200.yaml +++ b/cmk-nccltests/nccl-gb200.yaml @@ -13,6 +13,7 @@ kind: MPIJob metadata: name: nccl-tests-gb200 spec: + launcherCreationPolicy: WaitForWorkersReady slotsPerWorker: 4 runPolicy: cleanPodPolicy: Running @@ -22,11 +23,6 @@ spec: template: spec: restartPolicy: OnFailure - initContainers: - - image: ghcr.io/crusoecloud/nccl-tests:13.0.1-ubuntu24.04-nccl-2.29.2-1 - imagePullPolicy: IfNotPresent - name: init - command: ["sh", "-c", "sleep 5"] containers: - image: ghcr.io/crusoecloud/nccl-tests:13.0.1-ubuntu24.04-nccl-2.29.2-1 imagePullPolicy: IfNotPresent diff --git a/cmk-nccltests/nccl-h100.yaml b/cmk-nccltests/nccl-h100.yaml index 3e25ecd..9d87a51 100644 --- a/cmk-nccltests/nccl-h100.yaml +++ b/cmk-nccltests/nccl-h100.yaml @@ -3,6 +3,7 @@ kind: MPIJob metadata: name: nccl-test-h100 spec: + launcherCreationPolicy: WaitForWorkersReady slotsPerWorker: 8 runPolicy: cleanPodPolicy: Running @@ -12,11 +13,6 @@ spec: template: spec: restartPolicy: OnFailure - initContainers: - - image: ghcr.io/crusoecloud/nccl-tests:12.8.1-ubuntu24.04-nccl-2.26.5-1 - imagePullPolicy: IfNotPresent - name: init - command: ["sh", "-c", "sleep 5"] volumes: - name: nccl-topo hostPath: diff --git a/cmk-nccltests/nccl-h200.yaml b/cmk-nccltests/nccl-h200.yaml index 5f3f57b..7bedc69 100644 --- a/cmk-nccltests/nccl-h200.yaml +++ b/cmk-nccltests/nccl-h200.yaml @@ -3,6 +3,7 @@ kind: MPIJob metadata: name: nccl-test-h200 spec: + launcherCreationPolicy: WaitForWorkersReady slotsPerWorker: 8 runPolicy: cleanPodPolicy: Running @@ -12,11 +13,6 @@ spec: template: spec: restartPolicy: OnFailure - initContainers: - - image: ghcr.io/crusoecloud/nccl-tests:12.8.1-ubuntu24.04-nccl-2.26.5-1 - imagePullPolicy: IfNotPresent - name: init - command: ["sh", "-c", "sleep 5"] volumes: - name: nccl-topo hostPath: