Fixing cmk nccl image and initcontainers

cmk-nccltests/nccl-b200.yaml

@@ -3,6 +3,7 @@ kind: MPIJob
 metadata:
   name: nccl-test-b200
 spec:
+  launcherCreationPolicy: WaitForWorkersReady
   slotsPerWorker: 8
   runPolicy:
     cleanPodPolicy: Running
@@ -12,11 +13,6 @@ spec:
       template:
         spec:
           restartPolicy: OnFailure
-          initContainers:
-          - image: ghcr.io/crusoecloud/nccl-tests:13.0.1-ubuntu24.04-nccl-2.29.2-1
-            imagePullPolicy: IfNotPresent
-            name: init
-            command: ["sh", "-c", "sleep 5"]
           volumes:
           - name: nccl-topo
             hostPath:

cmk-nccltests/nccl-b300.yaml

@@ -0,0 +1,143 @@
+apiVersion: kubeflow.org/v2beta1
+kind: MPIJob
+metadata:
+  name: nccl-test-b300
+spec:
+  launcherCreationPolicy: WaitForWorkersReady
+  slotsPerWorker: 8
+  runPolicy:
+    cleanPodPolicy: Running
+  mpiReplicaSpecs:
+    Launcher:
+      replicas: 1
+      template:
+        spec:
+          restartPolicy: OnFailure
+          volumes:
+          - name: nccl-topo
+            hostPath:
+              path: /etc/crusoe/nccl_topo
+              type: Directory
+          containers:
+          - image: ghcr.io/crusoecloud/nccl-tests:13.0.1-ubuntu24.04-nccl-2.29.2-1
+            imagePullPolicy: IfNotPresent
+            volumeMounts:
+            - name: nccl-topo
+              mountPath: /opt/nccl_topo
+            name: nccl-test-launcher
+            securityContext:
+              capabilities:
+                add: ["IPC_LOCK"]
+            env:
+             - name: NCCL_TOPO_FILE
+               value: /opt/nccl_topo/b300-288gb-sxm-ib-cloud-hypervisor.xml
+             - name: UCX_RNDV_SCHEME
+               value: "get_zcopy"   # UCX memory setting
+             - name: UCX_TLS
+               value: "self,sm,cuda_copy"   # UCX memory setting
+            command:
+            - mpirun 
+            - --allow-run-as-root
+            - --tag-output
+            - -np
+            - "16"  # Total number of processes (8 GPUs per node, 2 nodes = 16 total if using 2 Worker Replicas)
+            - -N
+            - "8"
+            - -bind-to
+            - none
+            - -map-by
+            - slot
+            - -mca
+            - coll_hcoll_enable
+            - "0"
+            - -x
+            - NCCL_IB_PCI_RELAXED_ORDERING=1
+            - -x
+            - NCCL_IB_SPLIT_DATA_ON_QPS=0
+            - -x
+            - NCCL_IB_QPS_PER_CONNECTION=2
+            - -x
+            - NCCL_IB_MERGE_VFS=0
+            - -x
+            - NCCL_IB_HCA=mlx5_5:1,mlx5_6:1,mlx5_7:1,mlx5_8:1,mlx5_9:1,mlx5_10:1,mlx5_11:1,mlx5_12:1
+            - -x
+            - NCCL_TOPO_FILE
+            - -x
+            - PATH
+            - -x
+            - LD_LIBRARY_PATH
+            - -x
+            - NCCL_DEBUG=INFO
+            - -x
+            - NCCL_NVLS_ENABLE=1
+            - -x
+            - NCCL_IB_SL=1
+            - -x
+            - NCCL_IBEXT_DISABLE=1
+            - /opt/nccl-tests/build/all_reduce_perf
+            - -b
+            - "2G"
+            - -e
+            - "32G"
+            - -f
+            - "2"
+            - -t
+            - "1"
+            - -g
+            - "1"
+            - -c
+            - "1"
+            - -w # Warmup iterations - typically 1, but should be increased for larger clusters
+            - "50"
+            - -n
+            - "100"
+    Worker:
+      replicas: 2 # Specify how many worker nodes you have running in the Instances tab
+      template:
+        spec:
+          restartPolicy: OnFailure
+          runtimeClassName: nvidia
+          # In case you have GPU taints
+          tolerations:
+          - key: "nvidia.com/gpu"
+            operator: "Exists"
+            effect: "NoSchedule"
+          volumes:
+          - name: dshm
+            emptyDir:
+              medium: Memory
+              sizeLimit: 64Gi
+          - name: nccl-topo
+            hostPath:
+              path: /etc/crusoe/nccl_topo
+              type: Directory
+          containers:
+          - image: ghcr.io/crusoecloud/nccl-tests:13.0.1-ubuntu24.04-nccl-2.29.2-1
+            imagePullPolicy: IfNotPresent
+            name: nccl-worker
+            securityContext:
+              capabilities:
+                add: ["IPC_LOCK"]
+            env:
+             - name: NCCL_TOPO_FILE
+               value: /opt/nccl_topo/b300-288gb-sxm-ib-cloud-hypervisor.xml
+             - name: NCCL_DEBUG
+               value: TRACE
+             - name: UCX_RNDV_SCHEME
+               value: "get_zcopy"   # UCX memory setting
+             - name: UCX_TLS
+               value: "self,sm,cuda_copy"   # UCX memory setting
+            volumeMounts:
+            - mountPath: /dev/shm
+              name: dshm
+            - name: nccl-topo
+              mountPath: /opt/nccl_topo
+            resources:
+              limits:
+                nvidia.com/gpu: 8 # 8 GPUs per node
+                nvidia.com/hostdev: 8
+                memory: 128000Mi
+              requests:
+                nvidia.com/gpu: 8 # 8 GPUs per node
+                nvidia.com/hostdev: 8
+                memory: 128000Mi

cmk-nccltests/nccl-gb200.yaml

@@ -13,6 +13,7 @@ kind: MPIJob
 metadata:
   name: nccl-tests-gb200
 spec:
+  launcherCreationPolicy: WaitForWorkersReady
   slotsPerWorker: 4
   runPolicy:
     cleanPodPolicy: Running
@@ -22,11 +23,6 @@ spec:
       template:
         spec:
           restartPolicy: OnFailure
-          initContainers:
-          - image: ghcr.io/crusoecloud/nccl-tests:13.0.1-ubuntu24.04-nccl-2.29.2-1
-            imagePullPolicy: IfNotPresent
-            name: init
-            command: ["sh", "-c", "sleep 5"]
           containers:
           - image: ghcr.io/crusoecloud/nccl-tests:13.0.1-ubuntu24.04-nccl-2.29.2-1
             imagePullPolicy: IfNotPresent

cmk-nccltests/nccl-h100.yaml

@@ -3,6 +3,7 @@ kind: MPIJob
 metadata:
   name: nccl-test-h100
 spec:
+  launcherCreationPolicy: WaitForWorkersReady
   slotsPerWorker: 8
   runPolicy:
     cleanPodPolicy: Running
@@ -12,11 +13,6 @@ spec:
       template:
         spec:
           restartPolicy: OnFailure
-          initContainers:
-          - image: ghcr.io/crusoecloud/nccl-tests:12.8.1-ubuntu24.04-nccl-2.26.5-1
-            imagePullPolicy: IfNotPresent
-            name: init
-            command: ["sh", "-c", "sleep 5"]
           volumes:
           - name: nccl-topo
             hostPath:

cmk-nccltests/nccl-h200.yaml

@@ -3,6 +3,7 @@ kind: MPIJob
 metadata:
   name: nccl-test-h200
 spec:
+  launcherCreationPolicy: WaitForWorkersReady
   slotsPerWorker: 8
   runPolicy:
     cleanPodPolicy: Running
@@ -12,11 +13,6 @@ spec:
       template:
         spec:
           restartPolicy: OnFailure
-          initContainers:
-          - image: ghcr.io/crusoecloud/nccl-tests:12.8.1-ubuntu24.04-nccl-2.26.5-1
-            imagePullPolicy: IfNotPresent
-            name: init
-            command: ["sh", "-c", "sleep 5"]
           volumes:
           - name: nccl-topo
             hostPath: