[WIP] Allow deploy multiple clusters in same ns

api/v1alpha1/nodeset_types.go

@@ -113,6 +113,12 @@ const (
 
 // NodeSetSpec defines the desired state of NodeSet
 type NodeSetSpec struct {
+	// SlurmClusterRefName is the name of the parent SlurmCluster this NodeSet belongs to.
+	// Must be in the same namespace as the NodeSet.
+	//
+	// +kubebuilder:validation:Required
+	SlurmClusterRefName string `json:"slurmClusterRefName"`
+
 	// Replicas specifies the number of worker nodes in the NodeSet.
 	//
 	// Defaults to 1 if not specified.

cmd/sconfigcontroller/main.go

@@ -176,7 +176,7 @@ func main() {
 		WebhookServer:                 webhookServer,
 		HealthProbeBindAddress:        probeAddr,
 		LeaderElection:                enableLeaderElection,
-		LeaderElectionID:              "vqeyz6ae.nebius.ai",
+		LeaderElectionID:              clusterName + ".vqeyz6ae.nebius.ai",
 		LeaderElectionReleaseOnCancel: true,
 		Cache: cache.Options{
 			DefaultNamespaces: map[string]cache.Config{
@@ -215,6 +215,7 @@ func main() {
 	if err = (sconfigcontroller.NewJailedConfigReconciler(
 		mgr.GetClient(),
 		mgr.GetScheme(),
+		clusterName,
 		slurmAPIClient,
 		jailFs,
 		reconfigurePollInterval,

config/crd/bases/slurm.nebius.ai_nodesets.yaml

@@ -2680,6 +2680,11 @@ spec:
                   Defaults to 1 if not specified.
                 format: int32
                 type: integer
+              slurmClusterRefName:
+                description: |-
+                  SlurmClusterRefName is the name of the parent SlurmCluster this NodeSet belongs to.
+                  Must be in the same namespace as the NodeSet.
+                type: string
               slurmd:
                 description: Slurmd defines the Slurm worker daemon configuration.
                 properties:
@@ -11028,6 +11033,7 @@ spec:
                 type: object
             required:
             - munge
+            - slurmClusterRefName
             - slurmd
             type: object
           status:

helm/nodesets/templates/nodeset.yaml

@@ -19,6 +19,8 @@ metadata:
     {{- end }}
 
 spec:
+  slurmClusterRefName: {{ $.Values.slurmClusterRefName | quote }}
+
   {{- with (.replicas | default 1) }}
   replicas: {{ . }}
   {{- end }}

helm/nodesets/tests/basic_test.yaml

@@ -129,6 +129,34 @@ tests:
       - exists:
           path: metadata.labels["app.kubernetes.io/managed-by"]
 
+  - it: should set slurmClusterRefName from top-level value
+    set:
+      slurmClusterRefName: "my-cluster"
+      nodesets:
+        - name: test-workers
+          slurmd:
+            image:
+              repository: "test/slurm"
+            resources:
+              cpu: "1"
+              memory: "1Gi"
+            volumes:
+              spool:
+                emptyDir: {}
+              jail:
+                emptyDir: {}
+              jailSubMounts: []
+          munge:
+            image:
+              repository: "test/munge"
+            resources:
+              cpu: "100m"
+              memory: "128Mi"
+    asserts:
+      - equal:
+          path: spec.slurmClusterRefName
+          value: "my-cluster"
+
   - it: should set correct namespace
     set:
       nodesets:

helm/nodesets/values.yaml

@@ -5,6 +5,9 @@
 # Global settings
 nameOverride: ""
 fullnameOverride: ""
+# Name of the SlurmCluster this NodeSets release belongs to.
+# Must be in the same namespace as the NodeSets.
+slurmClusterRefName: "soperator"
 # Priority Classes configuration
 # Define priority classes that can be used by NodeSets
 priorityClasses:
@@ -122,7 +125,7 @@ nodesets:
         # Could be any corev1.VolumeSource
         jail:
           persistentVolumeClaim:
-            claimName: &jailPvcClaimName "jail-pvc"
+            claimName: &jailPvcClaimName "soperator-jail-pvc"
         # Volumes being mounted inside Jail mount
         # Optional, defaults to empty list
         jailSubMounts:

helm/slurm-cluster/templates/_helpers.tpl

@@ -62,7 +62,7 @@ Create the name of the service account to use for exporter
 */}}
 {{- define "slurm-cluster.exporter.serviceAccountName" -}}
 {{- if .Values.slurmNodes.exporter.serviceAccount.create -}}
-    {{- default "slurm-exporter-sa" .Values.slurmNodes.exporter.serviceAccount.name }}
+    {{- default (printf "%s-slurm-exporter-sa" (include "slurm-cluster.name" .)) .Values.slurmNodes.exporter.serviceAccount.name }}
 {{- else -}}
     {{- default "default" .Values.slurmNodes.exporter.serviceAccount.name }}
 {{- end -}}
@@ -107,3 +107,9 @@ Create the name of the role binding for slurm-controller
 {{- printf "%s-slurm-controller" (include "slurm-cluster.name" .) }}
 {{- end -}}
 
+{{/*
+Create the name of the slurm-scripts ConfigMap
+*/}}
+{{- define "slurm-cluster.slurmScripts.configMapName" -}}
+{{- printf "%s-slurm-scripts" (include "slurm-cluster.name" .) }}
+{{- end -}}

helm/slurm-cluster/templates/slurm-cluster-cr.yaml

@@ -91,9 +91,9 @@ spec:
     {{- end }}
     {{- end }}
   volumeSources:
-    - name: slurm-scripts
+    - name: {{ include "slurm-cluster.slurmScripts.configMapName" . }}
       configMap:
-        name: slurm-scripts
+        name: {{ include "slurm-cluster.slurmScripts.configMapName" . }}
         defaultMode: 0755
     {{- range .Values.volumeSources }}
     - name: {{ .name | quote }}
@@ -325,10 +325,10 @@ spec:
         customMounts:
           - name: slurm-scripts
             mountPath: /opt/slurm_scripts/
-            volumeSourceName: slurm-scripts
+            volumeSourceName: {{ include "slurm-cluster.slurmScripts.configMapName" . }}
           - name: slurm-scripts-jail
             mountPath: /mnt/jail.upper/opt/slurm_scripts/
-            volumeSourceName: slurm-scripts
+            volumeSourceName: {{ include "slurm-cluster.slurmScripts.configMapName" . }}
           {{- if .Values.slurmNodes.login.volumes.customMounts }}
           {{- .Values.slurmNodes.login.volumes.customMounts | toYaml | nindent 10 }}
           {{- end }}

helm/slurm-cluster/templates/slurm-scripts-cm.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   namespace: {{ .Release.Namespace }}
-  name: slurm-scripts
+  name: {{ include "slurm-cluster.slurmScripts.configMapName" . }}
   labels:
     app: {{ .Chart.Name }}
     release: {{ .Release.Name }}

helm/slurm-cluster/tests/exporter-rbac_test.yaml

@@ -15,7 +15,7 @@ tests:
           of: ServiceAccount
       - equal:
           path: metadata.name
-          value: slurm-exporter-sa
+          value: test-cluster-slurm-exporter-sa
       - equal:
           path: metadata.namespace
           value: NAMESPACE
@@ -114,7 +114,7 @@ tests:
           path: subjects
           content:
             kind: ServiceAccount
-            name: slurm-exporter-sa
+            name: test-cluster-exporter-sa
             namespace: NAMESPACE
 
   # Test RoleBinding with custom ServiceAccount name
@@ -162,7 +162,7 @@ tests:
     asserts:
       - equal:
           path: spec.slurmNodes.exporter.serviceAccountName
-          value: slurm-exporter-sa
+          value: test-cluster-exporter-sa
 
   # Test SlurmCluster CR uses custom ServiceAccount name
   - it: should set custom serviceAccountName in SlurmCluster CR
@@ -207,4 +207,4 @@ tests:
     asserts:
       - equal:
           path: spec.slurmNodes.exporter.serviceAccountName
-          value: slurm-exporter-sa
+          value: test-cluster-exporter-sa

helm/slurm-cluster/tests/volume-sources-filtering_test.yaml

@@ -117,7 +117,7 @@ tests:
       # Verify correct names are preserved (slurm-scripts is always first)
       - equal:
           path: spec.volumeSources[0].name
-          value: "slurm-scripts"
+          value: "test-cluster-slurm-scripts"
       - equal:
           path: spec.volumeSources[1].name
           value: "jail"
@@ -206,7 +206,7 @@ tests:
       # Verify correct values
       - equal:
           path: spec.volumeSources[0].name
-          value: "slurm-scripts"
+          value: "test-cluster-slurm-scripts"
       - equal:
           path: spec.volumeSources[1].name
           value: "simple-pvc"

helm/soperator-crds/templates/slurmcluster-crd.yaml

@@ -17766,6 +17766,11 @@ spec:
                   Defaults to 1 if not specified.
                 format: int32
                 type: integer
+              slurmClusterRefName:
+                description: |-
+                  SlurmClusterRefName is the name of the parent SlurmCluster this NodeSet belongs to.
+                  Must be in the same namespace as the NodeSet.
+                type: string
               slurmd:
                 description: Slurmd defines the Slurm worker daemon configuration.
                 properties:
@@ -26114,6 +26119,7 @@ spec:
                 type: object
             required:
             - munge
+            - slurmClusterRefName
             - slurmd
             type: object
           status:

helm/soperator/crds/slurmcluster-crd.yaml

@@ -17766,6 +17766,11 @@ spec:
                   Defaults to 1 if not specified.
                 format: int32
                 type: integer
+              slurmClusterRefName:
+                description: |-
+                  SlurmClusterRefName is the name of the parent SlurmCluster this NodeSet belongs to.
+                  Must be in the same namespace as the NodeSet.
+                type: string
               slurmd:
                 description: Slurmd defines the Slurm worker daemon configuration.
                 properties:
@@ -26114,6 +26119,7 @@ spec:
                 type: object
             required:
             - munge
+            - slurmClusterRefName
             - slurmd
             type: object
           status:

internal/consts/annotation.go

@@ -6,6 +6,4 @@ const (
 	AnnotationDefaultContainerName = "kubectl.kubernetes.io/default-container"
 	AnnotationClusterName          = K8sGroupNameSoperator + "/cluster"
 	AnnotationActiveCheckName      = K8sGroupNameSoperator + "/activecheck"
-
-	AnnotationParentalClusterRefName = K8sGroupNameSoperator + "/parental-cluster-ref"
 )

internal/controller/clustercontroller/accounting.go

@@ -340,7 +340,7 @@ func (r SlurmClusterReconciler) ReconcileAccounting(
 
 					if !isAccountingEnabled {
 						stepLogger.V(1).Info("Removing")
-						deploymentName := naming.BuildDeploymentName(consts.ComponentTypeAccounting)
+						deploymentName := naming.BuildDeploymentName(consts.ComponentTypeAccounting, clusterValues.Name)
 						if err = r.Deployment.Cleanup(stepCtx, cluster, deploymentName); err != nil {
 							return fmt.Errorf("cleanup accounting Deployment: %w", err)
 						}

internal/controller/clustercontroller/sconfigcontroller.go

@@ -93,7 +93,7 @@ func (r SlurmClusterReconciler) ValidateSConfigController(
 		ctx,
 		types.NamespacedName{
 			Namespace: clusterValues.Namespace,
-			Name:      naming.BuildDeploymentName(consts.ComponentTypeSConfigController),
+			Name:      naming.BuildDeploymentName(consts.ComponentTypeSConfigController, clusterValues.Name),
 		},
 		existing,
 	)

internal/controller/nodesetcontroller/reconcile.go

@@ -68,9 +68,9 @@ func (r *NodeSetReconciler) reconcile(ctx context.Context, nodeSet *slurmv1alpha
 		err     error
 	)
 	{
-		clusterName, hasClusterRef := nodeSet.GetAnnotations()[consts.AnnotationParentalClusterRefName]
-		if !hasClusterRef {
-			err = fmt.Errorf("getting parental cluster ref from annotations")
+		clusterName := nodeSet.Spec.SlurmClusterRefName
+		if clusterName == "" {
+			err = fmt.Errorf("NodeSet %q has empty spec.slurmClusterRefName", nodeSet.Name)
 			logger.Error(err, "No parent cluster ref found")
 			return ctrl.Result{}, err
 		}

internal/controller/sconfigcontroller/jailedconfig_controller.go

@@ -62,6 +62,7 @@ type JailedConfigReconciler struct {
 	client.Client
 	Scheme *runtime.Scheme
 
+	clusterName             string
 	slurmAPIClient          slurmapi.Client
 	clock                   Clock
 	fs                      Fs
@@ -331,7 +332,10 @@ func (r *JailedConfigReconciler) reconcileWithAggregation(ctx context.Context, j
 	jailedConfigs := &slurmv1alpha1.JailedConfigList{}
 	err := r.Client.List(ctx, jailedConfigs,
 		client.InNamespace(jailedConfig.Namespace),
-		client.MatchingLabels{consts.LabelJailedAggregationKey: aggregationKey},
+		client.MatchingLabels{
+			consts.LabelJailedAggregationKey: aggregationKey,
+			consts.LabelInstanceKey:          r.clusterName,
+		},
 	)
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("listing JailedConfigs with aggregation key %q: %w", aggregationKey, err)
@@ -557,6 +561,7 @@ func (r *JailedConfigReconciler) shouldInitializeConditions(ctx context.Context,
 func NewJailedConfigReconciler(
 	client client.Client,
 	scheme *runtime.Scheme,
+	clusterName string,
 	slurmAPIClient slurmapi.Client,
 	fs Fs,
 	reconfigurePollInterval time.Duration,
@@ -571,6 +576,7 @@ func NewJailedConfigReconciler(
 	return &JailedConfigReconciler{
 		Client:                  client,
 		Scheme:                  scheme,
+		clusterName:             clusterName,
 		slurmAPIClient:          slurmAPIClient,
 		fs:                      fs,
 		reconfigurePollInterval: reconfigurePollInterval,
@@ -592,14 +598,18 @@ func (r *JailedConfigReconciler) SetupWithManager(mgr ctrl.Manager, maxConcurren
 	}); err != nil {
 		return err
 	}
+	clusterPredicate := predicate.NewPredicateFuncs(func(obj client.Object) bool {
+		return obj.GetLabels()[consts.LabelInstanceKey] == r.clusterName
+	})
+
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&slurmv1alpha1.JailedConfig{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
+		For(&slurmv1alpha1.JailedConfig{}, builder.WithPredicates(predicate.GenerationChangedPredicate{}, clusterPredicate)).
 		Watches(
 			&corev1.ConfigMap{},
 			handler.EnqueueRequestsFromMapFunc(r.findObjectsForConfigMap),
-			builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
+			builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}, clusterPredicate),
 		).
-		Named("jailedconfig").
+		Named(r.clusterName + "-jailedconfig").
 		WithOptions(controllerconfig.ControllerOptionsWithRateLimit(maxConcurrency, cacheSyncTimeout, 15*time.Second, 1*time.Minute)).
 		Complete(r)
 }

internal/controller/sconfigcontroller/jailedconfig_controller_test.go

@@ -87,6 +87,7 @@ func newTestJailedConfigController(
 	sctrl := NewJailedConfigReconciler(
 		mgr.GetClient(),
 		mgr.GetScheme(),
+		"soperator",
 		apiClient,
 		fakeFs,
 		1*time.Second, // Poll interval for tests