backport partial #2406

This is required so that newer version of dcgm-exporter can have correct rbac permissions if needed

Signed-off-by: Rahul Sharma <rahulsharm@nvidia.com>

Author: rahulait

assets/state-dcgm-exporter/0210_clusterrole.yaml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: nvidia-dcgm-exporter-read-pods
+  labels:
+    app: nvidia-dcgm-exporter
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch

assets/state-dcgm-exporter/0310_clusterrolebinding.yaml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: nvidia-dcgm-exporter-read-pods
+  labels:
+    app: nvidia-dcgm-exporter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nvidia-dcgm-exporter-read-pods
+subjects:
+- kind: ServiceAccount
+  name: nvidia-dcgm-exporter
+  namespace: "FILLED BY THE OPERATOR"

assets/state-dcgm-exporter/0800_daemonset.yaml

@@ -24,6 +24,7 @@ spec:
           effect: NoSchedule
       priorityClassName: system-node-critical
       serviceAccountName: nvidia-dcgm-exporter
+      automountServiceAccountToken: false
       initContainers:
       - name: toolkit-validation
         image: "FILLED BY THE OPERATOR"

controllers/object_controls.go

@@ -445,6 +445,33 @@ func RoleBinding(n ClusterPolicyController) (gpuv1.State, error) {
 	return gpuv1.Ready, nil
 }
 
+var rbacGates = map[string]func(*gpuv1.ClusterPolicySpec) bool{
+	"nvidia-dcgm-exporter-read-pods": func(config *gpuv1.ClusterPolicySpec) bool {
+		return isDCGMExporterKubernetesPodMetadataEnabled(&config.DCGMExporter)
+	},
+}
+
+func isRBACEnabled(name string, config *gpuv1.ClusterPolicySpec) bool {
+	gate, ok := rbacGates[name]
+	if !ok {
+		return true
+	}
+	return gate(config)
+}
+
+func isDCGMExporterKubernetesPodMetadataEnabled(config *gpuv1.DCGMExporterSpec) bool {
+	for _, env := range config.Env {
+		switch env.Name {
+		case "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID":
+			enabled, err := strconv.ParseBool(env.Value)
+			if err == nil && enabled {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // ClusterRole creates ClusterRole resource
 func ClusterRole(n ClusterPolicyController) (gpuv1.State, error) {
 	ctx := n.ctx
@@ -464,6 +491,15 @@ func ClusterRole(n ClusterPolicyController) (gpuv1.State, error) {
 		return gpuv1.Disabled, nil
 	}
 
+	if !isRBACEnabled(obj.Name, &n.singleton.Spec) {
+		err := n.client.Delete(ctx, obj)
+		if err != nil && !apierrors.IsNotFound(err) {
+			logger.Info("Couldn't delete", "Error", err)
+			return gpuv1.NotReady, err
+		}
+		return gpuv1.Disabled, nil
+	}
+
 	if err := controllerutil.SetControllerReference(n.singleton, obj, n.scheme); err != nil {
 		return gpuv1.NotReady, err
 	}
@@ -505,6 +541,15 @@ func ClusterRoleBinding(n ClusterPolicyController) (gpuv1.State, error) {
 		return gpuv1.Disabled, nil
 	}
 
+	if !isRBACEnabled(obj.Name, &n.singleton.Spec) {
+		err := n.client.Delete(ctx, obj)
+		if err != nil && !apierrors.IsNotFound(err) {
+			logger.Info("Couldn't delete", "Error", err)
+			return gpuv1.NotReady, err
+		}
+		return gpuv1.Disabled, nil
+	}
+
 	for idx := range obj.Subjects {
 		obj.Subjects[idx].Namespace = n.operatorNamespace
 	}
@@ -1820,6 +1865,10 @@ func TransformDCGMExporter(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpe
 		obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, jobMappingVol)
 	}
 
+	if isDCGMExporterKubernetesPodMetadataEnabled(&config.DCGMExporter) {
+		obj.Spec.Template.Spec.AutomountServiceAccountToken = ptr.To(true)
+	}
+
 	// mount configmap for custom metrics if provided by user
 	if config.DCGMExporter.MetricsConfig != nil && config.DCGMExporter.MetricsConfig.Name != "" {
 		metricsConfigVolMount := corev1.VolumeMount{Name: "metrics-config", ReadOnly: true, MountPath: MetricsConfigMountPath, SubPath: MetricsConfigFileName}

controllers/object_controls_test.go

@@ -526,7 +526,7 @@ func testDaemonsetCommon(t *testing.T, cp *gpuv1.ClusterPolicy, component string
 		require.Equal(t, spec.args, mainCtr.Args, "unexpected Args")
 	}
 	for _, env := range spec.env {
-		require.Contains(t, mainCtr.Env, env, "env var not present")
+		require.Equal(t, env.Value, getContainerEnv(&mainCtr, env.Name), "unexpected value for env var %s", env.Name)
 	}
 	// TODO: implement checks for other common fields (i.e. Resources, securityContext, Tolerations, etc.)
 
@@ -1151,6 +1151,14 @@ func getDCGMExporterTestInput(testCase string) *gpuv1.ClusterPolicy {
 	case "standalone-dcgm":
 		dcgmEnabled := true
 		cp.Spec.DCGM.Enabled = &dcgmEnabled
+	case "pod-labels-enabled":
+		cp.Spec.DCGMExporter.Env = []gpuv1.EnvVar{
+			{Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS", Value: "true"},
+		}
+	case "pod-uid-enabled":
+		cp.Spec.DCGMExporter.Env = []gpuv1.EnvVar{
+			{Name: "DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID", Value: "true"},
+		}
 	default:
 		return nil
 	}
@@ -1166,6 +1174,7 @@ func getDCGMExporterTestOutput(testCase string) map[string]interface{} {
 		"numDaemonsets":     1,
 		"dcgmExporterImage": "nvcr.io/nvidia/k8s/dcgm-exporter:3.3.0-3.2.0-ubuntu22.04",
 		"imagePullSecret":   "ngc-secret",
+		"clusterRoleExists": false,
 	}
 
 	switch testCase {
@@ -1175,6 +1184,16 @@ func getDCGMExporterTestOutput(testCase string) map[string]interface{} {
 		output["env"] = map[string]string{
 			"DCGM_REMOTE_HOSTENGINE_INFO": "nvidia-dcgm:5555",
 		}
+	case "pod-labels-enabled":
+		output["env"] = map[string]string{
+			"DCGM_EXPORTER_KUBERNETES_ENABLE_POD_LABELS": "true",
+		}
+		output["clusterRoleExists"] = true
+	case "pod-uid-enabled":
+		output["env"] = map[string]string{
+			"DCGM_EXPORTER_KUBERNETES_ENABLE_POD_UID": "true",
+		}
+		output["clusterRoleExists"] = true
 	default:
 		return nil
 	}
@@ -1200,6 +1219,16 @@ func TestDCGMExporter(t *testing.T) {
 			getDCGMExporterTestInput("standalone-dcgm"),
 			getDCGMExporterTestOutput("standalone-dcgm"),
 		},
+		{
+			"PodLabelsEnabled",
+			getDCGMExporterTestInput("pod-labels-enabled"),
+			getDCGMExporterTestOutput("pod-labels-enabled"),
+		},
+		{
+			"PodUIDEnabled",
+			getDCGMExporterTestInput("pod-uid-enabled"),
+			getDCGMExporterTestOutput("pod-uid-enabled"),
+		},
 	}
 
 	for _, tc := range testCases {
@@ -1231,6 +1260,23 @@ func TestDCGMExporter(t *testing.T) {
 				}
 			}
 
+			clusterRoleKey := client.ObjectKey{Name: "nvidia-dcgm-exporter-read-pods"}
+			clusterRole := &rbacv1.ClusterRole{}
+			clusterRoleGetErr := clusterPolicyController.client.Get(context.Background(), clusterRoleKey, clusterRole)
+			clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
+			clusterRoleBindingGetErr := clusterPolicyController.client.Get(context.Background(), clusterRoleKey, clusterRoleBinding)
+			if tc.output["clusterRoleExists"].(bool) {
+				require.NoError(t, clusterRoleGetErr, "ClusterRole should exist when pod metadata enrichment is enabled")
+				require.NoError(t, clusterRoleBindingGetErr, "ClusterRoleBinding should exist when pod metadata enrichment is enabled")
+				require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be set when pod metadata enrichment is enabled")
+				require.True(t, *ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be true when pod metadata enrichment is enabled")
+			} else {
+				require.True(t, apierrors.IsNotFound(clusterRoleGetErr), "ClusterRole should not exist when pod metadata enrichment is disabled (got err=%v)", clusterRoleGetErr)
+				require.True(t, apierrors.IsNotFound(clusterRoleBindingGetErr), "ClusterRoleBinding should not exist when pod metadata enrichment is disabled (got err=%v)", clusterRoleBindingGetErr)
+				require.NotNil(t, ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be explicitly set false when pod metadata enrichment is disabled")
+				require.False(t, *ds.Spec.Template.Spec.AutomountServiceAccountToken, "AutomountServiceAccountToken should be false when pod metadata enrichment is disabled")
+			}
+
 			require.Equal(t, tc.output["dcgmExporterImage"], dcgmExporterImage, "Unexpected configuration for dcgm-exporter image")
 
 			// cleanup by deleting all kubernetes objects