Add scrapeTimeout to DCGM Exporter ServiceMonitor

Expose scrapeTimeout on DCGMExporterServiceMonitorConfig in the
ClusterPolicy CR and propagate it to the rendered ServiceMonitor's
endpoint. The field is also surfaced in the gpu-operator Helm values
under dcgmExporter.serviceMonitor. Empty string is the unset sentinel;
Prometheus' global default applies otherwise.

Signed-off-by: Rajath Agasthya <ragasthya@nvidia.com>

Author: rajathagasthya

api/nvidia/v1/clusterpolicy_types.go

@@ -1082,6 +1082,14 @@ type DCGMExporterServiceMonitorConfig struct {
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:text"
 	Interval promv1.Duration `json:"interval,omitempty"`
 
+	// ScrapeTimeout to use when scraping metrics from NVIDIA DCGM Exporter. Must not be greater than Interval.
+	// If not specified, Prometheus' global scrape timeout is used.
+	// Supported units: y, w, d, h, m, s, ms
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Scrape timeout for NVIDIA DCGM Exporter"
+	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:text"
+	ScrapeTimeout promv1.Duration `json:"scrapeTimeout,omitempty"`
+
 	// HonorLabels chooses the metric’s labels on collisions with target labels.
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
 	// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Choose the metric's label on collisions with target labels"

bundle/manifests/nvidia.com_clusterpolicies.yaml

@@ -785,6 +785,13 @@ spec:
                               type: string
                           type: object
                         type: array
+                      scrapeTimeout:
+                        description: |-
+                          ScrapeTimeout to use when scraping metrics from NVIDIA DCGM Exporter. Must not be greater than Interval.
+                          If not specified, Prometheus' global scrape timeout is used.
+                          Supported units: y, w, d, h, m, s, ms
+                        pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
+                        type: string
                     type: object
                   version:
                     description: NVIDIA DCGM Exporter image tag

config/crd/bases/nvidia.com_clusterpolicies.yaml

@@ -785,6 +785,13 @@ spec:
                               type: string
                           type: object
                         type: array
+                      scrapeTimeout:
+                        description: |-
+                          ScrapeTimeout to use when scraping metrics from NVIDIA DCGM Exporter. Must not be greater than Interval.
+                          If not specified, Prometheus' global scrape timeout is used.
+                          Supported units: y, w, d, h, m, s, ms
+                        pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
+                        type: string
                     type: object
                   version:
                     description: NVIDIA DCGM Exporter image tag

controllers/object_controls.go

@@ -4963,6 +4963,10 @@ func ServiceMonitor(n ClusterPolicyController) (gpuv1.State, error) {
 			obj.Spec.Endpoints[0].Interval = serviceMonitor.Interval
 		}
 
+		if serviceMonitor.ScrapeTimeout != "" {
+			obj.Spec.Endpoints[0].ScrapeTimeout = serviceMonitor.ScrapeTimeout
+		}
+
 		if serviceMonitor.HonorLabels != nil {
 			obj.Spec.Endpoints[0].HonorLabels = *serviceMonitor.HonorLabels
 		}

controllers/object_controls_test.go

@@ -1378,6 +1378,7 @@ func TestServiceMonitor(t *testing.T) {
 					ServiceMonitor: &gpuv1.DCGMExporterServiceMonitorConfig{
 						Enabled:          ptr.To(true),
 						Interval:         promv1.Duration("15s"),
+						ScrapeTimeout:    promv1.Duration("10s"),
 						HonorLabels:      ptr.To(true),
 						AdditionalLabels: map[string]string{"a": "b"},
 						Relabelings:      []*promv1.RelabelConfig{{Action: "keep"}},
@@ -1394,8 +1395,9 @@ func TestServiceMonitor(t *testing.T) {
 				Spec: promv1.ServiceMonitorSpec{
 					NamespaceSelector: promv1.NamespaceSelector{MatchNames: []string{"test-namespace"}},
 					Endpoints: []promv1.Endpoint{{
-						Interval:    promv1.Duration("15s"),
-						HonorLabels: true,
+						Interval:      promv1.Duration("15s"),
+						ScrapeTimeout: promv1.Duration("10s"),
+						HonorLabels:   true,
 						RelabelConfigs: []promv1.RelabelConfig{{
 							Action: "keep",
 						}},

deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml

@@ -785,6 +785,13 @@ spec:
                               type: string
                           type: object
                         type: array
+                      scrapeTimeout:
+                        description: |-
+                          ScrapeTimeout to use when scraping metrics from NVIDIA DCGM Exporter. Must not be greater than Interval.
+                          If not specified, Prometheus' global scrape timeout is used.
+                          Supported units: y, w, d, h, m, s, ms
+                        pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
+                        type: string
                     type: object
                   version:
                     description: NVIDIA DCGM Exporter image tag

deployments/gpu-operator/values.yaml

@@ -302,6 +302,7 @@ dcgmExporter:
   serviceMonitor:
     enabled: false
     interval: 15s
+    scrapeTimeout: ""
     honorLabels: false
     additionalLabels: {}
     relabelings: []