test(prometheus.exporter.cadvisor): Add integration test

integration-tests/k8s/deps/mimir.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"strconv"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -36,6 +37,21 @@ type metadataResponse struct {
 	Data   map[string][]ExpectedMetadata `json:"data"`
 }
 
+type instantQueryResponse struct {
+	Status string `json:"status"`
+	Data   struct {
+		Result []struct {
+			Metric map[string]string `json:"metric"`
+			Value  []any             `json:"value"`
+		} `json:"result"`
+	} `json:"data"`
+}
+
+type seriesResponse struct {
+	Status string              `json:"status"`
+	Data   []map[string]string `json:"data"`
+}
+
 // ExpectedMetadata is both the JSON payload shape from Mimir's metadata
 // endpoint and the input to QueryMetadata. Empty fields are not asserted.
 type ExpectedMetadata struct {
@@ -136,6 +152,72 @@ func (m *Mimir) QueryMetrics(t *testing.T, testName string, expectedMetrics []st
 	}, timeout, retryInterval)
 }
 
+// QueryPositive polls an instant query for each metric (scoped to testName) and
+// asserts at least one returned sample is greater than zero. It checks a metric
+// is not just present but reports real data.
+func (m *Mimir) QueryPositive(t *testing.T, testName string, metrics []string) {
+	t.Helper()
+	base := m.endpoint("/prometheus/api/v1/query")
+
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		for _, metric := range metrics {
+			queryURL, err := url.Parse(base)
+			require.NoError(c, err)
+			values := queryURL.Query()
+			values.Set("query", metric+"{"+testNameLabel+"=\""+testName+"\"}")
+			queryURL.RawQuery = values.Encode()
+			resp := curl(c, queryURL.String(), nil)
+
+			var parsed instantQueryResponse
+			require.NoError(c, json.Unmarshal([]byte(resp), &parsed), "failed to parse query response: %s", resp)
+			require.Equal(c, "success", parsed.Status, "mimir query failed: %s", resp)
+			require.NotEmptyf(c, parsed.Data.Result, "%s: no samples for %s=%s", metric, testNameLabel, testName)
+
+			var maxValue float64
+			for _, result := range parsed.Data.Result {
+				if len(result.Value) != 2 {
+					continue
+				}
+				sample, ok := result.Value[1].(string)
+				if !ok {
+					continue
+				}
+				value, convErr := strconv.ParseFloat(sample, 64)
+				if convErr == nil && value > maxValue {
+					maxValue = value
+				}
+			}
+			require.Greaterf(c, maxValue, 0.0, "%s: expected a positive sample", metric)
+		}
+	}, timeout, retryInterval)
+}
+
+// QueryLabelsPresent asserts at least one series for testName carries every
+// given label with a non-empty value. Passing multiple labels requires them on
+// the same series. It checks cAdvisor attaches container labels.
+func (m *Mimir) QueryLabelsPresent(t *testing.T, testName string, labelNames ...string) {
+	t.Helper()
+
+	matchers := testNameLabel + "=\"" + testName + "\""
+	for _, labelName := range labelNames {
+		matchers += "," + labelName + "=~\".+\""
+	}
+
+	require.EventuallyWithT(t, func(c *assert.CollectT) {
+		queryURL, err := url.Parse(m.endpoint("/prometheus/api/v1/series"))
+		require.NoError(c, err)
+		values := queryURL.Query()
+		values.Add("match[]", "{"+matchers+"}")
+		queryURL.RawQuery = values.Encode()
+		resp := curl(c, queryURL.String(), nil)
+
+		var parsed seriesResponse
+		require.NoError(c, json.Unmarshal([]byte(resp), &parsed), "failed to parse series response: %s", resp)
+		require.Equal(c, "success", parsed.Status, "mimir series query failed: %s", resp)
+		require.NotEmptyf(c, parsed.Data, "no series carrying labels %v for %s=%s", labelNames, testNameLabel, testName)
+	}, timeout, retryInterval)
+}
+
 // QueryMetadata asserts each expected metric appears in Mimir's
 // /api/v1/metadata with the requested Type/Help/Unit.
 func (m *Mimir) QueryMetadata(t *testing.T, expected map[string]ExpectedMetadata) {

integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml

@@ -0,0 +1,42 @@
+# The containerd socket lets cAdvisor's containerd plugin resolve pod metadata
+# and attach container_label_* labels. Only the socket is mounted, not all of
+# /var/run: a read-only /var/run blocks Kubernetes from mounting the service
+# account token, so the pod would fail to start.
+controller:
+  type: daemonset
+  volumes:
+    extra:
+      - name: rootfs
+        hostPath:
+          path: /
+      - name: sys
+        hostPath:
+          path: /sys
+      - name: disk
+        hostPath:
+          path: /dev/disk
+      - name: containerd-sock
+        hostPath:
+          path: /run/containerd/containerd.sock
+          type: Socket
+
+alloy:
+  stabilityLevel: experimental
+  initialDelaySeconds: 1
+  # The raw cgroup driver needs a privileged container to read /sys/fs/cgroup.
+  securityContext:
+    privileged: true
+  mounts:
+    extra:
+      - name: rootfs
+        mountPath: /rootfs
+        readOnly: true
+      - name: sys
+        mountPath: /sys
+        readOnly: true
+      - name: disk
+        mountPath: /dev/disk
+        readOnly: true
+      - name: containerd-sock
+        mountPath: /run/containerd/containerd.sock
+        readOnly: true

integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/config.alloy

@@ -0,0 +1,33 @@
+logging {
+  level = "debug"
+}
+
+prometheus.exporter.cadvisor "test" {
+  storage_duration = "1m"
+}
+
+prometheus.scrape "cadvisor" {
+  targets         = prometheus.exporter.cadvisor.test.targets
+  forward_to      = [prometheus.relabel.cadvisor.receiver]
+  scrape_interval = "1s"
+  scrape_timeout  = "500ms"
+}
+
+prometheus.relabel "cadvisor" {
+  forward_to = [prometheus.remote_write.mimir.receiver]
+  // The k8s harness filters series on alloy_test_name.
+  rule {
+    action       = "replace"
+    target_label = "alloy_test_name"
+    replacement  = "cadvisor"
+  }
+}
+
+prometheus.remote_write "mimir" {
+  endpoint {
+    url = "http://mimir:9009/api/v1/push"
+    queue_config {
+      max_samples_per_send = 100
+    }
+  }
+}

integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go

@@ -0,0 +1,99 @@
+package prometheusexportercadvisor
+
+import (
+	"testing"
+
+	"github.com/grafana/alloy/integration-tests/k8s/deps"
+	"github.com/grafana/alloy/integration-tests/k8s/harness"
+)
+
+func TestPrometheusExporterCadvisor(t *testing.T) {
+	ns := deps.NewNamespace(deps.NamespaceOptions{
+		Name:   "test-prometheus-exporter-cadvisor",
+		Labels: map[string]string{"alloy-integration-test": "true"},
+	})
+	mimir := deps.NewMimir(deps.MimirOptions{Namespace: ns.Name()})
+	alloy := deps.NewAlloy(deps.AlloyOptions{
+		Namespace:  ns.Name(),
+		Release:    "alloy-test-prometheus-exporter-cadvisor",
+		ConfigPath: "./config/config.alloy",
+		ValuesPath: "./config/alloy-values.yaml",
+	})
+	harness.Setup(t, harness.Options{
+		Dependencies: []harness.Dependency{ns, mimir, alloy},
+	})
+
+	// Covers the cAdvisor collector families: version, cpu, memory, filesystem,
+	// network, and blkio.
+	//
+	// Two families are left out on purpose:
+	//   - container_pressure_* (PSI) needs kernel CONFIG_PSI. Not every host has it.
+	//   - container_health_state needs a container with a Docker HEALTHCHECK. It
+	//     depends on the sibling workloads, not the exporter.
+	mimir.QueryMetrics(t, "cadvisor", []string{
+		"cadvisor_build_info",
+		"cadvisor_version_info",
+		"container_blkio_device_usage_total",
+		"container_cpu_load_average_10s",
+		"container_cpu_load_d_average_10s",
+		"container_cpu_system_seconds_total",
+		"container_cpu_usage_seconds_total",
+		"container_cpu_user_seconds_total",
+		"container_fs_inodes_free",
+		"container_fs_inodes_total",
+		"container_fs_io_current",
+		"container_fs_io_time_seconds_total",
+		"container_fs_io_time_weighted_seconds_total",
+		"container_fs_limit_bytes",
+		"container_fs_read_seconds_total",
+		"container_fs_reads_bytes_total",
+		"container_fs_reads_merged_total",
+		"container_fs_reads_total",
+		"container_fs_sector_reads_total",
+		"container_fs_sector_writes_total",
+		"container_fs_usage_bytes",
+		"container_fs_write_seconds_total",
+		"container_fs_writes_bytes_total",
+		"container_fs_writes_merged_total",
+		"container_fs_writes_total",
+		"container_last_seen",
+		"container_memory_cache",
+		"container_memory_failcnt",
+		"container_memory_failures_total",
+		"container_memory_kernel_usage",
+		"container_memory_mapped_file",
+		"container_memory_max_usage_bytes",
+		"container_memory_rss",
+		"container_memory_swap",
+		"container_memory_total_active_file_bytes",
+		"container_memory_total_inactive_file_bytes",
+		"container_memory_usage_bytes",
+		"container_memory_working_set_bytes",
+		"container_network_receive_bytes_total",
+		"container_network_receive_errors_total",
+		"container_network_receive_packets_dropped_total",
+		"container_network_receive_packets_total",
+		"container_network_transmit_bytes_total",
+		"container_network_transmit_errors_total",
+		"container_network_transmit_packets_dropped_total",
+		"container_network_transmit_packets_total",
+		"container_oom_events_total",
+	})
+
+	// The filesystem metrics must report real data, not just be present. These
+	// come from the root cgroup's machine filesystem, so they are always > 0 and
+	// prove the explicit filesystem-plugin wiring produces values.
+	mimir.QueryPositive(t, "cadvisor", []string{
+		"container_fs_usage_bytes",
+		"container_fs_limit_bytes",
+	})
+
+	// The containerd socket lets cAdvisor resolve pod metadata, so container
+	// labels are attached. Kubernetes sets these io.kubernetes.* labels on every
+	// container, so a single series carries all three when the plugin works.
+	mimir.QueryLabelsPresent(t, "cadvisor",
+		"container_label_io_kubernetes_pod_name",
+		"container_label_io_kubernetes_pod_namespace",
+		"container_label_io_kubernetes_container_name",
+	)
+}