From 8333c0bcb1c23aeabe2a064b8f02d9f1d0f069ad Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Fri, 4 Sep 2026 16:26:45 -0400 Subject: [PATCH 1/9] test(prometheus.exporter.cadvisor): Add docker integration test (POC) Add a docker integration test for the cAdvisor exporter. The test scrapes a single prometheus.exporter.cadvisor instance into Mimir and asserts that cAdvisor metrics appear. cAdvisor uses its raw cgroup driver, so the test needs a privileged container but no container runtime. This POC includes a temporary discovery test. It lists every cAdvisor metric that reaches Mimir so the assertion list can be pinned from a real Linux run. The discovery test fails on purpose to surface its output. Co-Authored-By: Claude Opus 4.8 --- .../tests/cadvisor/cadvisor_metrics_test.go | 82 +++++++++++++++++++ .../docker/tests/cadvisor/config.alloy | 25 ++++++ .../docker/tests/cadvisor/test.yaml | 5 ++ 3 files changed, 112 insertions(+) create mode 100644 integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go create mode 100644 integration-tests/docker/tests/cadvisor/config.alloy create mode 100644 integration-tests/docker/tests/cadvisor/test.yaml diff --git a/integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go b/integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go new file mode 100644 index 00000000000..c189581277f --- /dev/null +++ b/integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go @@ -0,0 +1,82 @@ +//go:build alloyintegrationtests + +package main + +import ( + "fmt" + "runtime" + "sort" + "strings" + "testing" + + "github.com/grafana/alloy/integration-tests/docker/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// cadvisorPrefixes are the metric name prefixes cAdvisor's collectors emit. +var cadvisorPrefixes = []string{"machine_", "container_", "cadvisor_"} + +// TestCadvisorMetrics asserts a stable core of cAdvisor metrics reach Mimir. +// +// The assertion list is not pinned yet. It is skipped until a real run on Linux +// tells us which metrics land reliably. Use TestCadvisorDiscoverMetrics to get +// that list, then fill it in here and remove the skip and the discovery test. +func TestCadvisorMetrics(t *testing.T) { + t.Skip("TEMPORARY: pin expectedMetrics from TestCadvisorDiscoverMetrics output, then remove this skip") + + if runtime.GOOS != "linux" { + t.Skip("Skipping cAdvisor metrics test on non-Linux platform") + } + + expectedMetrics := []string{ + // Filled in from a real run. + } + + common.MimirMetricsTest(t, expectedMetrics, []string{}, "cadvisor_metrics") +} + +// TestCadvisorDiscoverMetrics is a TEMPORARY POC helper. It lists every +// cAdvisor-prefixed metric that reached Mimir, sorted and formatted as a +// paste-ready Go slice. It fails on purpose. The harness prints test output +// only for failing tests, so failing is how we surface the list. +// +// Remove this test once TestCadvisorMetrics has a real assertion list. +func TestCadvisorDiscoverMetrics(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("Skipping cAdvisor metrics discovery on non-Linux platform") + } + common.AssertStatefulTestEnv(t) + + var found []string + require.EventuallyWithT(t, func(c *assert.CollectT) { + var resp common.MetricsResponse + _, err := common.FetchDataFromURL(common.MetricsQuery("cadvisor_metrics"), &resp) + assert.NoError(c, err) + + names := map[string]struct{}{} + for _, m := range resp.Data { + for _, p := range cadvisorPrefixes { + if strings.HasPrefix(m.Name, p) { + names[m.Name] = struct{}{} + break + } + } + } + + found = found[:0] + for name := range names { + found = append(found, name) + } + // Wait for a reasonable set before reporting, so the list is not cut short. + assert.GreaterOrEqual(c, len(found), 5, "waiting for cAdvisor metrics to appear") + }, common.TestTimeoutEnv(t), common.DefaultRetryInterval) + + sort.Strings(found) + + var b strings.Builder + for _, name := range found { + fmt.Fprintf(&b, "\t\t%q,\n", name) + } + t.Errorf("TEMPORARY cAdvisor metric discovery — %d metrics found. Copy into TestCadvisorMetrics:\n%s", len(found), b.String()) +} diff --git a/integration-tests/docker/tests/cadvisor/config.alloy b/integration-tests/docker/tests/cadvisor/config.alloy new file mode 100644 index 00000000000..87d8d2038cd --- /dev/null +++ b/integration-tests/docker/tests/cadvisor/config.alloy @@ -0,0 +1,25 @@ +// No container runtime sockets are configured. cAdvisor uses its raw cgroup +// driver, which reads /sys/fs/cgroup. This needs a privileged container but no +// docker or containerd runtime. See test.yaml. +prometheus.exporter.cadvisor "example" { + storage_duration = "1m" +} + +prometheus.scrape "cadvisor" { + targets = prometheus.exporter.cadvisor.example.targets + forward_to = [prometheus.remote_write.default.receiver] + scrape_interval = "1s" + scrape_timeout = "500ms" +} + +prometheus.remote_write "default" { + endpoint { + url = "http://mimir:9009/api/v1/push" + queue_config { + max_samples_per_send = 100 + } + } + external_labels = { + test_name = "cadvisor_metrics", + } +} diff --git a/integration-tests/docker/tests/cadvisor/test.yaml b/integration-tests/docker/tests/cadvisor/test.yaml new file mode 100644 index 00000000000..498a3bf2eee --- /dev/null +++ b/integration-tests/docker/tests/cadvisor/test.yaml @@ -0,0 +1,5 @@ +# cAdvisor reads /sys/fs/cgroup through its raw driver. A privileged container +# gives it access. The harness already runs Alloy privileged, but we set it here +# so the requirement is visible at the test. +alloy_container: + privileged: true From 06a31c27c34bee632eb6dd2a8dc4eb38b1a7dd88 Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Fri, 4 Sep 2026 18:11:55 -0400 Subject: [PATCH 2/9] test(prometheus.exporter.cadvisor): Pin metric assertion list Replace the temporary discovery test with the real assertion list, pinned from a CI run. The list covers every cAdvisor collector family: build and version, cpu, memory, filesystem, network, and blkio. Leave out container_pressure_* (needs kernel CONFIG_PSI) and container_health_state (needs a Docker HEALTHCHECK on the target container), since neither is guaranteed on every host. Co-Authored-By: Claude Opus 4.8 --- .../tests/cadvisor/cadvisor_metrics_test.go | 118 +++++++++--------- 1 file changed, 57 insertions(+), 61 deletions(-) diff --git a/integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go b/integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go index c189581277f..982ea449431 100644 --- a/integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go +++ b/integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go @@ -3,80 +3,76 @@ package main import ( - "fmt" "runtime" - "sort" - "strings" "testing" "github.com/grafana/alloy/integration-tests/docker/common" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -// cadvisorPrefixes are the metric name prefixes cAdvisor's collectors emit. -var cadvisorPrefixes = []string{"machine_", "container_", "cadvisor_"} - -// TestCadvisorMetrics asserts a stable core of cAdvisor metrics reach Mimir. -// -// The assertion list is not pinned yet. It is skipped until a real run on Linux -// tells us which metrics land reliably. Use TestCadvisorDiscoverMetrics to get -// that list, then fill it in here and remove the skip and the discovery test. func TestCadvisorMetrics(t *testing.T) { - t.Skip("TEMPORARY: pin expectedMetrics from TestCadvisorDiscoverMetrics output, then remove this skip") - + // cAdvisor only runs on Linux. The test exercises the Grafana cAdvisor fork's + // collectors, so it must run against a real Linux cgroup tree. if runtime.GOOS != "linux" { t.Skip("Skipping cAdvisor metrics test on non-Linux platform") } + // Pinned from a real CI run. This covers every cAdvisor collector family: + // build/version, cpu, memory, filesystem, network, and blkio. + // + // Two families are left out on purpose: + // - container_pressure_* (PSI) needs kernel CONFIG_PSI and is not present + // on every host. + // - container_health_state is emitted only for containers with a Docker + // HEALTHCHECK, so it depends on the sibling images, not the exporter. expectedMetrics := []string{ - // Filled in from a real run. + "cadvisor_build_info", + "cadvisor_version_info", + "container_blkio_device_usage_total", + "container_cpu_load_average_10s", + "container_cpu_load_d_average_10s", + "container_cpu_system_seconds_total", + "container_cpu_usage_seconds_total", + "container_cpu_user_seconds_total", + "container_fs_inodes_free", + "container_fs_inodes_total", + "container_fs_io_current", + "container_fs_io_time_seconds_total", + "container_fs_io_time_weighted_seconds_total", + "container_fs_limit_bytes", + "container_fs_read_seconds_total", + "container_fs_reads_bytes_total", + "container_fs_reads_merged_total", + "container_fs_reads_total", + "container_fs_sector_reads_total", + "container_fs_sector_writes_total", + "container_fs_usage_bytes", + "container_fs_write_seconds_total", + "container_fs_writes_bytes_total", + "container_fs_writes_merged_total", + "container_fs_writes_total", + "container_last_seen", + "container_memory_cache", + "container_memory_failcnt", + "container_memory_failures_total", + "container_memory_kernel_usage", + "container_memory_mapped_file", + "container_memory_max_usage_bytes", + "container_memory_rss", + "container_memory_swap", + "container_memory_total_active_file_bytes", + "container_memory_total_inactive_file_bytes", + "container_memory_usage_bytes", + "container_memory_working_set_bytes", + "container_network_receive_bytes_total", + "container_network_receive_errors_total", + "container_network_receive_packets_dropped_total", + "container_network_receive_packets_total", + "container_network_transmit_bytes_total", + "container_network_transmit_errors_total", + "container_network_transmit_packets_dropped_total", + "container_network_transmit_packets_total", + "container_oom_events_total", } common.MimirMetricsTest(t, expectedMetrics, []string{}, "cadvisor_metrics") } - -// TestCadvisorDiscoverMetrics is a TEMPORARY POC helper. It lists every -// cAdvisor-prefixed metric that reached Mimir, sorted and formatted as a -// paste-ready Go slice. It fails on purpose. The harness prints test output -// only for failing tests, so failing is how we surface the list. -// -// Remove this test once TestCadvisorMetrics has a real assertion list. -func TestCadvisorDiscoverMetrics(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("Skipping cAdvisor metrics discovery on non-Linux platform") - } - common.AssertStatefulTestEnv(t) - - var found []string - require.EventuallyWithT(t, func(c *assert.CollectT) { - var resp common.MetricsResponse - _, err := common.FetchDataFromURL(common.MetricsQuery("cadvisor_metrics"), &resp) - assert.NoError(c, err) - - names := map[string]struct{}{} - for _, m := range resp.Data { - for _, p := range cadvisorPrefixes { - if strings.HasPrefix(m.Name, p) { - names[m.Name] = struct{}{} - break - } - } - } - - found = found[:0] - for name := range names { - found = append(found, name) - } - // Wait for a reasonable set before reporting, so the list is not cut short. - assert.GreaterOrEqual(c, len(found), 5, "waiting for cAdvisor metrics to appear") - }, common.TestTimeoutEnv(t), common.DefaultRetryInterval) - - sort.Strings(found) - - var b strings.Builder - for _, name := range found { - fmt.Fprintf(&b, "\t\t%q,\n", name) - } - t.Errorf("TEMPORARY cAdvisor metric discovery — %d metrics found. Copy into TestCadvisorMetrics:\n%s", len(found), b.String()) -} From bdc7ff9ad5765c5ff0e6152b541039a153a03b80 Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Wed, 9 Sep 2026 14:48:11 -0400 Subject: [PATCH 3/9] test(prometheus.exporter.cadvisor): Move integration test to k8s harness. --- .../docker/tests/cadvisor/config.alloy | 25 ---------- .../docker/tests/cadvisor/test.yaml | 5 -- .../config/alloy-values.yaml | 40 ++++++++++++++++ .../config/config.alloy | 33 +++++++++++++ .../prometheus-exporter-cadvisor/k8s_test.go} | 47 ++++++++++--------- 5 files changed, 99 insertions(+), 51 deletions(-) delete mode 100644 integration-tests/docker/tests/cadvisor/config.alloy delete mode 100644 integration-tests/docker/tests/cadvisor/test.yaml create mode 100644 integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml create mode 100644 integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/config.alloy rename integration-tests/{docker/tests/cadvisor/cadvisor_metrics_test.go => k8s/tests/prometheus-exporter-cadvisor/k8s_test.go} (60%) diff --git a/integration-tests/docker/tests/cadvisor/config.alloy b/integration-tests/docker/tests/cadvisor/config.alloy deleted file mode 100644 index 87d8d2038cd..00000000000 --- a/integration-tests/docker/tests/cadvisor/config.alloy +++ /dev/null @@ -1,25 +0,0 @@ -// No container runtime sockets are configured. cAdvisor uses its raw cgroup -// driver, which reads /sys/fs/cgroup. This needs a privileged container but no -// docker or containerd runtime. See test.yaml. -prometheus.exporter.cadvisor "example" { - storage_duration = "1m" -} - -prometheus.scrape "cadvisor" { - targets = prometheus.exporter.cadvisor.example.targets - forward_to = [prometheus.remote_write.default.receiver] - scrape_interval = "1s" - scrape_timeout = "500ms" -} - -prometheus.remote_write "default" { - endpoint { - url = "http://mimir:9009/api/v1/push" - queue_config { - max_samples_per_send = 100 - } - } - external_labels = { - test_name = "cadvisor_metrics", - } -} diff --git a/integration-tests/docker/tests/cadvisor/test.yaml b/integration-tests/docker/tests/cadvisor/test.yaml deleted file mode 100644 index 498a3bf2eee..00000000000 --- a/integration-tests/docker/tests/cadvisor/test.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# cAdvisor reads /sys/fs/cgroup through its raw driver. A privileged container -# gives it access. The harness already runs Alloy privileged, but we set it here -# so the requirement is visible at the test. -alloy_container: - privileged: true diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml new file mode 100644 index 00000000000..a6e7b91ec9e --- /dev/null +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml @@ -0,0 +1,40 @@ +# cAdvisor reads the node's cgroup tree and machine stats. The Alloy pod runs +# privileged and mounts the host paths read-only. This mirrors the canonical +# cAdvisor DaemonSet. +controller: + type: daemonset + volumes: + extra: + - name: rootfs + hostPath: + path: / + - name: var-run + hostPath: + path: /var/run + - name: sys + hostPath: + path: /sys + - name: disk + hostPath: + path: /dev/disk + +alloy: + stabilityLevel: experimental + initialDelaySeconds: 1 + # The raw cgroup driver needs a privileged container to read /sys/fs/cgroup. + securityContext: + privileged: true + mounts: + extra: + - name: rootfs + mountPath: /rootfs + readOnly: true + - name: var-run + mountPath: /var/run + readOnly: true + - name: sys + mountPath: /sys + readOnly: true + - name: disk + mountPath: /dev/disk + readOnly: true diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/config.alloy b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/config.alloy new file mode 100644 index 00000000000..61780d03679 --- /dev/null +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/config.alloy @@ -0,0 +1,33 @@ +logging { + level = "debug" +} + +prometheus.exporter.cadvisor "test" { + storage_duration = "1m" +} + +prometheus.scrape "cadvisor" { + targets = prometheus.exporter.cadvisor.test.targets + forward_to = [prometheus.relabel.cadvisor.receiver] + scrape_interval = "1s" + scrape_timeout = "500ms" +} + +prometheus.relabel "cadvisor" { + forward_to = [prometheus.remote_write.mimir.receiver] + // The k8s harness filters series on alloy_test_name. + rule { + action = "replace" + target_label = "alloy_test_name" + replacement = "cadvisor" + } +} + +prometheus.remote_write "mimir" { + endpoint { + url = "http://mimir:9009/api/v1/push" + queue_config { + max_samples_per_send = 100 + } + } +} diff --git a/integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go similarity index 60% rename from integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go rename to integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go index 982ea449431..9c329c81134 100644 --- a/integration-tests/docker/tests/cadvisor/cadvisor_metrics_test.go +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go @@ -1,30 +1,37 @@ -//go:build alloyintegrationtests - -package main +package prometheusexportercadvisor import ( - "runtime" "testing" - "github.com/grafana/alloy/integration-tests/docker/common" + "github.com/grafana/alloy/integration-tests/k8s/deps" + "github.com/grafana/alloy/integration-tests/k8s/harness" ) -func TestCadvisorMetrics(t *testing.T) { - // cAdvisor only runs on Linux. The test exercises the Grafana cAdvisor fork's - // collectors, so it must run against a real Linux cgroup tree. - if runtime.GOOS != "linux" { - t.Skip("Skipping cAdvisor metrics test on non-Linux platform") - } +func TestPrometheusExporterCadvisor(t *testing.T) { + ns := deps.NewNamespace(deps.NamespaceOptions{ + Name: "test-prometheus-exporter-cadvisor", + Labels: map[string]string{"alloy-integration-test": "true"}, + }) + mimir := deps.NewMimir(deps.MimirOptions{Namespace: ns.Name()}) + alloy := deps.NewAlloy(deps.AlloyOptions{ + Namespace: ns.Name(), + Release: "alloy-test-prometheus-exporter-cadvisor", + ConfigPath: "./config/config.alloy", + ValuesPath: "./config/alloy-values.yaml", + }) + harness.Setup(t, harness.Options{ + Dependencies: []harness.Dependency{ns, mimir, alloy}, + }) - // Pinned from a real CI run. This covers every cAdvisor collector family: - // build/version, cpu, memory, filesystem, network, and blkio. + // Covers the cAdvisor collector families: version, cpu, memory, filesystem, + // network, and blkio. cadvisor_build_info comes from the Alloy integration + // wrapper, not from cAdvisor. // // Two families are left out on purpose: - // - container_pressure_* (PSI) needs kernel CONFIG_PSI and is not present - // on every host. - // - container_health_state is emitted only for containers with a Docker - // HEALTHCHECK, so it depends on the sibling images, not the exporter. - expectedMetrics := []string{ + // - container_pressure_* (PSI) needs kernel CONFIG_PSI. Not every host has it. + // - container_health_state needs a container with a Docker HEALTHCHECK. It + // depends on the sibling workloads, not the exporter. + mimir.QueryMetrics(t, "cadvisor", []string{ "cadvisor_build_info", "cadvisor_version_info", "container_blkio_device_usage_total", @@ -72,7 +79,5 @@ func TestCadvisorMetrics(t *testing.T) { "container_network_transmit_packets_dropped_total", "container_network_transmit_packets_total", "container_oom_events_total", - } - - common.MimirMetricsTest(t, expectedMetrics, []string{}, "cadvisor_metrics") + }) } From 8e6e232777417d6a972a3814cb2bec275565e8a7 Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Wed, 9 Sep 2026 14:58:45 -0400 Subject: [PATCH 4/9] remove unneeded comment --- .../k8s/tests/prometheus-exporter-cadvisor/k8s_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go index 9c329c81134..ef52b750c98 100644 --- a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go @@ -24,8 +24,7 @@ func TestPrometheusExporterCadvisor(t *testing.T) { }) // Covers the cAdvisor collector families: version, cpu, memory, filesystem, - // network, and blkio. cadvisor_build_info comes from the Alloy integration - // wrapper, not from cAdvisor. + // network, and blkio. // // Two families are left out on purpose: // - container_pressure_* (PSI) needs kernel CONFIG_PSI. Not every host has it. From f9733e85c954a57e3a681fcfeeb237db4c400521 Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Wed, 9 Sep 2026 15:20:29 -0400 Subject: [PATCH 5/9] fix(prometheus.exporter.cadvisor): Drop read-only /var/run mount that blocked pod start A read-only host /var/run mount stops Kubernetes from mounting the service account token, so the Alloy pod failed with RunContainerError. The raw cgroup driver does not need /var/run. Only the docker and containerd plugins use it, and this test does not use them. Co-Authored-By: Claude Opus 4.8 --- .../config/alloy-values.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml index a6e7b91ec9e..14f725b235b 100644 --- a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml @@ -1,6 +1,9 @@ # cAdvisor reads the node's cgroup tree and machine stats. The Alloy pod runs -# privileged and mounts the host paths read-only. This mirrors the canonical -# cAdvisor DaemonSet. +# privileged and mounts the host paths read-only. This follows the canonical +# cAdvisor DaemonSet, but does not mount /var/run. A read-only /var/run blocks +# Kubernetes from mounting the service account token and the pod fails to start. +# The raw driver does not need /var/run; only the docker and containerd plugins +# use it, and this test does not use them. controller: type: daemonset volumes: @@ -8,9 +11,6 @@ controller: - name: rootfs hostPath: path: / - - name: var-run - hostPath: - path: /var/run - name: sys hostPath: path: /sys @@ -29,9 +29,6 @@ alloy: - name: rootfs mountPath: /rootfs readOnly: true - - name: var-run - mountPath: /var/run - readOnly: true - name: sys mountPath: /sys readOnly: true From a368a46207bdd6f3b5a5d1e5aa340628d75ecad1 Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Thu, 10 Sep 2026 16:33:45 -0400 Subject: [PATCH 6/9] test(prometheus.exporter.cadvisor): Check filesystem values and container labels Strengthen the k8s cAdvisor test beyond metric presence: - Assert container_fs_usage_bytes and container_fs_limit_bytes report values greater than zero, via a new Mimir.QueryPositive helper. - Mount the node containerd socket so the containerd plugin resolves pod metadata, and assert container_label_io_kubernetes_pod_name is attached, via a new Mimir.QueryLabelPresent helper. Docker-runtime-specific behaviours (container_health_state, application metrics) are not covered here: kind uses containerd, not Docker, so they cannot be exercised in this harness. Co-Authored-By: Claude Opus 4.8 --- integration-tests/k8s/deps/mimir.go | 76 +++++++++++++++++++ .../config/alloy-values.yaml | 19 +++-- .../prometheus-exporter-cadvisor/k8s_test.go | 13 ++++ 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/integration-tests/k8s/deps/mimir.go b/integration-tests/k8s/deps/mimir.go index 45c842e43c7..c439475c59d 100644 --- a/integration-tests/k8s/deps/mimir.go +++ b/integration-tests/k8s/deps/mimir.go @@ -6,6 +6,7 @@ import ( "fmt" "net/url" "os" + "strconv" "testing" "github.com/stretchr/testify/assert" @@ -36,6 +37,21 @@ type metadataResponse struct { Data map[string][]ExpectedMetadata `json:"data"` } +type instantQueryResponse struct { + Status string `json:"status"` + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Value []any `json:"value"` + } `json:"result"` + } `json:"data"` +} + +type seriesResponse struct { + Status string `json:"status"` + Data []map[string]string `json:"data"` +} + // ExpectedMetadata is both the JSON payload shape from Mimir's metadata // endpoint and the input to QueryMetadata. Empty fields are not asserted. type ExpectedMetadata struct { @@ -136,6 +152,66 @@ func (m *Mimir) QueryMetrics(t *testing.T, testName string, expectedMetrics []st }, timeout, retryInterval) } +// QueryPositive polls an instant query for each metric (scoped to testName) and +// asserts at least one returned sample is greater than zero. It checks a metric +// is not just present but reports real data. +func (m *Mimir) QueryPositive(t *testing.T, testName string, metrics []string) { + t.Helper() + base := m.endpoint("/prometheus/api/v1/query") + + require.EventuallyWithT(t, func(c *assert.CollectT) { + for _, metric := range metrics { + queryURL, err := url.Parse(base) + require.NoError(c, err) + values := queryURL.Query() + values.Set("query", metric+"{"+testNameLabel+"=\""+testName+"\"}") + queryURL.RawQuery = values.Encode() + resp := curl(c, queryURL.String(), nil) + + var parsed instantQueryResponse + require.NoError(c, json.Unmarshal([]byte(resp), &parsed), "failed to parse query response: %s", resp) + require.Equal(c, "success", parsed.Status, "mimir query failed: %s", resp) + require.NotEmptyf(c, parsed.Data.Result, "%s: no samples for %s=%s", metric, testNameLabel, testName) + + var maxValue float64 + for _, result := range parsed.Data.Result { + if len(result.Value) != 2 { + continue + } + sample, ok := result.Value[1].(string) + if !ok { + continue + } + value, convErr := strconv.ParseFloat(sample, 64) + if convErr == nil && value > maxValue { + maxValue = value + } + } + require.Greaterf(c, maxValue, 0.0, "%s: expected a positive sample", metric) + } + }, timeout, retryInterval) +} + +// QueryLabelPresent asserts at least one series for testName carries labelName +// with a non-empty value. It checks cAdvisor attaches container labels. +func (m *Mimir) QueryLabelPresent(t *testing.T, testName, labelName string) { + t.Helper() + + require.EventuallyWithT(t, func(c *assert.CollectT) { + queryURL, err := url.Parse(m.endpoint("/prometheus/api/v1/series")) + require.NoError(c, err) + values := queryURL.Query() + values.Add("match[]", "{"+testNameLabel+"=\""+testName+"\","+labelName+"=~\".+\"}") + queryURL.RawQuery = values.Encode() + resp := curl(c, queryURL.String(), nil) + + var parsed seriesResponse + require.NoError(c, json.Unmarshal([]byte(resp), &parsed), "failed to parse series response: %s", resp) + require.Equal(c, "success", parsed.Status, "mimir series query failed: %s", resp) + require.NotEmptyf(c, parsed.Data, "no series carrying label %s for %s=%s", labelName, testNameLabel, testName) + }, timeout, retryInterval) +} + // QueryMetadata asserts each expected metric appears in Mimir's // /api/v1/metadata with the requested Type/Help/Unit. func (m *Mimir) QueryMetadata(t *testing.T, expected map[string]ExpectedMetadata) { diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml index 14f725b235b..9a6decc2994 100644 --- a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml @@ -1,9 +1,11 @@ # cAdvisor reads the node's cgroup tree and machine stats. The Alloy pod runs -# privileged and mounts the host paths read-only. This follows the canonical -# cAdvisor DaemonSet, but does not mount /var/run. A read-only /var/run blocks -# Kubernetes from mounting the service account token and the pod fails to start. -# The raw driver does not need /var/run; only the docker and containerd plugins -# use it, and this test does not use them. +# privileged and mounts the host paths read-only, following the canonical +# cAdvisor DaemonSet. It also mounts the containerd socket so the containerd +# plugin resolves pod metadata and attaches container_label_* labels. +# +# It does not mount all of /var/run: a read-only /var/run blocks Kubernetes from +# mounting the service account token and the pod fails to start. Mounting only +# the containerd socket avoids that. controller: type: daemonset volumes: @@ -17,6 +19,10 @@ controller: - name: disk hostPath: path: /dev/disk + - name: containerd-sock + hostPath: + path: /run/containerd/containerd.sock + type: Socket alloy: stabilityLevel: experimental @@ -35,3 +41,6 @@ alloy: - name: disk mountPath: /dev/disk readOnly: true + # Not read-only: cAdvisor connects to the containerd socket. + - name: containerd-sock + mountPath: /run/containerd/containerd.sock diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go index ef52b750c98..b0c32aab269 100644 --- a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go @@ -79,4 +79,17 @@ func TestPrometheusExporterCadvisor(t *testing.T) { "container_network_transmit_packets_total", "container_oom_events_total", }) + + // The filesystem metrics must report real data, not just be present. These + // come from the root cgroup's machine filesystem, so they are always > 0 and + // prove the explicit filesystem-plugin wiring produces values. + mimir.QueryPositive(t, "cadvisor", []string{ + "container_fs_usage_bytes", + "container_fs_limit_bytes", + }) + + // The containerd socket lets cAdvisor resolve pod metadata, so container + // labels are attached. Kubernetes sets io.kubernetes.* labels on every + // container, so this label is always present when the containerd plugin works. + mimir.QueryLabelPresent(t, "cadvisor", "container_label_io_kubernetes_pod_name") } From a51e53aa43335a347d7ad8dce7627b05a311ae2b Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Mon, 14 Sep 2026 12:01:27 -0400 Subject: [PATCH 7/9] test(prometheus.exporter.cadvisor): Trim values comment, mount containerd socket read-only Address review feedback: drop the comment sentences that restate the config, keeping only the non-obvious containerd-socket and /var/run rationale, and mount the containerd socket read-only (cAdvisor only connects to it). Co-Authored-By: Claude Opus 4.8 --- .../config/alloy-values.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml index 9a6decc2994..72a1f5ac5b9 100644 --- a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml @@ -1,11 +1,7 @@ -# cAdvisor reads the node's cgroup tree and machine stats. The Alloy pod runs -# privileged and mounts the host paths read-only, following the canonical -# cAdvisor DaemonSet. It also mounts the containerd socket so the containerd -# plugin resolves pod metadata and attaches container_label_* labels. -# -# It does not mount all of /var/run: a read-only /var/run blocks Kubernetes from -# mounting the service account token and the pod fails to start. Mounting only -# the containerd socket avoids that. +# The containerd socket lets cAdvisor's containerd plugin resolve pod metadata +# and attach container_label_* labels. Only the socket is mounted, not all of +# /var/run: a read-only /var/run blocks Kubernetes from mounting the service +# account token, so the pod would fail to start. controller: type: daemonset volumes: @@ -41,6 +37,6 @@ alloy: - name: disk mountPath: /dev/disk readOnly: true - # Not read-only: cAdvisor connects to the containerd socket. - name: containerd-sock mountPath: /run/containerd/containerd.sock + readOnly: true From 9e415112a509bd1caaef8723ae62a1073afab9a4 Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Mon, 14 Sep 2026 12:02:47 -0400 Subject: [PATCH 8/9] test(prometheus.exporter.cadvisor): Assert more container labels Assert the pod namespace and container name labels alongside the pod name, per the reviewer's suggested set, to more fully cover the containerd metadata path. Co-Authored-By: Claude Opus 4.8 --- .../tests/prometheus-exporter-cadvisor/k8s_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go index b0c32aab269..5879fa0a7e0 100644 --- a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go @@ -89,7 +89,13 @@ func TestPrometheusExporterCadvisor(t *testing.T) { }) // The containerd socket lets cAdvisor resolve pod metadata, so container - // labels are attached. Kubernetes sets io.kubernetes.* labels on every - // container, so this label is always present when the containerd plugin works. - mimir.QueryLabelPresent(t, "cadvisor", "container_label_io_kubernetes_pod_name") + // labels are attached. Kubernetes sets these io.kubernetes.* labels on every + // container, so they are present whenever the containerd plugin works. + for _, label := range []string{ + "container_label_io_kubernetes_pod_name", + "container_label_io_kubernetes_pod_namespace", + "container_label_io_kubernetes_container_name", + } { + mimir.QueryLabelPresent(t, "cadvisor", label) + } } From 7eabf7a4d254d226ded81e556198c1fea5558dd6 Mon Sep 17 00:00:00 2001 From: Sam DeHaan Date: Mon, 14 Sep 2026 12:20:40 -0400 Subject: [PATCH 9/9] test(prometheus.exporter.cadvisor): Check all container labels in one query Make the label helper variadic (QueryLabelsPresent) and combine the label matchers into a single series query, so the three io.kubernetes.* labels are verified in one check that also requires them on the same series. Co-Authored-By: Claude Opus 4.8 --- integration-tests/k8s/deps/mimir.go | 16 +++++++++++----- .../prometheus-exporter-cadvisor/k8s_test.go | 8 +++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/integration-tests/k8s/deps/mimir.go b/integration-tests/k8s/deps/mimir.go index c439475c59d..b606423bbed 100644 --- a/integration-tests/k8s/deps/mimir.go +++ b/integration-tests/k8s/deps/mimir.go @@ -192,23 +192,29 @@ func (m *Mimir) QueryPositive(t *testing.T, testName string, metrics []string) { }, timeout, retryInterval) } -// QueryLabelPresent asserts at least one series for testName carries labelName -// with a non-empty value. It checks cAdvisor attaches container labels. -func (m *Mimir) QueryLabelPresent(t *testing.T, testName, labelName string) { +// QueryLabelsPresent asserts at least one series for testName carries every +// given label with a non-empty value. Passing multiple labels requires them on +// the same series. It checks cAdvisor attaches container labels. +func (m *Mimir) QueryLabelsPresent(t *testing.T, testName string, labelNames ...string) { t.Helper() + matchers := testNameLabel + "=\"" + testName + "\"" + for _, labelName := range labelNames { + matchers += "," + labelName + "=~\".+\"" + } + require.EventuallyWithT(t, func(c *assert.CollectT) { queryURL, err := url.Parse(m.endpoint("/prometheus/api/v1/series")) require.NoError(c, err) values := queryURL.Query() - values.Add("match[]", "{"+testNameLabel+"=\""+testName+"\","+labelName+"=~\".+\"}") + values.Add("match[]", "{"+matchers+"}") queryURL.RawQuery = values.Encode() resp := curl(c, queryURL.String(), nil) var parsed seriesResponse require.NoError(c, json.Unmarshal([]byte(resp), &parsed), "failed to parse series response: %s", resp) require.Equal(c, "success", parsed.Status, "mimir series query failed: %s", resp) - require.NotEmptyf(c, parsed.Data, "no series carrying label %s for %s=%s", labelName, testNameLabel, testName) + require.NotEmptyf(c, parsed.Data, "no series carrying labels %v for %s=%s", labelNames, testNameLabel, testName) }, timeout, retryInterval) } diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go index 5879fa0a7e0..675b3b5dfe6 100644 --- a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go @@ -90,12 +90,10 @@ func TestPrometheusExporterCadvisor(t *testing.T) { // The containerd socket lets cAdvisor resolve pod metadata, so container // labels are attached. Kubernetes sets these io.kubernetes.* labels on every - // container, so they are present whenever the containerd plugin works. - for _, label := range []string{ + // container, so a single series carries all three when the plugin works. + mimir.QueryLabelsPresent(t, "cadvisor", "container_label_io_kubernetes_pod_name", "container_label_io_kubernetes_pod_namespace", "container_label_io_kubernetes_container_name", - } { - mimir.QueryLabelPresent(t, "cadvisor", label) - } + ) }