diff --git a/integration-tests/k8s/deps/mimir.go b/integration-tests/k8s/deps/mimir.go index 45c842e43c7..b606423bbed 100644 --- a/integration-tests/k8s/deps/mimir.go +++ b/integration-tests/k8s/deps/mimir.go @@ -6,6 +6,7 @@ import ( "fmt" "net/url" "os" + "strconv" "testing" "github.com/stretchr/testify/assert" @@ -36,6 +37,21 @@ type metadataResponse struct { Data map[string][]ExpectedMetadata `json:"data"` } +type instantQueryResponse struct { + Status string `json:"status"` + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Value []any `json:"value"` + } `json:"result"` + } `json:"data"` +} + +type seriesResponse struct { + Status string `json:"status"` + Data []map[string]string `json:"data"` +} + // ExpectedMetadata is both the JSON payload shape from Mimir's metadata // endpoint and the input to QueryMetadata. Empty fields are not asserted. type ExpectedMetadata struct { @@ -136,6 +152,72 @@ func (m *Mimir) QueryMetrics(t *testing.T, testName string, expectedMetrics []st }, timeout, retryInterval) } +// QueryPositive polls an instant query for each metric (scoped to testName) and +// asserts at least one returned sample is greater than zero. It checks a metric +// is not just present but reports real data. +func (m *Mimir) QueryPositive(t *testing.T, testName string, metrics []string) { + t.Helper() + base := m.endpoint("/prometheus/api/v1/query") + + require.EventuallyWithT(t, func(c *assert.CollectT) { + for _, metric := range metrics { + queryURL, err := url.Parse(base) + require.NoError(c, err) + values := queryURL.Query() + values.Set("query", metric+"{"+testNameLabel+"=\""+testName+"\"}") + queryURL.RawQuery = values.Encode() + resp := curl(c, queryURL.String(), nil) + + var parsed instantQueryResponse + require.NoError(c, json.Unmarshal([]byte(resp), &parsed), "failed to parse query response: %s", resp) + require.Equal(c, "success", parsed.Status, "mimir query failed: %s", resp) + require.NotEmptyf(c, parsed.Data.Result, "%s: no samples for %s=%s", metric, testNameLabel, testName) + + var maxValue float64 + for _, result := range parsed.Data.Result { + if len(result.Value) != 2 { + continue + } + sample, ok := result.Value[1].(string) + if !ok { + continue + } + value, convErr := strconv.ParseFloat(sample, 64) + if convErr == nil && value > maxValue { + maxValue = value + } + } + require.Greaterf(c, maxValue, 0.0, "%s: expected a positive sample", metric) + } + }, timeout, retryInterval) +} + +// QueryLabelsPresent asserts at least one series for testName carries every +// given label with a non-empty value. Passing multiple labels requires them on +// the same series. It checks cAdvisor attaches container labels. +func (m *Mimir) QueryLabelsPresent(t *testing.T, testName string, labelNames ...string) { + t.Helper() + + matchers := testNameLabel + "=\"" + testName + "\"" + for _, labelName := range labelNames { + matchers += "," + labelName + "=~\".+\"" + } + + require.EventuallyWithT(t, func(c *assert.CollectT) { + queryURL, err := url.Parse(m.endpoint("/prometheus/api/v1/series")) + require.NoError(c, err) + values := queryURL.Query() + values.Add("match[]", "{"+matchers+"}") + queryURL.RawQuery = values.Encode() + resp := curl(c, queryURL.String(), nil) + + var parsed seriesResponse + require.NoError(c, json.Unmarshal([]byte(resp), &parsed), "failed to parse series response: %s", resp) + require.Equal(c, "success", parsed.Status, "mimir series query failed: %s", resp) + require.NotEmptyf(c, parsed.Data, "no series carrying labels %v for %s=%s", labelNames, testNameLabel, testName) + }, timeout, retryInterval) +} + // QueryMetadata asserts each expected metric appears in Mimir's // /api/v1/metadata with the requested Type/Help/Unit. func (m *Mimir) QueryMetadata(t *testing.T, expected map[string]ExpectedMetadata) { diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml new file mode 100644 index 00000000000..72a1f5ac5b9 --- /dev/null +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/alloy-values.yaml @@ -0,0 +1,42 @@ +# The containerd socket lets cAdvisor's containerd plugin resolve pod metadata +# and attach container_label_* labels. Only the socket is mounted, not all of +# /var/run: a read-only /var/run blocks Kubernetes from mounting the service +# account token, so the pod would fail to start. +controller: + type: daemonset + volumes: + extra: + - name: rootfs + hostPath: + path: / + - name: sys + hostPath: + path: /sys + - name: disk + hostPath: + path: /dev/disk + - name: containerd-sock + hostPath: + path: /run/containerd/containerd.sock + type: Socket + +alloy: + stabilityLevel: experimental + initialDelaySeconds: 1 + # The raw cgroup driver needs a privileged container to read /sys/fs/cgroup. + securityContext: + privileged: true + mounts: + extra: + - name: rootfs + mountPath: /rootfs + readOnly: true + - name: sys + mountPath: /sys + readOnly: true + - name: disk + mountPath: /dev/disk + readOnly: true + - name: containerd-sock + mountPath: /run/containerd/containerd.sock + readOnly: true diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/config.alloy b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/config.alloy new file mode 100644 index 00000000000..61780d03679 --- /dev/null +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/config/config.alloy @@ -0,0 +1,33 @@ +logging { + level = "debug" +} + +prometheus.exporter.cadvisor "test" { + storage_duration = "1m" +} + +prometheus.scrape "cadvisor" { + targets = prometheus.exporter.cadvisor.test.targets + forward_to = [prometheus.relabel.cadvisor.receiver] + scrape_interval = "1s" + scrape_timeout = "500ms" +} + +prometheus.relabel "cadvisor" { + forward_to = [prometheus.remote_write.mimir.receiver] + // The k8s harness filters series on alloy_test_name. + rule { + action = "replace" + target_label = "alloy_test_name" + replacement = "cadvisor" + } +} + +prometheus.remote_write "mimir" { + endpoint { + url = "http://mimir:9009/api/v1/push" + queue_config { + max_samples_per_send = 100 + } + } +} diff --git a/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go new file mode 100644 index 00000000000..675b3b5dfe6 --- /dev/null +++ b/integration-tests/k8s/tests/prometheus-exporter-cadvisor/k8s_test.go @@ -0,0 +1,99 @@ +package prometheusexportercadvisor + +import ( + "testing" + + "github.com/grafana/alloy/integration-tests/k8s/deps" + "github.com/grafana/alloy/integration-tests/k8s/harness" +) + +func TestPrometheusExporterCadvisor(t *testing.T) { + ns := deps.NewNamespace(deps.NamespaceOptions{ + Name: "test-prometheus-exporter-cadvisor", + Labels: map[string]string{"alloy-integration-test": "true"}, + }) + mimir := deps.NewMimir(deps.MimirOptions{Namespace: ns.Name()}) + alloy := deps.NewAlloy(deps.AlloyOptions{ + Namespace: ns.Name(), + Release: "alloy-test-prometheus-exporter-cadvisor", + ConfigPath: "./config/config.alloy", + ValuesPath: "./config/alloy-values.yaml", + }) + harness.Setup(t, harness.Options{ + Dependencies: []harness.Dependency{ns, mimir, alloy}, + }) + + // Covers the cAdvisor collector families: version, cpu, memory, filesystem, + // network, and blkio. + // + // Two families are left out on purpose: + // - container_pressure_* (PSI) needs kernel CONFIG_PSI. Not every host has it. + // - container_health_state needs a container with a Docker HEALTHCHECK. It + // depends on the sibling workloads, not the exporter. + mimir.QueryMetrics(t, "cadvisor", []string{ + "cadvisor_build_info", + "cadvisor_version_info", + "container_blkio_device_usage_total", + "container_cpu_load_average_10s", + "container_cpu_load_d_average_10s", + "container_cpu_system_seconds_total", + "container_cpu_usage_seconds_total", + "container_cpu_user_seconds_total", + "container_fs_inodes_free", + "container_fs_inodes_total", + "container_fs_io_current", + "container_fs_io_time_seconds_total", + "container_fs_io_time_weighted_seconds_total", + "container_fs_limit_bytes", + "container_fs_read_seconds_total", + "container_fs_reads_bytes_total", + "container_fs_reads_merged_total", + "container_fs_reads_total", + "container_fs_sector_reads_total", + "container_fs_sector_writes_total", + "container_fs_usage_bytes", + "container_fs_write_seconds_total", + "container_fs_writes_bytes_total", + "container_fs_writes_merged_total", + "container_fs_writes_total", + "container_last_seen", + "container_memory_cache", + "container_memory_failcnt", + "container_memory_failures_total", + "container_memory_kernel_usage", + "container_memory_mapped_file", + "container_memory_max_usage_bytes", + "container_memory_rss", + "container_memory_swap", + "container_memory_total_active_file_bytes", + "container_memory_total_inactive_file_bytes", + "container_memory_usage_bytes", + "container_memory_working_set_bytes", + "container_network_receive_bytes_total", + "container_network_receive_errors_total", + "container_network_receive_packets_dropped_total", + "container_network_receive_packets_total", + "container_network_transmit_bytes_total", + "container_network_transmit_errors_total", + "container_network_transmit_packets_dropped_total", + "container_network_transmit_packets_total", + "container_oom_events_total", + }) + + // The filesystem metrics must report real data, not just be present. These + // come from the root cgroup's machine filesystem, so they are always > 0 and + // prove the explicit filesystem-plugin wiring produces values. + mimir.QueryPositive(t, "cadvisor", []string{ + "container_fs_usage_bytes", + "container_fs_limit_bytes", + }) + + // The containerd socket lets cAdvisor resolve pod metadata, so container + // labels are attached. Kubernetes sets these io.kubernetes.* labels on every + // container, so a single series carries all three when the plugin works. + mimir.QueryLabelsPresent(t, "cadvisor", + "container_label_io_kubernetes_pod_name", + "container_label_io_kubernetes_pod_namespace", + "container_label_io_kubernetes_container_name", + ) +}