grafana · thampiotr · Aug 5, 2025 · Jul 23, 2025 · Jul 23, 2025 · Jul 23, 2025
@@ -18,8 +18,6 @@ Main (unreleased)
 
   - The `enable_http2` in `prometheus.remote_write` component's endpoints has been changed to `false` by default. Previously, in Prometheus v2 the remote write http client would default to use http2. In order to parallelize multiple remote write queues across multiple sockets its preferable to not default to http2. If you prefer to use http2 for remote write you must now set `enable_http2` to `true` in your `prometheus.remote_write` endpoints configuration section.
 
-  - Prometheus components such as `prometheus.scrape` and `prometheus.operator.*` now support UTF-8 in metric and label names by default. This means metric and label names can change after upgrading according to what is exposed by endpoints. Furthermore, metric and label names that would have previously been flagged as invalid no longer will be. Users wishing to preserve the original validation behavior can update their `prometheus.scrape` configuration to specify the legacy validation scheme: `metric_name_validation_scheme = "legacy"` and optionally setting the `metric_name_escaping_scheme` to a desired value. See `prometheus.scrape` reference documentation.
-
   - The experimental CLI flag `--feature.prometheus.metric-validation-scheme` has been deprecated and has no effect. You can configure the metric validation scheme individually for each `prometheus.scrape` component.
 
   - Log message format has changed for some of the `prometheus.*` components as part of the upgrade to Prometheus v3.
@@ -41,7 +39,7 @@ Main (unreleased)
 
 ### Enhancements
 
-- `prometheus.scrape` now supports `convert_classic_histograms_to_nhcb`, `enable_compression`, `native_histogram_bucket_limit`, and `native_histogram_min_bucket_factor` arguments. (@thampiotr)
+- `prometheus.scrape` now supports `convert_classic_histograms_to_nhcb`, `enable_compression`, `metric_name_validation_scheme`, `metric_name_escaping_scheme`, `native_histogram_bucket_limit`, and `native_histogram_min_bucket_factor` arguments. See reference documentation for more details. (@thampiotr)
 
 - Add `max_send_message_size` configuration option to `loki.source.api` component to control the maximum size of requests to the push API. (@thampiotr)
 

@@ -24,8 +24,6 @@ Prometheus dependency had a major version upgrade from v2.55.1 to v3.4.2.
 
 - The `enable_http2` in `prometheus.remote_write` component's endpoints has been changed to `false` by default. Previously, in Prometheus v2 the remote write http client would default to use http2. In order to parallelize multiple remote write queues across multiple sockets its preferable to not default to http2. If you prefer to use http2 for remote write you must now set `enable_http2` to `true` in your `prometheus.remote_write` endpoints configuration section.
 
-- Prometheus components such as `prometheus.scrape` and `prometheus.operator.*` now support UTF-8 in metric and label names by default. This means metric and label names can change after upgrading according to what is exposed by endpoints. Furthermore, metric and label names that would have previously been flagged as invalid no longer will be. Users wishing to preserve the original validation behavior can update their `prometheus.scrape` configuration to specify the legacy validation scheme: `metric_name_validation_scheme = "legacy"` and optionally setting the `metric_name_escaping_scheme` to a desired value. See `prometheus.scrape` reference documentation.
-
 - The experimental CLI flag `--feature.prometheus.metric-validation-scheme` has been deprecated and has no effect. You can configure the metric validation scheme individually for each `prometheus.scrape` component.
 
 - Log message format has changed for some of the `prometheus.*` components as part of the upgrade to Prometheus v3.

@@ -188,7 +188,7 @@ require (
 	github.com/prometheus/blackbox_exporter v0.24.1-0.20230623125439-bd22efa1c900
 	github.com/prometheus/client_golang v1.22.0
 	github.com/prometheus/client_model v0.6.2
-	github.com/prometheus/common v0.65.0
+	github.com/prometheus/common v0.65.1-0.20250804173848-0ad974f9af53
 	github.com/prometheus/consul_exporter v0.8.0
 	github.com/prometheus/memcached_exporter v0.13.0
 	github.com/prometheus/mysqld_exporter v0.17.2
@@ -1056,7 +1056,7 @@ replace github.com/fsnotify/fsnotify v1.8.0 => github.com/fsnotify/fsnotify v1.7
 // TODO: remove replace directive once:
 // * There is a release of Prometheus which addresses https://github.com/prometheus/prometheus/issues/14049,
 // for example, via this implementation: https://github.com/grafana/prometheus/pull/34
-replace github.com/prometheus/prometheus => github.com/grafana/prometheus v1.8.2-0.20250709144109-5551df55271b
+replace github.com/prometheus/prometheus => github.com/thampiotr/prometheus v0.0.0-20250731131506-a28a7d9c5b3b // staleness_disabling_v3.4.2 branch
 
 replace gopkg.in/yaml.v2 => github.com/rfratto/go-yaml v0.0.0-20211119180816-77389c3526dc
 

@@ -1328,8 +1328,6 @@ github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF
 github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls=
 github.com/grafana/postgres_exporter v0.0.0-20250714124518-c5d0a4dad445 h1:1rC8i0khfZdRA0/k7RBQQ84PhASNQLssezXHmFcWZ6Y=
 github.com/grafana/postgres_exporter v0.0.0-20250714124518-c5d0a4dad445/go.mod h1:0elZWWHfbwJNwKbiKgBKaEXTDewatxCt9UFZaN6LZsA=
-github.com/grafana/prometheus v1.8.2-0.20250709144109-5551df55271b h1:7e6NMPNpqETo6ID2oddAl7+Vt91SrpGGP3v7qwgrgG0=
-github.com/grafana/prometheus v1.8.2-0.20250709144109-5551df55271b/go.mod h1:ioGx2SGKTY+fLnJSQCdTHqARVldGNS8OlIe3kvp98so=
 github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
 github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
 github.com/grafana/pyroscope/api v1.2.0 h1:SfHDZcEZ4Vbj/Jj3bTOSpm4IDB33wLA2xBYxROhiL4U=
@@ -2372,8 +2370,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9
 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
-github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
+github.com/prometheus/common v0.65.1-0.20250804173848-0ad974f9af53 h1:9UzEu3LNsyYUSEkNU6mfBwG8TpiURfaJQgz1rVQKmhs=
+github.com/prometheus/common v0.65.1-0.20250804173848-0ad974f9af53/go.mod h1:LL3lcZII3UXGO4InbF+BTSsiAAPUBnwFVbp4gBWIMqw=
 github.com/prometheus/common/assets v0.2.0 h1:0P5OrzoHrYBOSM1OigWL3mY8ZvV2N4zIE/5AahrSrfM=
 github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI=
 github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4=
@@ -2613,6 +2611,8 @@ github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0 h1:o3bgcECyB
 github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0/go.mod h1:ljLR42dN7k40CX0dp30R8BRIB3OOdvr7rBANEpfmMs4=
 github.com/tg123/go-htpasswd v1.2.4 h1:HgH8KKCjdmo7jjXWN9k1nefPBd7Be3tFCTjc2jPraPU=
 github.com/tg123/go-htpasswd v1.2.4/go.mod h1:EKThQok9xHkun6NBMynNv6Jmu24A33XdZzzl4Q7H1+0=
+github.com/thampiotr/prometheus v0.0.0-20250731131506-a28a7d9c5b3b h1:OIY3jwIlF38C4a4uLw8PrnzzSEmwyMRgUUGCtfmQDTg=
+github.com/thampiotr/prometheus v0.0.0-20250731131506-a28a7d9c5b3b/go.mod h1:ioGx2SGKTY+fLnJSQCdTHqARVldGNS8OlIe3kvp98so=
 github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c=
 github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw=
 github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=

@@ -1,28 +1,29 @@
+logging {
+    level = "debug"
+}
+
+livedebugging {}
+
 prometheus.scrape "scrape_prom_metrics" {
   targets = [
     {"__address__" = "prom-gen:9001"},
   ]
-  forward_to = [prometheus.remote_write.scrape_prom_metrics.receiver, otelcol.receiver.prometheus.scrape_prom_metrics_to_otlp.receiver]
+  forward_to = [prometheus.write.queue.scrape_prom_metrics.receiver, otelcol.receiver.prometheus.scrape_prom_metrics_to_otlp.receiver]
   scrape_classic_histograms = true
-  scrape_protocols = ["PrometheusProto", "OpenMetricsText1.0.0", "OpenMetricsText0.0.1", "PrometheusText0.0.4"]
+  scrape_native_histograms = true
   scrape_interval = "1s"
   scrape_timeout = "500ms"
 }
 
-prometheus.remote_write "scrape_prom_metrics" {
-  endpoint {
-    url = "http://mimir:9009/api/v1/push"
-    send_native_histograms = true
-    metadata_config {
-        send_interval = "1s"
+prometheus.write.queue "scrape_prom_metrics" {
+    endpoint "mimir" {
+        url = "http://mimir:9009/api/v1/push"
+        flush_interval = "1s"
+        batch_count = 10
+        external_labels = {
+            test_name = "scrape_prom_metrics",
+        }
     }
-    queue_config {
-        max_samples_per_send = 100
-    }
-  }
-  external_labels = {
-    test_name = "scrape_prom_metrics",
-  }
 }
 
 otelcol.receiver.prometheus "scrape_prom_metrics_to_otlp" {

@@ -5,10 +5,11 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/gorilla/mux"
-	"github.com/grafana/alloy/internal/runtime/logging/level"
 	dskit "github.com/grafana/dskit/server"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
+
+	"github.com/grafana/alloy/internal/runtime/logging/level"
 )
 
 // TargetServer is wrapper around dskit.Server that handles some common
@@ -25,6 +26,8 @@ type TargetServer struct {
 // NewTargetServer creates a new TargetServer, applying some defaults to the server configuration.
 // If provided config is nil, a default configuration will be used instead.
 func NewTargetServer(logger log.Logger, metricsNamespace string, reg prometheus.Registerer, config *ServerConfig) (*TargetServer, error) {
+	// TODO: add support for different validation schemes.
+	//nolint:staticcheck
 	if !model.IsValidMetricName(model.LabelValue(metricsNamespace)) {
 		return nil, fmt.Errorf("metrics namespace is not prometheus compatible: %s", metricsNamespace)
 	}

@@ -171,12 +171,16 @@ func (rc *Config) Validate() error {
 	if (rc.Action == Replace || rc.Action == HashMod || rc.Action == Lowercase || rc.Action == Uppercase || rc.Action == KeepEqual || rc.Action == DropEqual) && rc.TargetLabel == "" {
 		return fmt.Errorf("relabel configuration for %s action requires 'target_label' value", rc.Action)
 	}
+	// TODO: add support for different validation schemes.
+	//nolint:staticcheck
 	if rc.Action == Replace && !strings.Contains(rc.TargetLabel, "$") && !model.LabelName(rc.TargetLabel).IsValid() {
 		return fmt.Errorf("%q is invalid 'target_label' for %s action", rc.TargetLabel, rc.Action)
 	}
 	if rc.Action == Replace && strings.Contains(rc.TargetLabel, "$") && !relabelTarget.MatchString(rc.TargetLabel) {
 		return fmt.Errorf("%q is invalid 'target_label' for %s action", rc.TargetLabel, rc.Action)
 	}
+	// TODO: add support for different validation schemes.
+	//nolint:staticcheck
 	if (rc.Action == Lowercase || rc.Action == Uppercase || rc.Action == KeepEqual || rc.Action == DropEqual) && !model.LabelName(rc.TargetLabel).IsValid() {
 		return fmt.Errorf("%q is invalid 'target_label' for %s action", rc.TargetLabel, rc.Action)
 	}
@@ -186,6 +190,8 @@ func (rc *Config) Validate() error {
 	if rc.Action == LabelMap && !relabelTarget.MatchString(rc.Replacement) {
 		return fmt.Errorf("%q is invalid 'replacement' for %s action", rc.Replacement, rc.Action)
 	}
+	// TODO: add support for different validation schemes.
+	//nolint:staticcheck
 	if rc.Action == HashMod && !model.LabelName(rc.TargetLabel).IsValid() {
 		return fmt.Errorf("%q is invalid 'target_label' for %s action", rc.TargetLabel, rc.Action)
 	}
@@ -261,6 +267,8 @@ func doRelabel(cfg *Config, lb LabelBuilder) (keep bool) {
 			break
 		}
 		target := model.LabelName(cfg.Regex.ExpandString([]byte{}, cfg.TargetLabel, val, indexes))
+		// TODO: add support for different validation schemes.
+		//nolint:staticcheck
 		if !target.IsValid() {
 			break
 		}

@@ -67,6 +67,8 @@ func cleanLabels(set model.LabelSet) model.LabelSet {
 	for k, v := range set {
 		// Performing the same label validity check the prometheus go client library does.
 		// https://github.com/prometheus/client_golang/blob/618194de6ad3db637313666104533639011b470d/prometheus/labels.go#L85
+		// TODO: add support for different validation schemes.
+		//nolint:staticcheck
 		if !k.IsValid() || strings.HasPrefix(string(k), "__") {
 			continue
 		}

@@ -20,6 +20,8 @@ type EventLogMessageConfig struct {
 }
 
 func (e *EventLogMessageConfig) Validate() error {
+	// TODO: add support for different validation schemes.
+	//nolint:staticcheck
 	if !model.LabelName(e.Source).IsValidLegacy() {
 		return fmt.Errorf(ErrInvalidLabelName, e.Source)
 	}
@@ -82,6 +84,8 @@ func (m *eventLogMessageStage) processEntry(extracted map[string]interface{}, ke
 			continue
 		}
 		mkey := parts[0]
+		// TODO: add support for different validation schemes.
+		//nolint:staticcheck
 		if !model.LabelName(mkey).IsValidLegacy() {
 			if m.cfg.DropInvalidLabels {
 				if Debug {

@@ -30,6 +30,8 @@ func validateLabelsConfig(c LabelsConfig) (map[string]string, error) {
 		return nil, errors.New(ErrEmptyLabelStageConfig)
 	}
 	for labelName, labelSrc := range c.Values {
+		// TODO: add support for different validation schemes.
+		//nolint:staticcheck
 		if !model.LabelName(labelName).IsValid() {
 			return nil, fmt.Errorf(ErrInvalidLabelName, labelName)
 		}

@@ -123,6 +123,8 @@ func (r *regexStage) Process(labels model.LabelSet, extracted map[string]interfa
 				labelName := model.LabelName(name)
 				labelValue := model.LabelValue(match[i])
 
+				// TODO: add support for different validation schemes.
+				//nolint:staticcheck
 				if !labelName.IsValid() {
 					if Debug {
 						level.Debug(r.logger).Log("msg", "invalid label name from regex capture group", "labelName", labelName)

@@ -37,6 +37,8 @@ func validateLabelStaticConfig(c StaticLabelsConfig) error {
 		return ErrEmptyStaticLabelStageConfig
 	}
 	for labelName := range c.Values {
+		// TODO: add support for different validation schemes.
+		//nolint:staticcheck
 		if !model.LabelName(labelName).IsValid() {
 			return fmt.Errorf(ErrInvalidLabelName, labelName)
 		}

@@ -22,6 +22,8 @@ type WindowsEventConfig struct {
 }
 
 func (e *WindowsEventConfig) Validate() error {
+	// TODO: add support for different validation schemes.
+	//nolint:staticcheck
 	if !model.LabelName(e.Source).IsValid() {
 		return fmt.Errorf(ErrInvalidLabelName, e.Source)
 	}
@@ -165,6 +167,8 @@ func (w *WindowsEventStage) processEntry(extracted map[string]interface{}, key s
 
 func (w *WindowsEventStage) sanitizeKey(ekey string, extracted map[string]interface{}) (string, error) {
 	k := w.keyReplacer.Replace(ekey)
+	// TODO: add support for different validation schemes.
+	//nolint:staticcheck
 	if !model.LabelName(k).IsValid() {
 		if w.cfg.DropInvalidLabels {
 			return "", fmt.Errorf("invalid label parsed from message, key: %s", k)

@@ -194,6 +194,8 @@ func (h *Handler) postProcessLabels(lbs labels.Labels) model.LabelSet {
 		}
 
 		// ignore invalid labels
+		// TODO: add support for different validation schemes.
+		//nolint:staticcheck
 		if !model.LabelName(lbl.Name).IsValidLegacy() || !model.LabelValue(lbl.Value).IsValid() {
 			continue
 		}
@@ -319,12 +321,16 @@ func (h *Handler) tryToGetStaticLabelsFromRequest(req *http.Request, tenantID st
 		// construct model.LabelName from the header value, if the raw data is not valid label name, try to fix it and use
 		rawLabelName := strings.TrimPrefix(name, commonAttributesLabelPrefix)
 		labelName := model.LabelName(rawLabelName)
+		// TODO: add support for different validation schemes.
+		//nolint:staticcheck
 		if !labelName.IsValidLegacy() {
 			level.Debug(h.logger).Log(fmt.Sprintf("label name is not valid, trying to fix: %s", rawLabelName))
 
 			// try to sanitize label name
 			sanitizedLabelName := yacepromutil.PromString(rawLabelName)
 			labelName = model.LabelName(sanitizedLabelName)
+			// TODO: add support for different validation schemes.
+			//nolint:staticcheck
 			if !labelName.IsValidLegacy() {
 				// This situation can happen when:
 				// - the header with label information is a valid JSON

@@ -222,6 +222,8 @@ func (e *AzureEventHubsTargetMessageParser) getLabels(logRecord *azureMonitorRes
 			continue
 		}
 		// ignore invalid labels
+		// TODO: add support for different validation schemes.
+		//nolint:staticcheck
 		if !model.LabelName(lbl.Name).IsValid() || !model.LabelValue(lbl.Value).IsValid() {
 			continue
 		}

@@ -89,6 +89,8 @@ func parseGCPLogsEntry(data []byte, other model.LabelSet, otherInternal labels.L
 			continue
 		}
 		// ignore invalid labels
+		// TODO: add support for different validation schemes.
+		//nolint:staticcheck
 		if !model.LabelName(lbl.Name).IsValid() || !model.LabelValue(lbl.Value).IsValid() {
 			continue
 		}

@@ -89,8 +89,8 @@ func TestGeneratePodMonitorConfig(t *testing.T) {
 					},
 				},
 				ConvertClassicHistogramsToNHCB: ptr.To(false),
-				MetricNameValidationScheme:     "utf8",
-				MetricNameEscapingScheme:       "allow-utf-8",
+				MetricNameValidationScheme:     "legacy",
+				MetricNameEscapingScheme:       "underscores",
 			},
 		},
 		{
@@ -150,8 +150,8 @@ func TestGeneratePodMonitorConfig(t *testing.T) {
 					},
 				},
 				ConvertClassicHistogramsToNHCB: ptr.To(false),
-				MetricNameValidationScheme:     "utf8",
-				MetricNameEscapingScheme:       "allow-utf-8",
+				MetricNameValidationScheme:     "legacy",
+				MetricNameEscapingScheme:       "underscores",
 			},
 		},
 		{
@@ -211,8 +211,8 @@ func TestGeneratePodMonitorConfig(t *testing.T) {
 					},
 				},
 				ConvertClassicHistogramsToNHCB: ptr.To(false),
-				MetricNameValidationScheme:     "utf8",
-				MetricNameEscapingScheme:       "allow-utf-8",
+				MetricNameValidationScheme:     "legacy",
+				MetricNameEscapingScheme:       "underscores",
 			},
 		},
 		{
@@ -272,8 +272,8 @@ func TestGeneratePodMonitorConfig(t *testing.T) {
 					},
 				},
 				ConvertClassicHistogramsToNHCB: ptr.To(false),
-				MetricNameValidationScheme:     "utf8",
-				MetricNameEscapingScheme:       "allow-utf-8",
+				MetricNameValidationScheme:     "legacy",
+				MetricNameEscapingScheme:       "underscores",
 			},
 		},
 		{
@@ -438,8 +438,8 @@ func TestGeneratePodMonitorConfig(t *testing.T) {
 				LabelNameLengthLimit:           104,
 				LabelValueLengthLimit:          105,
 				ConvertClassicHistogramsToNHCB: ptr.To(false),
-				MetricNameValidationScheme:     "utf8",
-				MetricNameEscapingScheme:       "allow-utf-8",
+				MetricNameValidationScheme:     "legacy",
+				MetricNameEscapingScheme:       "underscores",
 			},
 		},
 	}

@@ -102,8 +102,8 @@ func TestGenerateProbeConfig(t *testing.T) {
 					},
 				},
 				ConvertClassicHistogramsToNHCB: ptr.To(false),
-				MetricNameValidationScheme:     "utf8",
-				MetricNameEscapingScheme:       "allow-utf-8",
+				MetricNameValidationScheme:     "legacy",
+				MetricNameEscapingScheme:       "underscores",
 			},
 		},
 		{
@@ -194,8 +194,8 @@ func TestGenerateProbeConfig(t *testing.T) {
 					},
 				},
 				ConvertClassicHistogramsToNHCB: ptr.To(false),
-				MetricNameValidationScheme:     "utf8",
-				MetricNameEscapingScheme:       "allow-utf-8",
+				MetricNameValidationScheme:     "legacy",
+				MetricNameEscapingScheme:       "underscores",
 			},
 		},
 		{
@@ -288,8 +288,8 @@ func TestGenerateProbeConfig(t *testing.T) {
 					},
 				},
 				ConvertClassicHistogramsToNHCB: ptr.To(false),
-				MetricNameValidationScheme:     "utf8",
-				MetricNameEscapingScheme:       "allow-utf-8",
+				MetricNameValidationScheme:     "legacy",
+				MetricNameEscapingScheme:       "underscores",
 			},
 		},
 	}