Implement Memory Limiter prototype

.github/workflows/ci.yml

@@ -223,7 +223,7 @@ jobs:
         run: |
           mkdir -p .prebuilt-ui
           tar -xzvf .tarballs/prometheus-web-ui-*.tar.gz -C .prebuilt-ui
-      - uses: prometheus/promci/build@13941414d409d227afd67544e5d306827db5a1a2 # v0.8.5
+      - uses: prometheus/promci/build@370e8c15dcec50043cbe66f2f34633d9efc0a190 # v0.9.0
         with:
           checkout: false
           # Feed the restored UI assets into the crossbuild containers so make
@@ -260,7 +260,7 @@ jobs:
         run: |
           mkdir -p .prebuilt-ui
           tar -xzvf .tarballs/prometheus-web-ui-*.tar.gz -C .prebuilt-ui
-      - uses: prometheus/promci/build@13941414d409d227afd67544e5d306827db5a1a2 # v0.8.5
+      - uses: prometheus/promci/build@370e8c15dcec50043cbe66f2f34633d9efc0a190 # v0.9.0
         with:
           checkout: false
           parallelism: 12
@@ -370,7 +370,7 @@ jobs:
     needs: [test_ui, test_go, test_go_more, test_go_oldest, test_windows, golangci, codeql, build_all]
     if: github.event_name == 'push' && github.event.ref == 'refs/heads/main'
     steps:
-      - uses: prometheus/promci/publish_main@13941414d409d227afd67544e5d306827db5a1a2 # v0.8.5
+      - uses: prometheus/promci/publish_main@370e8c15dcec50043cbe66f2f34633d9efc0a190 # v0.9.0
         with:
           docker_hub_login: ${{ secrets.docker_hub_login }}
           docker_hub_password: ${{ secrets.docker_hub_password }}
@@ -389,7 +389,7 @@ jobs:
       ||
       (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v3.'))
     steps:
-      - uses: prometheus/promci/publish_release@13941414d409d227afd67544e5d306827db5a1a2 # v0.8.5
+      - uses: prometheus/promci/publish_release@370e8c15dcec50043cbe66f2f34633d9efc0a190 # v0.9.0
         with:
           docker_hub_login: ${{ secrets.docker_hub_login }}
           docker_hub_password: ${{ secrets.docker_hub_password }}
@@ -411,7 +411,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Install nodejs
-        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
         with:
           node-version-file: "web/ui/.nvmrc"
           registry-url: "https://registry.npmjs.org"

.github/workflows/stale.yml

@@ -14,7 +14,7 @@ jobs:
     if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks.
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
+      - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           # opt out of defaults to avoid marking issues as stale and closing them

Makefile

@@ -80,8 +80,12 @@ ui-build-module:
 	cd $(UI_PATH) && pnpm run build:module
 
 .PHONY: ui-test
-ui-test:
+ui-test: ui-build-module
 	cd $(UI_PATH) && CI=true pnpm run test
+	# The old React app has been separated from the npm workspaces setup to avoid
+	# issues with conflicting dependencies. This is a temporary solution until the
+	# new Mantine-based UI is fully integrated and the old app can be removed.
+	cd $(UI_PATH)/react-app && CI=true pnpm run test
 
 .PHONY: ui-lint
 ui-lint:
@@ -172,7 +176,7 @@ install-goyacc:
 ifeq ($(GO_ONLY),1)
 test: common-test check-go-mod-version
 else
-test: check-generated-parser common-test check-node-version ui-build-module ui-test ui-lint check-go-mod-version
+test: check-generated-parser common-test check-node-version ui-test ui-lint check-go-mod-version
 endif
 
 .PHONY: tarball
@@ -222,8 +226,8 @@ update-all-go-deps: update-go-deps
 .PHONY: update-go-deps-in-dir
 update-go-deps-in-dir:
 	@echo ">> updating Go dependencies in ./$(DIR)/"
+	# GOTOOLCHAIN prevents go get from pulling deps that require a newer Go version.
 	@cd ./$(DIR) && for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \
-		# Prevent go get from pulling deps that require a newer Go version.
 		GOTOOLCHAIN=go$$($(GO) mod edit -json | jq -r .Go) $(GO) get $$m; \
 	done
 	@cd ./$(DIR) && $(GO) mod tidy

Makefile.common

@@ -165,8 +165,8 @@ common-deps:
 .PHONY: update-go-deps
 update-go-deps:
 	@echo ">> updating Go dependencies"
+	# GOTOOLCHAIN prevents go get from pulling deps that require a newer Go version.
 	@for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \
-		# Prevent go get from pulling deps that require a newer Go version.
 		GOTOOLCHAIN=go$$($(GO) mod edit -json | jq -r .Go) $(GO) get $$m; \
 	done
 	$(GO) mod tidy

cmd/prometheus/features_test.go

@@ -23,7 +23,6 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/require"
 
@@ -53,15 +52,7 @@ func TestFeaturesAPI(t *testing.T) {
 
 	baseURL := fmt.Sprintf("http://127.0.0.1:%d", port)
 
-	// Wait for Prometheus to be ready.
-	require.Eventually(t, func() bool {
-		resp, err := http.Get(baseURL + "/-/ready")
-		if err != nil {
-			return false
-		}
-		defer resp.Body.Close()
-		return resp.StatusCode == http.StatusOK
-	}, 10*time.Second, 100*time.Millisecond, "Prometheus didn't become ready in time")
+	waitForPrometheusReady(t, port)
 
 	// Fetch features from the API.
 	resp, err := http.Get(baseURL + "/api/v1/features")

cmd/prometheus/main.go

@@ -84,6 +84,7 @@ import (
 	"github.com/prometheus/prometheus/util/documentcli"
 	"github.com/prometheus/prometheus/util/features"
 	"github.com/prometheus/prometheus/util/logging"
+	"github.com/prometheus/prometheus/util/memorylimiter"
 	"github.com/prometheus/prometheus/util/notifications"
 	prom_runtime "github.com/prometheus/prometheus/util/runtime"
 	"github.com/prometheus/prometheus/web"
@@ -217,6 +218,7 @@ type flagConfig struct {
 	enablePerStepStats       bool
 	enableConcurrentRuleEval bool
 	useStartTimestamps       bool
+	enableMemoryLimiter      bool
 
 	prometheusURL   string
 	corsRegexString string
@@ -345,6 +347,9 @@ func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error {
 			case "search-api":
 				c.web.EnableSearch = true
 				logger.Info("Experimental search API enabled.")
+			case "memory-limiter":
+				c.enableMemoryLimiter = true
+				logger.Info("Experimental memory limiter is enabled.")
 			default:
 				logger.Warn("Unknown option for --enable-feature", "option", o)
 			}
@@ -378,6 +383,7 @@ func main() {
 			collectors.NewGoCollector(
 				collectors.WithGoCollectorRuntimeMetrics(
 					collectors.MetricsGC,
+					collectors.MetricsMemory,
 					collectors.MetricsScheduler,
 					collectors.GoRuntimeMetricsRule{Matcher: goregexp.MustCompile(`^/sync/mutex/wait/total:seconds$`)},
 				),
@@ -645,7 +651,7 @@ func main() {
 	a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates.").
 		Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval)
 
-	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: concurrent-rule-eval, created-timestamp-zero-ingestion, delayed-compaction, exemplar-storage, extra-scrape-metrics, histograms-st-encoding, memory-snapshot-on-shutdown, metadata-wal-records, old-ui, otlp-deltatocumulative, otlp-native-delta-ingestion, promql-binop-fill-modifiers, promql-delayed-name-removal, promql-experimental-functions, promql-extended-range-selectors, promql-per-step-stats, search-api, st-storage, st-synthesis, type-and-unit-labels, use-start-timestamps, use-uncached-io, xor2-encoding. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: concurrent-rule-eval, created-timestamp-zero-ingestion, delayed-compaction, exemplar-storage, extra-scrape-metrics, histograms-st-encoding, memory-limiter, memory-snapshot-on-shutdown, metadata-wal-records, old-ui, otlp-deltatocumulative, otlp-native-delta-ingestion, promql-binop-fill-modifiers, promql-delayed-name-removal, promql-experimental-functions, promql-extended-range-selectors, promql-per-step-stats, search-api, st-storage, st-synthesis, type-and-unit-labels, use-start-timestamps, use-uncached-io, xor2-encoding. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
 		StringsVar(&cfg.featureList)
 
 	a.Flag("agent", "Run Prometheus in 'Agent mode'.").BoolVar(&agentMode)
@@ -969,6 +975,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	var memoryLimiter memorylimiter.MemoryLimiter
+	if cfg.enableMemoryLimiter {
+		ml, err := memorylimiter.NewManager(&cfgFile.Runtime.MemoryLimiter, logger.With("component", "memory limiter"), prometheus.DefaultRegisterer)
+		if err != nil {
+			logger.Error("failed to create memory limiter", "err", err)
+			os.Exit(1)
+		}
+		memoryLimiter = ml
+		cfg.scrape.MemoryLimiter = ml
+		cfg.web.MemoryLimiter = ml
+	}
+
 	scrapeManager, err := scrape.NewManager(
 		&cfg.scrape,
 		logger.With("component", "scrape manager"),
@@ -1037,6 +1055,7 @@ func main() {
 			},
 			FeatureRegistry: features.DefaultRegistry,
 			Parser:          promqlParser,
+			MemoryLimiter:   memoryLimiter,
 		})
 	}
 
@@ -1201,6 +1220,14 @@ func main() {
 		}, {
 			name:     "tracing",
 			reloader: tracingManager.ApplyConfig,
+		}, {
+			name: "memory_limiter",
+			reloader: func(cfg *config.Config) error {
+				if memoryLimiter != nil {
+					return memoryLimiter.ApplyConfig(&cfg.Runtime.MemoryLimiter)
+				}
+				return nil
+			},
 		},
 	}
 
@@ -1294,6 +1321,21 @@ func main() {
 			},
 		)
 	}
+	if memoryLimiter != nil {
+		ctxML, cancelML := context.WithCancel(context.Background())
+		g.Add(
+			func() error {
+				memoryLimiter.Start(ctxML)
+				<-ctxML.Done()
+				return nil
+			},
+			func(error) {
+				logger.Info("Stopping memory limiter...")
+				cancelML()
+				memoryLimiter.Stop()
+			},
+		)
+	}
 	if !agentMode {
 		// Rule manager.
 		g.Add(

cmd/prometheus/main_test.go

@@ -178,14 +178,7 @@ storage:
 	)
 	require.NoError(t, prom.Start())
 
-	require.Eventually(t, func() bool {
-		r, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", port))
-		if err != nil {
-			return false
-		}
-		defer r.Body.Close()
-		return r.StatusCode == http.StatusOK
-	}, startupTime, 100*time.Millisecond)
+	waitForPrometheusReady(t, port)
 	require.DirExists(t, storagePath)
 }
 
@@ -704,9 +697,6 @@ func TestModeSpecificFlags(t *testing.T) {
 }
 
 func TestDocumentation(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.SkipNow()
-	}
 	t.Parallel()
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -719,12 +709,17 @@ func TestDocumentation(t *testing.T) {
 
 	require.NoError(t, cmd.Run(), "failed to generate CLI documentation via --write-documentation")
 
-	generatedContent := strings.ReplaceAll(stdout.String(), filepath.Base(promPath), strings.TrimSuffix(filepath.Base(promPath), ".test"))
+	appTitle := strings.TrimSuffix(filepath.Base(promPath), ".test")
+	if runtime.GOOS == "windows" {
+		appTitle = strings.TrimSuffix(appTitle, ".test.exe")
+	}
+
+	generatedContent := strings.ReplaceAll(stdout.String(), filepath.Base(promPath), appTitle)
 
 	expectedContent, err := os.ReadFile(filepath.Join("..", "..", "docs", "command-line", "prometheus.md"))
 	require.NoError(t, err)
 
-	require.Equal(t, string(expectedContent), generatedContent, "Generated content does not match documentation. Hint: run `make cli-documentation`.")
+	require.Equal(t, strings.ReplaceAll(string(expectedContent), "\r\n", "\n"), generatedContent, "Generated content does not match documentation. Hint: run `make cli-documentation`.")
 }
 
 func TestRwProtoMsgFlagParser(t *testing.T) {
@@ -902,26 +897,24 @@ global:
 				fmt.Sprintf("--storage.tsdb.path=%s", tmpDir),
 				"--web.enable-lifecycle",
 			)
-			// Inject GOGC when set.
-			prom.Env = os.Environ()
+			// Inject GOGC when set, and drop any inherited one otherwise, so
+			// that the environment the test runs in cannot influence the cases
+			// expecting the default or the configured value.
+			prom.Env = slices.DeleteFunc(os.Environ(), func(kv string) bool {
+				return strings.HasPrefix(kv, "GOGC=")
+			})
 			if tc.gogcEnvVar != "" {
 				prom.Env = append(prom.Env, fmt.Sprintf("GOGC=%s", tc.gogcEnvVar))
 			}
 			require.NoError(t, prom.Start())
 
+			// Wait for the initial configuration to be applied: /metrics is
+			// already served before that happens.
+			waitForPrometheusReady(t, port)
+
 			ensureGOGCValue := func(val float64) {
-				var (
-					r   *http.Response
-					err error
-				)
-				// Wait for the /metrics endpoint to be ready.
-				require.Eventually(t, func() bool {
-					r, err = http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", port))
-					if err != nil {
-						return false
-					}
-					return r.StatusCode == http.StatusOK
-				}, 5*time.Second, 50*time.Millisecond)
+				r, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", port))
+				require.NoError(t, err)
 				defer r.Body.Close()
 
 				// Check the final GOGC that's set, consider go_gc_gogc_percent from /metrics as source of truth.
@@ -1216,9 +1209,6 @@ remote_write:
 // TestFeatureFlagsDocumented ensures the --enable-feature help text in main.go
 // and the documented flags in docs/feature_flags.md list the same set of flags.
 func TestFeatureFlagsDocumented(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.SkipNow()
-	}
 	h, err := os.ReadFile(filepath.Join("..", "..", "cmd", "prometheus", "main.go"))
 	require.NoError(t, err)
 	m := regexp.MustCompile(`a\.Flag\("enable-feature", "Comma separated feature names to enable. Valid options: (.+?)\.`).FindSubmatch(h)
@@ -1230,6 +1220,10 @@ func TestFeatureFlagsDocumented(t *testing.T) {
 	require.NotEmpty(t, helpFlags)
 
 	d, err := os.ReadFile(filepath.Join("..", "..", "docs", "feature_flags.md"))
+	if runtime.GOOS == "windows" {
+		d = bytes.ReplaceAll(d, []byte("\r\n"), []byte("\n"))
+	}
+
 	require.NoError(t, err)
 	var docFlags []string
 	for _, dm := range regexp.MustCompile("(?m)^`--enable-feature=(.+)`$").FindAllSubmatch(d, -1) {

cmd/prometheus/reload_test.go

@@ -123,14 +123,7 @@ func runTestSteps(t *testing.T, steps []struct {
 	require.NoError(t, prom.Start())
 
 	baseURL := "http://localhost:" + strconv.Itoa(port)
-	require.Eventually(t, func() bool {
-		resp, err := http.Get(baseURL + "/-/ready")
-		if err != nil {
-			return false
-		}
-		defer resp.Body.Close()
-		return resp.StatusCode == http.StatusOK
-	}, 5*time.Second, 100*time.Millisecond, "Prometheus didn't become ready in time")
+	waitForPrometheusReady(t, port)
 
 	for i, step := range steps {
 		t.Logf("Step %d", i)
@@ -241,3 +234,25 @@ func prometheusCommandWithLogging(t testing.TB, configFilePath string, port int,
 	args = append(args, extraArgs...)
 	return commandWithLogging(t, nil, promPath, args...)
 }
+
+// readyTimeout is how long we wait for a Prometheus process to become ready. It
+// is generous on purpose: CI runs these tests with the race detector, which
+// slows down both the test binary and the Prometheus processes it spawns in
+// parallel.
+const readyTimeout = 30 * time.Second
+
+// waitForPrometheusReady waits until the Prometheus instance listening on port
+// reports itself as ready, which means it has opened its storage and applied
+// its initial configuration.
+func waitForPrometheusReady(t testing.TB, port int) {
+	t.Helper()
+
+	require.Eventually(t, func() bool {
+		resp, err := http.Get("http://127.0.0.1:" + strconv.Itoa(port) + "/-/ready")
+		if err != nil {
+			return false
+		}
+		defer resp.Body.Close()
+		return resp.StatusCode == http.StatusOK
+	}, readyTimeout, 100*time.Millisecond, "Prometheus didn't become ready in time")
+}

compliance/go.mod

@@ -1,6 +1,6 @@
 module compliance
 
-go 1.25.8
+go 1.25.10
 
 require github.com/prometheus/compliance/remotewrite v0.0.0-20260223092825-818283e1171e