fix(metrics): cap the distinct domains the exporter labels by

CHANGELOG.md

@@ -6,6 +6,10 @@ Solverr follows its own [Semantic Versioning](https://semver.org/), starting at
 
 ### Changes
 
+- **Prometheus metrics now report at most 100 distinct domains, and every host past that as `other`.** Prometheus keeps one time series per label value for as long as the process runs and nothing evicted them, so pointing Solverr at many hosts grew the registry and the exported payload without limit. Only affects deployments running with `PROMETHEUS_ENABLED=true`.
+
+### Changes
+
 - **A request's `maxTimeout` is now capped at 180000 ms, raised with `MAX_TIMEOUT_MS`.** Nothing bounded it above, so a single request could hold a browser for as long as the caller asked, and because the session it was using counted as busy the whole time, the reaper could not reclaim that browser either. A larger value is clamped with a warning rather than refused, so callers already asking for more keep working.
 
 ### Changes

README.md

@@ -408,6 +408,8 @@ Rough guide to expected latency: Chrome solves take a few seconds; Camoufox solv
 
 Disabled by default. Enable with `PROMETHEUS_ENABLED=true` and expose `PROMETHEUS_PORT` (default 8192). Metrics include per-domain request counts, results, and duration histograms.
 
+The domain label is capped at 100 distinct hosts; every host after that is reported as `other`. Prometheus keeps a time series per label value for the life of the process, so an uncapped label would grow the registry with the number of hosts requested. A deployer pointing Solverr at a handful of sites never reaches the cap.
+
 ## Troubleshooting
 
 **A source shows no results but the log says `Challenge not detected!` with a 200.** An engine loaded the page but couldn't recognise a newer managed/Turnstile challenge and returned it as if solved. Solverr's auto-fallback is designed to catch this and retry on the other engine; make sure `ENGINE_FALLBACK` is on and the stealth engine is enabled. If it still fails, the site is likely gating on your IP — add a residential proxy.

src/bottle_plugins/prometheus_plugin.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import threading
 import urllib.parse
 
 from bottle import request
@@ -10,6 +11,45 @@
 PROMETHEUS_PORT = int(os.environ.get('PROMETHEUS_PORT', 8192))
 
 
+# Prometheus holds one time series per distinct label value for the life of the
+# process and nothing here evicts, so labelling by hostname let the registry grow
+# with the number of hosts a client asked for. Past this many distinct hosts every
+# further one is reported as _OTHER_DOMAIN: a deployer running a handful of
+# indexers keeps full per-domain data and never reaches the cap, while a broad
+# workload degrades to one bucket instead of growing without bound.
+_MAX_DOMAIN_LABELS = 100
+_OTHER_DOMAIN = 'other'
+_UNKNOWN_DOMAIN = 'unknown'
+
+_seen_domains = set()
+_domains_lock = threading.Lock()
+
+
+def reset_domain_labels() -> None:
+    """Forget every domain seen so far. Exists for tests."""
+    with _domains_lock:
+        _seen_domains.clear()
+
+
+def parse_domain_url(url) -> str:
+    """The metric label for ``url``: its hostname, or a sentinel.
+
+    Returns _OTHER_DOMAIN once _MAX_DOMAIN_LABELS distinct hosts have been seen,
+    and _UNKNOWN_DOMAIN when the URL carries no hostname, which would otherwise
+    label a series "None".
+    """
+    hostname = urllib.parse.urlparse(url).hostname if url else None
+    if not hostname:
+        return _UNKNOWN_DOMAIN
+    with _domains_lock:
+        if hostname in _seen_domains:
+            return hostname
+        if len(_seen_domains) >= _MAX_DOMAIN_LABELS:
+            return _OTHER_DOMAIN
+        _seen_domains.add(hostname)
+    return hostname
+
+
 def setup():
     if PROMETHEUS_ENABLED:
         start_metrics_http_server(PROMETHEUS_PORT)
@@ -38,7 +78,7 @@ def export_metrics(actual_response):
             # skip management and healthcheck endpoints
             return
 
-        domain = "unknown"
+        domain = _UNKNOWN_DOMAIN
         if res.solution and res.solution.url:
             domain = parse_domain_url(res.solution.url)
         else:
@@ -59,8 +99,4 @@ def export_metrics(actual_response):
             result = "error"
         REQUEST_COUNTER.labels(domain=domain, result=result).inc()
 
-    def parse_domain_url(url):
-        parsed_url = urllib.parse.urlparse(url)
-        return parsed_url.hostname
-
     return wrapper

src/test_prometheus_labels.py

@@ -0,0 +1,68 @@
+"""The metric domain label is bounded.
+
+Prometheus keeps one time series per distinct label value for the life of the
+process and nothing evicts, so labelling by request hostname made the registry
+grow with the number of hosts a client asked for.
+
+Run: PYTHONPATH=src uv run --no-project python -m unittest test_prometheus_labels
+"""
+import unittest
+
+from bottle_plugins import prometheus_plugin
+from bottle_plugins.prometheus_plugin import parse_domain_url, reset_domain_labels
+
+
+class DomainLabelTest(unittest.TestCase):
+
+    def setUp(self):
+        reset_domain_labels()
+        self.original_cap = prometheus_plugin._MAX_DOMAIN_LABELS
+        prometheus_plugin._MAX_DOMAIN_LABELS = 3
+
+    def tearDown(self):
+        prometheus_plugin._MAX_DOMAIN_LABELS = self.original_cap
+        reset_domain_labels()
+
+    def fill_to_cap(self):
+        parse_domain_url("https://one.tld/a")
+        parse_domain_url("https://two.tld/a")
+        parse_domain_url("https://three.tld/a")
+
+    def test_a_hostname_under_the_cap_is_reported_as_itself(self):
+        self.assertEqual(parse_domain_url("https://example.tld/path"), "example.tld")
+
+    def test_a_hostname_past_the_cap_becomes_other(self):
+        self.fill_to_cap()
+        self.assertEqual(parse_domain_url("https://fourth.tld/a"), "other")
+
+    def test_a_known_hostname_still_reports_itself_once_the_cap_is_reached(self):
+        self.fill_to_cap()
+        parse_domain_url("https://fourth.tld/a")
+        self.assertEqual(parse_domain_url("https://two.tld/b"), "two.tld")
+
+    def test_repeating_one_hostname_does_not_consume_the_cap(self):
+        parse_domain_url("https://same.tld/a")
+        parse_domain_url("https://same.tld/b")
+        parse_domain_url("https://same.tld/c")
+        self.assertEqual(parse_domain_url("https://other-host.tld/a"), "other-host.tld")
+
+    def test_a_url_without_a_hostname_is_unknown(self):
+        self.assertEqual(parse_domain_url("not-a-url"), "unknown")
+
+    def test_a_missing_url_is_unknown(self):
+        self.assertEqual(parse_domain_url(None), "unknown")
+
+    def test_the_other_sentinel_does_not_itself_consume_the_cap(self):
+        self.fill_to_cap()
+        parse_domain_url("https://fourth.tld/a")
+        parse_domain_url("https://fifth.tld/a")
+        self.assertEqual(parse_domain_url("https://three.tld/z"), "three.tld")
+
+    def test_reset_clears_the_seen_set(self):
+        self.fill_to_cap()
+        reset_domain_labels()
+        self.assertEqual(parse_domain_url("https://fourth.tld/a"), "fourth.tld")
+
+
+if __name__ == '__main__':
+    unittest.main()