fix(verify): cap the per-host rate-limit backoff

app/verify/http_check.py

@@ -41,6 +41,10 @@
 MAX_RETRY_AFTER_S = 15.0
 # How much to slow a host down for the rest of the run once it has pushed back.
 RATE_LIMIT_PENALTY = 4.0
+# The penalty is multiplicative, so without a ceiling a host that refuses often
+# walks the interval up without bound (1s -> 4 -> 16 -> 64 -> ...) and a run over
+# a few thousand URLs on that host turns into hours of sleeping.
+MAX_HOST_INTERVAL_S = 30.0
 
 
 class CheckResult(NamedTuple):
@@ -184,7 +188,7 @@ def back_off(self, host: str, factor: float = RATE_LIMIT_PENALTY) -> None:
         pace and every subsequent URL on it comes back 429.
         """
         with self._lock:
-            self._interval[host] = self.interval_for(host) * factor
+            self._interval[host] = min(self.interval_for(host) * factor, MAX_HOST_INTERVAL_S)
 
     def wait(self, host: str) -> None:
         with self._lock:

tests/verify/test_http_check.py

@@ -167,3 +167,10 @@ def test_cached_rate_limit_entries_are_not_cache_hits(tmp_path):
     cache = http_check.load_cache(path)
     assert "https://en.wikipedia.org/wiki/X" in cache
     assert "https://www.gsmarena.com/a-1.php" not in cache  # a 429 is not an answer
+
+
+def test_host_backoff_is_capped():
+    limiter = http_check.HostRateLimiter(min_interval=1.0)
+    for _ in range(20):
+        limiter.back_off("gsmarena.com")
+    assert limiter.interval_for("gsmarena.com") == http_check.MAX_HOST_INTERVAL_S