diff --git a/app/verify/http_check.py b/app/verify/http_check.py index eeb3e19..b0a8d60 100644 --- a/app/verify/http_check.py +++ b/app/verify/http_check.py @@ -41,6 +41,10 @@ MAX_RETRY_AFTER_S = 15.0 # How much to slow a host down for the rest of the run once it has pushed back. RATE_LIMIT_PENALTY = 4.0 +# The penalty is multiplicative, so without a ceiling a host that refuses often +# walks the interval up without bound (1s -> 4 -> 16 -> 64 -> ...) and a run over +# a few thousand URLs on that host turns into hours of sleeping. +MAX_HOST_INTERVAL_S = 30.0 class CheckResult(NamedTuple): @@ -184,7 +188,7 @@ def back_off(self, host: str, factor: float = RATE_LIMIT_PENALTY) -> None: pace and every subsequent URL on it comes back 429. """ with self._lock: - self._interval[host] = self.interval_for(host) * factor + self._interval[host] = min(self.interval_for(host) * factor, MAX_HOST_INTERVAL_S) def wait(self, host: str) -> None: with self._lock: diff --git a/tests/verify/test_http_check.py b/tests/verify/test_http_check.py index 315ef47..6ce05ad 100644 --- a/tests/verify/test_http_check.py +++ b/tests/verify/test_http_check.py @@ -167,3 +167,10 @@ def test_cached_rate_limit_entries_are_not_cache_hits(tmp_path): cache = http_check.load_cache(path) assert "https://en.wikipedia.org/wiki/X" in cache assert "https://www.gsmarena.com/a-1.php" not in cache # a 429 is not an answer + + +def test_host_backoff_is_capped(): + limiter = http_check.HostRateLimiter(min_interval=1.0) + for _ in range(20): + limiter.back_off("gsmarena.com") + assert limiter.interval_for("gsmarena.com") == http_check.MAX_HOST_INTERVAL_S