fix(hardcover): defensive rate limiting and 429 backoff

shelfmark/metadata_providers/hardcover.py

@@ -1,12 +1,15 @@
 """Hardcover.app metadata provider. Requires API key."""
 
+import math
+import random
 import re
+import threading
 import time
 from contextlib import suppress
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from http import HTTPStatus
-from typing import Any, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
 from urllib.parse import urlparse
 
 import requests
@@ -43,6 +46,9 @@
 
 logger = setup_logger(__name__)
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 HARDCOVER_API_URL = "https://api.hardcover.app/v1/graphql"
 HARDCOVER_PAGE_SIZE = 25  # Hardcover API returns max 25 results per page
 HARDCOVER_MIN_AUTHOR_PARTS = 2
@@ -507,6 +513,83 @@ class HardcoverGraphQLError(ValueError):
     """GraphQL request was rejected by Hardcover."""
 
 
+class HardcoverRateLimitError(RuntimeError):
+    """Hardcover kept answering 429 after the retries, or asked for a wait too long to sit out."""
+
+
+# Hardcover's documented limits (docs.hardcover.app, Getting Started > Rate Limits): a token
+# bucket of 10 (Free) that refills at 60 per minute, 5,000 requests a day. The per-user limit
+# is shared with every other app and session on the same account, so pace below it.
+HARDCOVER_RATE_BURST = 5
+HARDCOVER_RATE_PER_SECOND = 0.8
+HARDCOVER_RATE_LIMIT_ATTEMPTS = 3  # the first try plus two retries
+HARDCOVER_RETRY_AFTER_CAP_SECONDS = 30.0  # longer than this (spent daily quota): give up
+
+
+class _TokenBucket:
+    """Thread-safe token bucket. ``acquire`` blocks until a request may go out.
+
+    The clock and sleep are injectable so tests never wait for real.
+    """
+
+    def __init__(
+        self,
+        capacity: int,
+        per_second: float,
+        clock: Callable[[], float] = time.monotonic,
+        sleep: Callable[[float], None] = time.sleep,
+    ) -> None:
+        self._capacity = float(capacity)
+        self._per_second = per_second
+        self._clock = clock
+        self._sleep = sleep
+        self._lock = threading.Lock()
+        self._tokens = float(capacity)
+        self._updated = clock()
+
+    def _take_or_wait(self) -> float:
+        """Take a token and return 0, or return how long to wait before trying again."""
+        with self._lock:
+            now = self._clock()
+            if now > self._updated:
+                gained = (now - self._updated) * self._per_second
+                self._tokens = min(self._capacity, self._tokens + gained)
+                self._updated = now
+            if self._tokens >= 1.0:
+                self._tokens -= 1.0
+                return 0.0
+            # _updated can sit in the future after penalize(): nothing refills before then.
+            return (self._updated - now) + (1.0 - self._tokens) / self._per_second
+
+    def acquire(self) -> None:
+        while (wait := self._take_or_wait()) > 0.0:
+            self._sleep(wait)
+
+    def penalize(self, seconds: float) -> None:
+        """Empty the bucket and hold every caller back for ``seconds``."""
+        with self._lock:
+            self._tokens = 0.0
+            self._updated = max(self._updated, self._clock() + seconds)
+
+
+_rate_limiter = _TokenBucket(HARDCOVER_RATE_BURST, HARDCOVER_RATE_PER_SECOND)
+
+
+def _retry_jitter() -> float:
+    return random.uniform(0.0, 1.0)  # noqa: S311 - spreading retries, not security
+
+
+def _throttle_wait_seconds(response: requests.Response, attempt: int) -> float:
+    """Seconds to wait after a 429: Retry-After when sent, else exponential backoff."""
+    raw = response.headers.get("Retry-After")
+    if raw is not None:
+        with suppress(ValueError):
+            seconds = float(raw)
+            if math.isfinite(seconds) and seconds >= 0:
+                return seconds
+    return float(2**attempt) + _retry_jitter()
+
+
 class HardcoverTargetPayloadError(RuntimeError):
     """Hardcover returned an invalid payload while loading book targets."""
 
@@ -2557,7 +2640,9 @@ def get_book(self, book_id: str) -> BookMetadata | None:
 
         try:
             book_id_int = int(book_id)
-            result = self._execute_query(graphql_query, {"id": book_id_int})
+            # A failed lookup is not "no such book": let throttling and outages reach the
+            # caller instead of coming back as None.
+            result = self._execute_query(graphql_query, {"id": book_id_int}, raise_on_error=True)
             if not result:
                 return None
 
@@ -2567,6 +2652,9 @@ def get_book(self, book_id: str) -> BookMetadata | None:
 
             return self._parse_book(books[0])
 
+        except HardcoverGraphQLError:
+            logger.exception("Hardcover rejected the book lookup for %s", book_id)
+            return None
         except ValueError:
             logger.exception("Invalid book ID: %s", book_id)
             return None
@@ -2645,6 +2733,40 @@ def search_by_isbn(self, isbn: str) -> BookMetadata | None:
             logger.exception("Hardcover ISBN search error")
             return None
 
+    def _post_with_backoff(self, payload: dict[str, Any]) -> requests.Response:
+        """POST one GraphQL request, paced by the shared limiter, retrying a 429.
+
+        Raises HardcoverRateLimitError when the retries run out or Hardcover asks for a
+        wait longer than HARDCOVER_RETRY_AFTER_CAP_SECONDS.
+        """
+        for attempt in range(1, HARDCOVER_RATE_LIMIT_ATTEMPTS + 1):
+            _rate_limiter.acquire()
+            response = self.session.post(
+                HARDCOVER_API_URL,
+                json=payload,
+                timeout=15,
+                verify=get_ssl_verify(HARDCOVER_API_URL),
+            )
+            if response.status_code != HTTPStatus.TOO_MANY_REQUESTS:
+                return response
+
+            wait = _throttle_wait_seconds(response, attempt)
+            if wait > HARDCOVER_RETRY_AFTER_CAP_SECONDS:
+                msg = f"Hardcover rate limit reached; it asked for a {wait:.0f}s wait"
+                raise HardcoverRateLimitError(msg)
+            if attempt == HARDCOVER_RATE_LIMIT_ATTEMPTS:
+                msg = f"Hardcover rate limit still hit after {attempt} attempts"
+                raise HardcoverRateLimitError(msg)
+            logger.warning(
+                "Hardcover rate limited (429); retrying in %.0fs (attempt %d/%d)",
+                wait,
+                attempt,
+                HARDCOVER_RATE_LIMIT_ATTEMPTS,
+            )
+            _rate_limiter.penalize(wait)
+        msg = "unreachable"  # the loop always returns or raises
+        raise AssertionError(msg)
+
     def _execute_query(
         self,
         query: str,
@@ -2658,12 +2780,7 @@ def _raise_graphql_error(message: str) -> None:
             raise HardcoverGraphQLError(message)
 
         try:
-            response = self.session.post(
-                HARDCOVER_API_URL,
-                json={"query": query, "variables": variables},
-                timeout=15,
-                verify=get_ssl_verify(HARDCOVER_API_URL),
-            )
+            response = self._post_with_backoff({"query": query, "variables": variables})
             response.raise_for_status()
 
             data = response.json()
@@ -2699,6 +2816,11 @@ def _raise_graphql_error(message: str) -> None:
             return None
         except HardcoverGraphQLError:
             raise
+        except HardcoverRateLimitError as e:
+            logger.warning("Hardcover API: %s", e)
+            if raise_on_error:
+                raise
+            return None
         except ValueError as e:
             logger.exception("Hardcover API returned invalid JSON")
             if raise_on_error: