feat: Flickr rate limit retry with backoff

.claude/settings.local.json

@@ -2,17 +2,23 @@
   "permissions": {
     "allow": [
       "Bash(wc *)",
-      "Bash(poetry show *)",
-      "WebFetch(domain:pypi.org)",
+      "WebFetch(domain:commons.wikimedia.org)",
       "WebFetch(domain:doc.wikimedia.org)",
+      "WebFetch(domain:pypi.org)",
+      "Skill(code-review:code-review)",
       "mcp__plugin_context7_context7__resolve-library-id",
       "mcp__plugin_context7_context7__query-docs",
-      "Bash(poetry run ty check *)",
       "Bash(poetry run ruff check *)",
-      "Bash(but status *)",
+      "Bash(poetry run ty check *)",
+      "Bash(poetry show *)",
+      "Bash(gh pr view *)",
+      "Bash(but branch --help)",
+      "Bash(but commit *)",
       "Bash(but pr new *)",
-      "WebFetch(domain:commons.wikimedia.org)",
-      "Bash(but commit *)"
+      "Bash(but status *)",
+      "Bash(but --help)",
+      "Bash(poetry run pytest *)",
+      "Bash(poetry run isort *)"
     ]
   },
   "spinnerTipsEnabled": false

.github/workflows/lint.yml

@@ -32,4 +32,4 @@ jobs:
         run: poetry run isort --check .
 
       - name: Run type checker
-        run: poetry run ty check
+        run: poetry run ty check src/

.github/workflows/tests.yml

@@ -0,0 +1,29 @@
+name: Tests
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.13'
+          cache: 'poetry'
+
+      - name: Install dependencies
+        run: poetry install --with dev
+
+      - name: Run tests
+        run: poetry run pytest

CLAUDE.md

@@ -20,10 +20,11 @@ poetry run ruff format
 poetry run isort .
 
 # Type check
-poetry run ty check
-```
+poetry run ty check src/
 
-Tests are not yet set up in this project.
+# Run tests
+poetry run pytest
+```
 
 ## Architecture
 

pyproject.toml

@@ -45,8 +45,12 @@ exclude = [
 dev = [
     "ruff (>=0.15.9,<0.16.0)",
     "ty (>=0.0.28,<0.0.29)",
-    "isort (>=8.0.1,<9.0.0)"
+    "isort (>=8.0.1,<9.0.0)",
+    "pytest (>=9.0.3,<10.0.0)",
+    "pytest-mock (>=3.15.1,<4.0.0)",
+    "pytest-timeout (>=2.4.0,<3.0.0)"
 ]
 
+
 [tool.isort]
 profile = "black"

pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_functions = test_*
+pythonpath = src
+timeout = 0.25

src/wikibots/flickr.py

@@ -4,18 +4,21 @@
 from datetime import datetime
 from time import perf_counter
 
+import httpx
 from flickr_api import FlickrApi
 from flickr_api.exceptions import PermissionDenied, ResourceNotFound
 from flickr_api.models import SinglePhotoInfo
 from flickr_api.models.photo import Location
 from flickr_url_parser import parse_flickr_url
 
-from wikibots.lib.bot import BaseBot
+from wikibots.lib.bot import BaseBot, RateLimitExhausted
 from wikibots.lib.claim import WBTIME_PRECISION, Claim
 from wikibots.lib.wikidata import WikidataEntity, WikidataProperty
 
 logger = logging.getLogger(__name__)
 
+RATE_LIMIT_DELAYS = (60, 180, 300)
+
 
 class FlickrBot(BaseBot):
     redis_prefix = "xQ6cz5J84Viw/K6FIcOH1kxJjfiS8jO56AoSmhBgO/A="
@@ -137,18 +140,32 @@ def get_flickr_photo(self, flickr_photo_id: str) -> None:
             logger.warning("Flickr photo skipped due to Redis cache")
             return
 
-        try:
-            start = perf_counter()
-            self.photo = self.flickr_api.get_single_photo_info(photo_id=flickr_photo_id)
-            logger.info(
-                f"Retrieved Flickr photo in {(perf_counter() - start) * 1000:.0f} ms"
-            )
-        except (ResourceNotFound, PermissionDenied) as e:
-            logger.warning(f"[{flickr_photo_id}] {e}")
-            self.redis.set(redis_key_photo, 1)
-        except Exception as e:
-            logger.error(f"[{flickr_photo_id}] {e}")
-            time.sleep(60)
+        delays = iter(RATE_LIMIT_DELAYS)
+        while True:
+            try:
+                start = perf_counter()
+                self.photo = self.flickr_api.get_single_photo_info(photo_id=flickr_photo_id)
+                logger.info(
+                    f"Retrieved Flickr photo in {(perf_counter() - start) * 1000:.0f} ms"
+                )
+                return
+            except (ResourceNotFound, PermissionDenied) as e:
+                logger.warning(f"[{flickr_photo_id}] {e}")
+                self.redis.set(redis_key_photo, 1)
+                return
+            except httpx.HTTPStatusError as e:
+                if e.response.status_code != 429:
+                    logger.error(f"[{flickr_photo_id}] {e}")
+                    return
+                delay = next(delays, None)
+                if delay is None:
+                    logger.critical(f"[{flickr_photo_id}] Rate limit exhausted after all retries")
+                    raise RateLimitExhausted
+                logger.warning(f"[{flickr_photo_id}] Rate limited, retrying in {delay}s")
+                time.sleep(delay)
+            except Exception as e:
+                logger.error(f"[{flickr_photo_id}] {e}")
+                return
 
     def hook_creator_claim(self, claim: Claim) -> None:
         if not self.photo:

src/wikibots/lib/bot.py

@@ -22,7 +22,11 @@
 WIKIDATA_SPARQL = "https://query.wikidata.org/sparql"
 
 
-class _DryRunStop(Exception):
+class DryRunStop(Exception):
+    pass
+
+
+class RateLimitExhausted(Exception):
     pass
 
 
@@ -139,7 +143,7 @@ def run(self) -> None:
 
                 self.treat_page()
                 time.sleep(self.throttle)
-        except _DryRunStop:
+        except (DryRunStop, RateLimitExhausted):
             pass
 
     def skip_page(self, page: dict[str, Any]) -> bool:
@@ -269,7 +273,7 @@ def save(self) -> None:
 
             if self.dry_run:
                 logger.info("Dry run mode: skipping save operation")
-                raise _DryRunStop()
+                raise DryRunStop()
 
             if self.always_null_edit:
                 logger.info("Performing null edit to flush any tracker categories")
@@ -283,7 +287,7 @@ def save(self) -> None:
 
         if self.dry_run:
             logger.info("Dry run mode: skipping save operation")
-            raise _DryRunStop()
+            raise DryRunStop()
 
         try:
             start = perf_counter()