From d1133ae844cd276c56c2c79a5cf3b24af52067dd Mon Sep 17 00:00:00 2001 From: Roman Kalyakin Date: Thu, 18 Jun 2026 13:39:23 +0200 Subject: [PATCH 1/2] feat(api-client): add retry logic with exponential backoff and jitter --- .vscode/settings.json | 6 +- impresso/api_client/client.py | 46 ++- impresso/api_client/retry.py | 288 +++++++++++++++ impresso/client.py | 2 + pyproject.toml | 2 +- tests/impresso/test_api_client_retry.py | 454 ++++++++++++++++++++++++ 6 files changed, 783 insertions(+), 15 deletions(-) create mode 100644 impresso/api_client/retry.py create mode 100644 tests/impresso/test_api_client_retry.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 0b017f0..1363c68 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,5 +10,9 @@ }, "python.testing.pytestArgs": ["-vv"], "editor.formatOnSave": true, - "pylint.args": ["--generate-members"] + "pylint.args": [ + "--generate-members" + ], + "python-envs.defaultEnvManager": "ms-python.python:poetry", + "python-envs.defaultPackageManager": "ms-python.python:poetry" } diff --git a/impresso/api_client/client.py b/impresso/api_client/client.py index 63a2493..2b155c2 100644 --- a/impresso/api_client/client.py +++ b/impresso/api_client/client.py @@ -4,6 +4,8 @@ import httpx from attrs import define, evolve, field +from impresso.api_client.retry import AsyncRetryingClient, RetryingClient + @define class Client: @@ -38,9 +40,15 @@ class Client: _base_url: str = field(alias="base_url") _cookies: Dict[str, str] = field(factory=dict, kw_only=True, alias="cookies") _headers: Dict[str, str] = field(factory=dict, kw_only=True, alias="headers") - _timeout: Optional[httpx.Timeout] = field(default=None, kw_only=True, alias="timeout") - _verify_ssl: Union[str, bool, ssl.SSLContext] = field(default=True, kw_only=True, alias="verify_ssl") - _follow_redirects: bool = field(default=False, kw_only=True, alias="follow_redirects") + _timeout: Optional[httpx.Timeout] = field( + default=None, kw_only=True, alias="timeout" + ) + _verify_ssl: Union[str, bool, ssl.SSLContext] = field( + default=True, kw_only=True, alias="verify_ssl" + ) + _follow_redirects: bool = field( + default=False, kw_only=True, alias="follow_redirects" + ) _httpx_args: Dict[str, Any] = field(factory=dict, kw_only=True, alias="httpx_args") _client: Optional[httpx.Client] = field(default=None, init=False) _async_client: Optional[httpx.AsyncClient] = field(default=None, init=False) @@ -80,7 +88,7 @@ def set_httpx_client(self, client: httpx.Client) -> "Client": def get_httpx_client(self) -> httpx.Client: """Get the underlying httpx.Client, constructing a new one if not previously set""" if self._client is None: - self._client = httpx.Client( + self._client = RetryingClient( base_url=self._base_url, cookies=self._cookies, headers=self._headers, @@ -111,7 +119,7 @@ def set_async_httpx_client(self, async_client: httpx.AsyncClient) -> "Client": def get_async_httpx_client(self) -> httpx.AsyncClient: """Get the underlying httpx.AsyncClient, constructing a new one if not previously set""" if self._async_client is None: - self._async_client = httpx.AsyncClient( + self._async_client = AsyncRetryingClient( base_url=self._base_url, cookies=self._cookies, headers=self._headers, @@ -168,9 +176,15 @@ class AuthenticatedClient: _base_url: str = field(alias="base_url") _cookies: Dict[str, str] = field(factory=dict, kw_only=True, alias="cookies") _headers: Dict[str, str] = field(factory=dict, kw_only=True, alias="headers") - _timeout: Optional[httpx.Timeout] = field(default=None, kw_only=True, alias="timeout") - _verify_ssl: Union[str, bool, ssl.SSLContext] = field(default=True, kw_only=True, alias="verify_ssl") - _follow_redirects: bool = field(default=False, kw_only=True, alias="follow_redirects") + _timeout: Optional[httpx.Timeout] = field( + default=None, kw_only=True, alias="timeout" + ) + _verify_ssl: Union[str, bool, ssl.SSLContext] = field( + default=True, kw_only=True, alias="verify_ssl" + ) + _follow_redirects: bool = field( + default=False, kw_only=True, alias="follow_redirects" + ) _httpx_args: Dict[str, Any] = field(factory=dict, kw_only=True, alias="httpx_args") _client: Optional[httpx.Client] = field(default=None, init=False) _async_client: Optional[httpx.AsyncClient] = field(default=None, init=False) @@ -214,8 +228,10 @@ def set_httpx_client(self, client: httpx.Client) -> "AuthenticatedClient": def get_httpx_client(self) -> httpx.Client: """Get the underlying httpx.Client, constructing a new one if not previously set""" if self._client is None: - self._headers[self.auth_header_name] = f"{self.prefix} {self.token}" if self.prefix else self.token - self._client = httpx.Client( + self._headers[self.auth_header_name] = ( + f"{self.prefix} {self.token}" if self.prefix else self.token + ) + self._client = RetryingClient( base_url=self._base_url, cookies=self._cookies, headers=self._headers, @@ -235,7 +251,9 @@ def __exit__(self, *args: Any, **kwargs: Any) -> None: """Exit a context manager for internal httpx.Client (see httpx docs)""" self.get_httpx_client().__exit__(*args, **kwargs) - def set_async_httpx_client(self, async_client: httpx.AsyncClient) -> "AuthenticatedClient": + def set_async_httpx_client( + self, async_client: httpx.AsyncClient + ) -> "AuthenticatedClient": """Manually the underlying httpx.AsyncClient **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout. @@ -246,8 +264,10 @@ def set_async_httpx_client(self, async_client: httpx.AsyncClient) -> "Authentica def get_async_httpx_client(self) -> httpx.AsyncClient: """Get the underlying httpx.AsyncClient, constructing a new one if not previously set""" if self._async_client is None: - self._headers[self.auth_header_name] = f"{self.prefix} {self.token}" if self.prefix else self.token - self._async_client = httpx.AsyncClient( + self._headers[self.auth_header_name] = ( + f"{self.prefix} {self.token}" if self.prefix else self.token + ) + self._async_client = AsyncRetryingClient( base_url=self._base_url, cookies=self._cookies, headers=self._headers, diff --git a/impresso/api_client/retry.py b/impresso/api_client/retry.py new file mode 100644 index 0000000..b44ba74 --- /dev/null +++ b/impresso/api_client/retry.py @@ -0,0 +1,288 @@ +"""Retry-aware HTTP clients for generated API endpoint calls. + +At a glance: +- This module is used by ``Client`` and ``AuthenticatedClient`` when they build + their internal httpx clients, so generated endpoint functions automatically get + retry behavior without changing their public API. +- Retried responses are server-side failures (5xx), rate limits (429), teapots + (418), and WAF-looking 401/403 responses whose bodies are HTML instead of the + normal JSON auth errors. +- Retried transport failures are currently limited to ``httpx.ReadError``, which + covers connection resets such as "[Errno 54] Connection reset by peer". +- Each retry waits with exponential backoff, small jitter, and ``Retry-After`` + support when the API provides it. +- Console messages are intentionally user-facing: they say what will happen next + first, then explain why, with gentle barista-flavored humor. +- If all retries are exhausted, the final HTTP response is returned as usual, or + the original transport exception is re-raised after one final user-facing hint. +""" + +from __future__ import annotations + +import asyncio +import random +import time +from email.utils import parsedate_to_datetime +from typing import Callable + +import httpx + +MAX_RETRIES = 5 +MAX_ATTEMPTS = MAX_RETRIES + 1 +BASE_BACKOFF_SECONDS = 1.0 +MAX_BACKOFF_SECONDS = 16.0 +JITTER_SECONDS = 0.25 +RETRYABLE_AUTH_STATUSES = {401, 403} +RETRYABLE_EXCEPTIONS = (httpx.ReadError,) + + +def _response_looks_like_html(response: httpx.Response) -> bool: + content_type = response.headers.get("content-type", "").lower() + if "html" in content_type: + return True + + body_start = response.content[:512].lstrip().lower() + return body_start.startswith(b" bool: + """Return whether an API response matches the retry policy.""" + + if response.status_code >= 500 or response.status_code in {418, 429}: + return True + if response.status_code in RETRYABLE_AUTH_STATUSES: + return _response_looks_like_html(response) + return False + + +def _retry_after_seconds(value: str | None) -> float | None: + if not value: + return None + + try: + delay = float(value) + except ValueError: + try: + retry_at = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + delay = retry_at.timestamp() - time.time() + + return max(0.0, min(delay, MAX_BACKOFF_SECONDS)) + + +def _backoff_seconds(response: httpx.Response, retry_number: int) -> float: + retry_after = _retry_after_seconds(response.headers.get("retry-after")) + if retry_after is not None: + return retry_after + + exponential_backoff = min( + BASE_BACKOFF_SECONDS * (2 ** (retry_number - 1)), + MAX_BACKOFF_SECONDS, + ) + return min( + exponential_backoff + random.uniform(0, JITTER_SECONDS), MAX_BACKOFF_SECONDS + ) + + +def _exception_backoff_seconds(retry_number: int) -> float: + exponential_backoff = min( + BASE_BACKOFF_SECONDS * (2 ** (retry_number - 1)), + MAX_BACKOFF_SECONDS, + ) + return min( + exponential_backoff + random.uniform(0, JITTER_SECONDS), MAX_BACKOFF_SECONDS + ) + + +def _format_url(request: httpx.Request) -> str: + return request.url.raw_path.decode("ascii", errors="ignore") or str(request.url) + + +def _response_retry_explanation(response: httpx.Response) -> str: + if response.status_code == 429: + return "The API asked us to slow down the pour." + return "The API had a wobbly espresso shot." + + +def _print_retry_message( + *, + request: httpx.Request, + response: httpx.Response, + retry_number: int, + delay_seconds: float, +) -> None: + print( + "☕ I will try again " + f"in {delay_seconds:.1f}s ({retry_number}/{MAX_RETRIES}). " + f"{_response_retry_explanation(response)} " + f"Reason: HTTP {response.status_code} from {request.method} " + f"{_format_url(request)}." + ) + + +def _print_retry_exception_message( + *, + request: httpx.Request, + exception: Exception, + retry_number: int, + delay_seconds: float, +) -> None: + print( + "☕ I will try again " + f"in {delay_seconds:.1f}s ({retry_number}/{MAX_RETRIES}). " + "The API spilled the milk mid-pour. " + f"Reason: {exception.__class__.__name__} for {request.method} " + f"{_format_url(request)} ({exception})." + ) + + +def _print_retries_exhausted_message( + *, + request: httpx.Request, + reason: str, +) -> None: + if reason == "HTTP 429": + advice = ( + "☕ Sorry, the API is still asking us to slow down. " + "Please retry in a few minutes. If it keeps happening, contact the " + "Impresso team. My espresso queue has become a very tiny traffic jam. " + ) + else: + advice = ( + "☕ Sorry, I tried a few times and the API still is not cooperating. " + "Please retry in a few minutes. If it keeps happening, contact the " + "Impresso team. My espresso tamper has officially filed a complaint. " + ) + print(f"{advice}Reason: {reason} for {request.method} {_format_url(request)}.") + + +class RetryingClient(httpx.Client): + """Synchronous httpx client that retries flaky Impresso API responses.""" + + def __init__( + self, + *args: object, + sleep: Callable[[float], None] = time.sleep, + **kwargs: object, + ) -> None: + super().__init__(*args, **kwargs) + self._sleep = sleep + + def send(self, request: httpx.Request, **kwargs: object) -> httpx.Response: + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + response = super().send(request, **kwargs) + except RETRYABLE_EXCEPTIONS as exc: + if attempt == MAX_ATTEMPTS: + _print_retries_exhausted_message( + request=request, + reason=f"{exc.__class__.__name__} ({exc})", + ) + raise + + retry_number = attempt + delay_seconds = _exception_backoff_seconds(retry_number) + _print_retry_exception_message( + request=request, + exception=exc, + retry_number=retry_number, + delay_seconds=delay_seconds, + ) + self._sleep(delay_seconds) + continue + + retryable_response = is_retryable_response(response) + if attempt == MAX_ATTEMPTS: + if retryable_response: + response.read() + _print_retries_exhausted_message( + request=request, + reason=f"HTTP {response.status_code}", + ) + return response + if not retryable_response: + return response + + response.read() + retry_number = attempt + delay_seconds = _backoff_seconds(response, retry_number) + _print_retry_message( + request=request, + response=response, + retry_number=retry_number, + delay_seconds=delay_seconds, + ) + response.close() + self._sleep(delay_seconds) + + raise RuntimeError("Retry loop exited unexpectedly") + + +class AsyncRetryingClient(httpx.AsyncClient): + """Asynchronous httpx client that retries flaky Impresso API responses.""" + + def __init__( + self, + *args: object, + sleep: Callable[[float], object] = asyncio.sleep, + **kwargs: object, + ) -> None: + super().__init__(*args, **kwargs) + self._sleep = sleep + + async def send(self, request: httpx.Request, **kwargs: object) -> httpx.Response: + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + response = await super().send(request, **kwargs) + except RETRYABLE_EXCEPTIONS as exc: + if attempt == MAX_ATTEMPTS: + _print_retries_exhausted_message( + request=request, + reason=f"{exc.__class__.__name__} ({exc})", + ) + raise + + retry_number = attempt + delay_seconds = _exception_backoff_seconds(retry_number) + _print_retry_exception_message( + request=request, + exception=exc, + retry_number=retry_number, + delay_seconds=delay_seconds, + ) + await self._sleep(delay_seconds) + continue + + retryable_response = is_retryable_response(response) + if attempt == MAX_ATTEMPTS: + if retryable_response: + await response.aread() + _print_retries_exhausted_message( + request=request, + reason=f"HTTP {response.status_code}", + ) + return response + if not retryable_response: + return response + + await response.aread() + retry_number = attempt + delay_seconds = _backoff_seconds(response, retry_number) + _print_retry_message( + request=request, + response=response, + retry_number=retry_number, + delay_seconds=delay_seconds, + ) + await response.aclose() + await self._sleep(delay_seconds) + + raise RuntimeError("Retry loop exited unexpectedly") + + +__all__ = [ + "AsyncRetryingClient", + "RetryingClient", + "is_retryable_response", +] diff --git a/impresso/client.py b/impresso/client.py index 33d3979..bc46e5a 100644 --- a/impresso/client.py +++ b/impresso/client.py @@ -9,6 +9,7 @@ import httpx from impresso.api_client import AuthenticatedClient +from impresso.api_client.retry import is_retryable_response from impresso.client_base import ImpressoApiResourcesBase from impresso.config_file import DEFAULT_API_URL, ImpressoPyConfig from impresso.util.token import get_jwt_status @@ -28,6 +29,7 @@ def _is_localhost_netloc(netloc: str) -> bool: def _log_non_2xx(response: httpx.Response) -> None: if response.status_code >= 400: response.read() + if response.status_code >= 400 and not is_retryable_response(response): logging.error( f"Received error response ({response.status_code}): {response.text}" ) diff --git a/pyproject.toml b/pyproject.toml index 99ac8c7..4575996 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ name = "impresso" packages = [{ include = "impresso", from = "." }] readme = "README.md" repository = "https://github.com/impresso/impresso-py" -version = "0.9.17" +version = "0.9.18" [tool.poetry.urls] Endpoint = "https://impresso-project.ch/public-api/v1" diff --git a/tests/impresso/test_api_client_retry.py b/tests/impresso/test_api_client_retry.py new file mode 100644 index 0000000..0c7dd80 --- /dev/null +++ b/tests/impresso/test_api_client_retry.py @@ -0,0 +1,454 @@ +import asyncio + +import pytest +import httpx + +from impresso.api_client import Client +from impresso.api_client.api.reference_data import get_data_sources_csv_export +from impresso.api_client.retry import AsyncRetryingClient, RetryingClient + + +def _client_with_responses( + responses: list[httpx.Response], +) -> tuple[Client, list[httpx.Request]]: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + response = responses.pop(0) + return httpx.Response( + response.status_code, + headers=response.headers, + content=response.content, + request=request, + ) + + client = Client( + base_url="https://example.test", + httpx_args={"transport": httpx.MockTransport(handler)}, + ) + return client, requests + + +def _response( + status_code: int, + content: bytes | str = b"", + headers: dict[str, str] | None = None, +) -> httpx.Response: + if isinstance(content, str): + content = content.encode() + return httpx.Response(status_code, headers=headers, content=content) + + +def _disable_sync_sleep(client: Client) -> list[float]: + delays: list[float] = [] + httpx_client = client.get_httpx_client() + assert isinstance(httpx_client, RetryingClient) + httpx_client._sleep = delays.append + return delays + + +async def _disable_async_sleep(client: Client) -> list[float]: + delays: list[float] = [] + httpx_client = client.get_async_httpx_client() + assert isinstance(httpx_client, AsyncRetryingClient) + + async def sleep(delay: float) -> None: + delays.append(delay) + + httpx_client._sleep = sleep + return delays + + +def test_retries_500_responses_then_returns_success( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + client, requests = _client_with_responses( + [ + _response(500, b'{"type":"error","title":"Error","status":500}'), + _response(500, b'{"type":"error","title":"Error","status":500}'), + _response(200, "id,label\n1,Source\n"), + ] + ) + delays = _disable_sync_sleep(client) + + result = get_data_sources_csv_export.sync(client=client) + + assert result == "id,label\n1,Source\n" + assert len(requests) == 3 + assert delays == [1.0, 2.0] + captured = capsys.readouterr() + assert "☕ I will try again in 1.0s (1/5)" in captured.out + assert "☕ I will try again in 2.0s (2/5)" in captured.out + assert "The API had a wobbly espresso shot" in captured.out + assert "Reason: HTTP 500 from GET /reference-data/data-sources.csv" in captured.out + + +def test_retries_418_response(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + client, requests = _client_with_responses( + [ + _response(418, b'{"type":"error","title":"Error","status":418}'), + _response(200, "ok\n"), + ] + ) + delays = _disable_sync_sleep(client) + + result = get_data_sources_csv_export.sync(client=client) + + assert result == "ok\n" + assert len(requests) == 2 + assert delays == [1.0] + + +def test_retries_429_response_with_backoff( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + client, requests = _client_with_responses( + [ + _response( + 429, b'{"type":"error","title":"Too Many Requests","status":429}' + ), + _response(200, "ok\n"), + ] + ) + delays = _disable_sync_sleep(client) + + result = get_data_sources_csv_export.sync(client=client) + + assert result == "ok\n" + assert len(requests) == 2 + assert delays == [1.0] + captured = capsys.readouterr() + assert "☕ I will try again in 1.0s (1/5)" in captured.out + assert "The API asked us to slow down the pour" in captured.out + assert "Reason: HTTP 429 from GET /reference-data/data-sources.csv" in captured.out + + +def test_retries_429_response_with_retry_after_header( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + client, requests = _client_with_responses( + [ + _response( + 429, + b'{"type":"error","title":"Too Many Requests","status":429}', + {"retry-after": "7"}, + ), + _response(200, "ok\n"), + ] + ) + delays = _disable_sync_sleep(client) + + result = get_data_sources_csv_export.sync(client=client) + + assert result == "ok\n" + assert len(requests) == 2 + assert delays == [7.0] + captured = capsys.readouterr() + assert "☕ I will try again in 7.0s (1/5)" in captured.out + assert "The API asked us to slow down the pour" in captured.out + + +def test_exhausting_429_retries_prints_rate_limit_message( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + client, requests = _client_with_responses( + [ + _response( + 429, + b'{"type":"error","title":"Too Many Requests","status":429}', + ) + for _ in range(6) + ] + ) + delays = _disable_sync_sleep(client) + + result = get_data_sources_csv_export.sync_detailed(client=client) + + assert result.status_code == 429 + assert len(requests) == 6 + assert delays == [1.0, 2.0, 4.0, 8.0, 16.0] + captured = capsys.readouterr() + assert "the API is still asking us to slow down" in captured.out + assert "Please retry in a few minutes" in captured.out + assert "contact the Impresso team" in captured.out + assert "espresso queue has become a very tiny traffic jam" in captured.out + assert "Reason: HTTP 429 for GET /reference-data/data-sources.csv" in captured.out + + +@pytest.mark.parametrize("status_code", [401, 403]) +def test_retries_html_auth_responses(status_code: int, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + client, requests = _client_with_responses( + [ + _response( + status_code, + "blocked", + {"content-type": "text/html"}, + ), + _response(200, "ok\n"), + ] + ) + delays = _disable_sync_sleep(client) + + result = get_data_sources_csv_export.sync(client=client) + + assert result == "ok\n" + assert len(requests) == 2 + assert delays == [1.0] + + +@pytest.mark.parametrize("status_code", [401, 403]) +def test_does_not_retry_json_auth_responses( + status_code: int, capsys: pytest.CaptureFixture[str] +): + client, requests = _client_with_responses( + [ + _response( + status_code, + b'{"type":"auth","title":"Auth error","status":401}', + {"content-type": "application/json"}, + ), + _response(200, "should not be used\n"), + ] + ) + delays = _disable_sync_sleep(client) + + result = get_data_sources_csv_export.sync(client=client) + + assert result is None + assert len(requests) == 1 + assert delays == [] + assert capsys.readouterr().out == "" + + +def test_exhausting_retries_returns_final_response( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + client, requests = _client_with_responses( + [ + _response(500, b'{"type":"error","title":"Error","status":500}') + for _ in range(6) + ] + ) + delays = _disable_sync_sleep(client) + + result = get_data_sources_csv_export.sync(client=client) + + assert result.status == 500 + assert len(requests) == 6 + assert delays == [1.0, 2.0, 4.0, 8.0, 16.0] + captured = capsys.readouterr() + assert "Sorry, I tried a few times" in captured.out + assert "Please retry in a few minutes" in captured.out + assert "contact the Impresso team" in captured.out + assert "espresso tamper has officially filed a complaint" in captured.out + assert "Reason: HTTP 500 for GET /reference-data/data-sources.csv" in captured.out + + +def test_retry_after_header_overrides_backoff(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + client, _requests = _client_with_responses( + [ + _response( + 503, + b'{"type":"error","title":"Error","status":503}', + {"retry-after": "3"}, + ), + _response(200, "ok\n"), + ] + ) + delays = _disable_sync_sleep(client) + + result = get_data_sources_csv_export.sync(client=client) + + assert result == "ok\n" + assert delays == [3.0] + + +def test_retries_post_request_with_json_body(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + bodies: list[bytes] = [] + + def handler(request: httpx.Request) -> httpx.Response: + bodies.append(request.content) + status_code = 500 if len(bodies) == 1 else 200 + return httpx.Response(status_code, content=b"ok", request=request) + + client = Client( + base_url="https://example.test", + httpx_args={"transport": httpx.MockTransport(handler)}, + ) + delays = _disable_sync_sleep(client) + + response = client.get_httpx_client().post("/collections", json={"name": "demo"}) + + assert response.status_code == 200 + assert bodies == [b'{"name": "demo"}', b'{"name": "demo"}'] + assert delays == [1.0] + + +def test_retries_read_error_then_returns_success( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if len(requests) == 1: + raise httpx.ReadError("[Errno 54] Connection reset by peer") + return httpx.Response(200, content=b"ok\n", request=request) + + client = Client( + base_url="https://example.test", + httpx_args={"transport": httpx.MockTransport(handler)}, + ) + delays = _disable_sync_sleep(client) + + result = get_data_sources_csv_export.sync(client=client) + + assert result == "ok\n" + assert len(requests) == 2 + assert delays == [1.0] + captured = capsys.readouterr() + assert "☕ I will try again in 1.0s (1/5)" in captured.out + assert "The API spilled the milk mid-pour" in captured.out + assert "Reason: ReadError for GET /reference-data/data-sources.csv" in captured.out + assert "Connection reset by peer" in captured.out + + +def test_exhausting_read_error_retries_raises_original_error( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +): + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + raise httpx.ReadError("[Errno 54] Connection reset by peer") + + client = Client( + base_url="https://example.test", + httpx_args={"transport": httpx.MockTransport(handler)}, + ) + delays = _disable_sync_sleep(client) + + with pytest.raises(httpx.ReadError, match="Connection reset by peer"): + get_data_sources_csv_export.sync(client=client) + + assert len(requests) == 6 + assert delays == [1.0, 2.0, 4.0, 8.0, 16.0] + captured = capsys.readouterr() + assert "Sorry, I tried a few times" in captured.out + assert "Please retry in a few minutes" in captured.out + assert "contact the Impresso team" in captured.out + assert "Reason: ReadError" in captured.out + assert "Connection reset by peer" in captured.out + + +def test_async_retries_html_auth_response(monkeypatch: pytest.MonkeyPatch): + async def run_test() -> None: + delays = await _disable_async_sleep(client) + + result = await get_data_sources_csv_export.asyncio(client=client) + + assert result == "ok\n" + assert len(requests) == 2 + assert delays == [1.0] + + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + client, requests = _client_with_responses( + [ + _response(401, "blocked", {"content-type": "text/html"}), + _response(200, "ok\n"), + ] + ) + + asyncio.run(run_test()) + + +def test_async_retries_read_error_then_returns_success( + monkeypatch: pytest.MonkeyPatch, +): + async def run_test() -> None: + delays = await _disable_async_sleep(client) + + result = await get_data_sources_csv_export.asyncio(client=client) + + assert result == "ok\n" + assert len(requests) == 2 + assert delays == [1.0] + + monkeypatch.setattr( + "impresso.api_client.retry.random.uniform", lambda _min, _max: 0 + ) + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if len(requests) == 1: + raise httpx.ReadError("[Errno 54] Connection reset by peer") + return httpx.Response(200, content=b"ok\n", request=request) + + client = Client( + base_url="https://example.test", + httpx_args={"transport": httpx.MockTransport(handler)}, + ) + + asyncio.run(run_test()) + + +def test_async_does_not_retry_json_auth_response(capsys: pytest.CaptureFixture[str]): + async def run_test() -> None: + delays = await _disable_async_sleep(client) + + result = await get_data_sources_csv_export.asyncio(client=client) + + assert result is None + assert len(requests) == 1 + assert delays == [] + assert capsys.readouterr().out == "" + + client, requests = _client_with_responses( + [ + _response( + 403, + b'{"type":"auth","title":"Auth error","status":403}', + {"content-type": "application/json"}, + ), + _response(200, "should not be used\n"), + ] + ) + + asyncio.run(run_test()) From c801cb64c2e5bfffa6b7dee5f43965dede4412da Mon Sep 17 00:00:00 2001 From: Roman Kalyakin Date: Thu, 18 Jun 2026 13:44:30 +0200 Subject: [PATCH 2/2] linter --- tests/impresso/test_api_client_retry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/impresso/test_api_client_retry.py b/tests/impresso/test_api_client_retry.py index 0c7dd80..d52173b 100644 --- a/tests/impresso/test_api_client_retry.py +++ b/tests/impresso/test_api_client_retry.py @@ -253,9 +253,9 @@ def test_exhausting_retries_returns_final_response( ) delays = _disable_sync_sleep(client) - result = get_data_sources_csv_export.sync(client=client) + result = get_data_sources_csv_export.sync_detailed(client=client) - assert result.status == 500 + assert result.status_code == 500 assert len(requests) == 6 assert delays == [1.0, 2.0, 4.0, 8.0, 16.0] captured = capsys.readouterr()