From fddcaae1d830c9203ca0d299e9d5f385b5e5af8e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 25 Oct 2026 12:26:43 +0000 Subject: [PATCH] Retry transient GETs while restoring the current research site. AUTHOR-SITE-RESTORE-RETRY-01 / MEAS-SITE-RESTORE-FLAKE-01 Bound GitHubAPI.request and HistoryReader.get_bytes to four GET attempts within 30 seconds for HTTP 429/500/502/503/504 and URL errors. Ref writes are not retried, and each logical read still counts once toward the existing request budgets. Co-authored-by: Dylan Roy --- tests/test_current_site_history.py | 141 +++++++++++++++++++++ tests/test_current_site_publication.py | 162 +++++++++++++++++++++++++ tools/current_site_history.py | 151 ++++++++++++++++++++--- tools/current_site_publication.py | 143 ++++++++++++++++++++-- 4 files changed, 574 insertions(+), 23 deletions(-) diff --git a/tests/test_current_site_history.py b/tests/test_current_site_history.py index 038b8037b..a641ebb86 100644 --- a/tests/test_current_site_history.py +++ b/tests/test_current_site_history.py @@ -531,6 +531,147 @@ def open(self, request, timeout): return response with patch.object(self.history, "MAX_FILE_BYTES", 2), self.assertRaises(ValueError): self.history.HistoryReader(opener=ControlledOpener()).get_bytes("snapshots/index.json") + def test_history_reader_retries_transient_503_then_200(self): + history = self.history + self.assertEqual(history.GET_RETRY_ATTEMPTS, 4) + self.assertEqual(history.GET_RETRY_BUDGET_SECONDS, 30) + self.assertLessEqual( + sum(history.GET_RETRY_BACKOFF_SECONDS) + history.GET_RETRY_JITTER_SECONDS * (history.GET_RETRY_ATTEMPTS - 1), + history.GET_RETRY_BUDGET_SECONDS) + archive = "snapshots/" + ("ab" * 32) + "/archive.zip" + + class Response(io.BytesIO): + def __init__(self, raw, url, status=200, headers=None): + super().__init__(raw) + self.status = status + self.headers = {} if headers is None else headers + self._url = url + + def geturl(self): + return self._url + + class Opener: + def __init__(self, script): + self.script = list(script) + self.opens = [] + + def open(self, request, timeout): + self.opens.append((request.full_url, timeout)) + action = self.script.pop(0) + if callable(action): + action = action(request) + if isinstance(action, BaseException): + raise action + status, body = action + return Response(body, request.full_url, status) + + def reader(script, **kwargs): + opener = Opener(script) + sleeps = [] + client = history.HistoryReader(opener=opener, pause=sleeps.append, jitter=lambda: 0, **kwargs) + return client, opener, sleeps + + def http_error(request, code, reason, headers=None): + return urllib.error.HTTPError(request.full_url, code, reason, {} if headers is None else headers, None) + + client, opener, sleeps = reader([ + lambda request: http_error(request, 503, "Service Unavailable"), + (200, b"{}"), + ]) + self.assertEqual(client.get_bytes("current.json"), b"{}") + self.assertEqual(client.requests, 1) + self.assertEqual(len(opener.opens), 2) + self.assertEqual(opener.opens[0][1], 20) + self.assertEqual(sleeps, [2]) + self.assertTrue(all(url.startswith(history.ORIGIN) for url, _ in opener.opens)) + + client, opener, sleeps = reader([(503, b""), (200, b"{}")]) + self.assertEqual(client.get_bytes("snapshots/index.json"), b"{}") + self.assertEqual(len(opener.opens), 2) + self.assertEqual(client.requests, 1) + + client, opener, sleeps = reader([ + lambda request: http_error(request, 503, "Service Unavailable"), + (200, b"{}"), + ]) + client.requests = history.MAX_SNAPSHOTS + 2 + self.assertEqual(client.get_bytes("current.json"), b"{}") + self.assertEqual(client.requests, history.MAX_SNAPSHOTS + 3) + self.assertEqual(len(opener.opens), 2) + with self.assertRaises(ValueError): + client.get_bytes("current.json") + self.assertEqual(len(opener.opens), 2) + + client, opener, sleeps = reader([lambda request: http_error(request, 503, "Service Unavailable")] * 4) + with self.assertRaises(urllib.error.HTTPError) as caught: + client.get_bytes(archive) + self.assertEqual(caught.exception.code, 503) + self.assertIn(history.ORIGIN + archive, str(caught.exception)) + self.assertEqual(len(opener.opens), 4) + self.assertEqual(sleeps, [2, 4, 8]) + self.assertEqual(client.requests, 1) + self.assertLess(sum(sleeps), 30) + + client, opener, sleeps = reader([ + lambda request: http_error(request, 404, "Not Found"), + (200, b"{}"), + ]) + with self.assertRaises(urllib.error.HTTPError) as caught: + client.get_bytes("snapshots/index.json") + self.assertEqual(caught.exception.code, 404) + self.assertIn(history.ORIGIN + "snapshots/index.json", str(caught.exception)) + self.assertEqual(len(opener.opens), 1) + self.assertEqual(sleeps, []) + + client, opener, sleeps = reader([ + lambda request: http_error(request, 429, "Too Many Requests", {"Retry-After": "3"}), + (200, b"{}"), + ]) + self.assertEqual(client.get_bytes("current-meta.json"), b"{}") + self.assertEqual(sleeps, [3]) + self.assertEqual(len(opener.opens), 2) + + client, opener, sleeps = reader([ + lambda request: urllib.error.URLError("timed out"), + (200, b"{}"), + ]) + self.assertEqual(client.get_bytes("current.json"), b"{}") + self.assertEqual(len(opener.opens), 2) + self.assertTrue(opener.opens[0][0].startswith(history.ORIGIN)) + + class Clock: + def __init__(self): + self.value = 0.0 + + def __call__(self): + return self.value + + clock = Clock() + recorded = [] + + class BudgetOpener: + def __init__(self): + self.opens = [] + + def open(self, request, timeout): + self.opens.append((request.full_url, timeout)) + clock.value += timeout + raise urllib.error.URLError("timed out") + + def pause(delay): + recorded.append(delay) + clock.value += delay + + budget = BudgetOpener() + client = history.HistoryReader(opener=budget, pause=pause, clock=clock, jitter=lambda: 0) + with self.assertRaises(urllib.error.URLError) as caught: + client.get_bytes(archive) + self.assertIn(history.ORIGIN + archive, str(caught.exception)) + self.assertEqual([timeout for _, timeout in budget.opens], [20, 8]) + self.assertEqual(recorded, [2]) + self.assertLessEqual(clock.value, 30) + self.assertEqual(client.requests, 1) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_current_site_publication.py b/tests/test_current_site_publication.py index 0826b26f1..3e7fb47e6 100644 --- a/tests/test_current_site_publication.py +++ b/tests/test_current_site_publication.py @@ -443,6 +443,168 @@ def open(self, request, timeout): with self.assertRaisesRegex(ValueError, "request bound"): client.request("GET", self.module.REF_READ) + def test_get_retries_transient_503_then_200_and_writes_do_not_retry(self): + module = self.module + self.assertEqual(module.GET_RETRY_ATTEMPTS, 4) + self.assertEqual(module.GET_RETRY_BUDGET_SECONDS, 30) + self.assertLessEqual( + sum(module.GET_RETRY_BACKOFF_SECONDS) + module.GET_RETRY_JITTER_SECONDS * (module.GET_RETRY_ATTEMPTS - 1), + module.GET_RETRY_BUDGET_SECONDS) + + class Response(io.BytesIO): + def __init__(self, raw, url, status=200, headers=None): + super().__init__(raw) + self.status = status + self.headers = {} if headers is None else headers + self._url = url + + def geturl(self): + return self._url + + class Opener: + def __init__(self, script): + self.script = list(script) + self.opens = [] + + def open(self, request, timeout): + self.opens.append((request.get_method(), request.full_url, timeout)) + action = self.script.pop(0) + if callable(action): + action = action(request) + if isinstance(action, BaseException): + raise action + status, body = action + return Response(body, request.full_url, status) + + def client(script, **kwargs): + opener = Opener(script) + sleeps = [] + api = module.GitHubAPI(opener=opener, pause=sleeps.append, jitter=lambda: 0, **kwargs) + return api, opener, sleeps + + def http_error(request, code, reason, headers=None): + return urllib.error.HTTPError(request.full_url, code, reason, {} if headers is None else headers, None) + + api, opener, sleeps = client([ + lambda request: http_error(request, 503, "Service Unavailable"), + (200, b"{}"), + ]) + self.assertEqual(api.request("GET", module.REF_READ), {}) + self.assertEqual(api.requests, 1) + self.assertEqual([item[0] for item in opener.opens], ["GET", "GET"]) + self.assertEqual(opener.opens[0][2], 20) + self.assertEqual(sleeps, [2]) + self.assertTrue(all(url.startswith(module.API_ORIGIN) for _, url, _ in opener.opens)) + + api, opener, sleeps = client([(503, b""), (200, b"{}")]) + self.assertEqual(api.request("GET", module.REF_READ), {}) + self.assertEqual(len(opener.opens), 2) + self.assertEqual(api.requests, 1) + + api, opener, sleeps = client([ + lambda request: http_error(request, 503, "Service Unavailable"), + (200, b"{}"), + ]) + api.requests = 63 + self.assertEqual(api.request("GET", module.REF_READ), {}) + self.assertEqual(api.requests, 64) + self.assertEqual(len(opener.opens), 2) + with self.assertRaisesRegex(ValueError, "request bound"): + api.request("GET", module.REF_READ) + + api, opener, sleeps = client([lambda request: http_error(request, 503, "Service Unavailable")] * 4) + with self.assertRaises(urllib.error.HTTPError) as caught: + api.request("GET", module.REF_READ) + self.assertEqual(caught.exception.code, 503) + self.assertIn(module.API_ORIGIN + module.REF_READ, str(caught.exception)) + self.assertEqual(len(opener.opens), 4) + self.assertEqual(sleeps, [2, 4, 8]) + self.assertEqual(api.requests, 1) + self.assertLess(sum(sleeps), 30) + + api, opener, sleeps = client([ + lambda request: http_error(request, 404, "Not Found"), + (200, b"{}"), + ]) + with self.assertRaises(urllib.error.HTTPError) as caught: + api.request("GET", module.REF_READ) + self.assertEqual(caught.exception.code, 404) + self.assertIn(module.API_ORIGIN + module.REF_READ, str(caught.exception)) + self.assertEqual(len(opener.opens), 1) + self.assertEqual(sleeps, []) + + api, opener, sleeps = client([ + lambda request: http_error(request, 429, "Too Many Requests", {"Retry-After": "3"}), + (200, b"{}"), + ]) + self.assertEqual(api.request("GET", module.REF_READ), {}) + self.assertEqual(sleeps, [3]) + self.assertEqual(len(opener.opens), 2) + + api, opener, sleeps = client([ + lambda request: http_error(request, 429, "Too Many Requests", {"Retry-After": "120"}), + ]) + with self.assertRaises(urllib.error.HTTPError) as caught: + api.request("GET", module.REF_READ) + self.assertEqual(caught.exception.code, 429) + self.assertIn(module.API_ORIGIN + module.REF_READ, str(caught.exception)) + self.assertEqual(len(opener.opens), 1) + self.assertEqual(sleeps, []) + + api, opener, sleeps = client([ + lambda request: urllib.error.URLError("timed out"), + (200, b"{}"), + ]) + self.assertEqual(api.request("GET", module.REF_READ), {}) + self.assertEqual(len(opener.opens), 2) + self.assertIn(module.API_ORIGIN + module.REF_READ, str(opener.opens[0][1])) + self.assertEqual(sleeps, [2]) + + api, opener, sleeps = client([ + lambda request: http_error(request, 503, "Service Unavailable"), + (200, b"{}"), + ]) + with self.assertRaises(urllib.error.HTTPError) as caught: + api.request("PATCH", module.REF_WRITE, {"sha": "a" * 40, "force": False}) + self.assertEqual(len(opener.opens), 1) + self.assertEqual(opener.opens[0][0], "PATCH") + self.assertIn(module.API_ORIGIN + module.REF_WRITE, str(caught.exception)) + self.assertEqual(sleeps, []) + self.assertEqual(api.requests, 1) + + class Clock: + def __init__(self): + self.value = 0.0 + + def __call__(self): + return self.value + + clock = Clock() + recorded = [] + + class BudgetOpener: + def __init__(self): + self.opens = [] + + def open(self, request, timeout): + self.opens.append(timeout) + clock.value += timeout + raise urllib.error.URLError("timed out") + + def pause(delay): + recorded.append(delay) + clock.value += delay + + budget = BudgetOpener() + api = module.GitHubAPI(opener=budget, pause=pause, clock=clock, jitter=lambda: 0) + with self.assertRaises(urllib.error.URLError) as caught: + api.request("GET", module.REF_READ) + self.assertIn(module.API_ORIGIN + module.REF_READ, str(caught.exception)) + self.assertEqual(budget.opens, [20, 8]) + self.assertEqual(recorded, [2]) + self.assertLessEqual(clock.value, 30) + self.assertEqual(api.requests, 1) + def test_qualified_metadata_package_binds_real_artifact_and_browser_evidence(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/tools/current_site_history.py b/tools/current_site_history.py index 0aea56b88..1d34ec5ec 100644 --- a/tools/current_site_history.py +++ b/tools/current_site_history.py @@ -16,8 +16,11 @@ import os from pathlib import Path, PurePosixPath import re +import email.utils +import random import stat import tempfile +import time import urllib.error import urllib.request from urllib.parse import urlsplit @@ -31,6 +34,11 @@ ORIGIN = "https://d6g8k5htny-coder.github.io/main/site/" MAX_SNAPSHOTS = 512 +TRANSIENT_HTTP = frozenset({429, 500, 502, 503, 504}) +GET_RETRY_ATTEMPTS = 4 +GET_RETRY_BUDGET_SECONDS = 30 +GET_RETRY_BACKOFF_SECONDS = (2, 4, 8) +GET_RETRY_JITTER_SECONDS = 0.25 MAX_HISTORY_BYTES = 768 * 1024 * 1024 MAX_FILE_BYTES = 16 * 1024 * 1024 MAX_MEMBERS = 20006 @@ -509,11 +517,78 @@ def redirect_request(self, request, fp, code, msg, headers, newurl): raise ValueError("public history redirect refused") +def _default_jitter(): + return random.uniform(0.0, GET_RETRY_JITTER_SECONDS) + + +def _note_http_error(error, url): + marker = "[" + url + "]" + message = str(error.msg or "") + if marker not in message: + error.msg = (message + " " + marker).strip() + error.filename = url + return error + + +def _note_url_error(error, url): + marker = "[" + url + "]" + reason = getattr(error, "reason", error) + text = str(reason) + if marker not in text: + text = text + " " + marker + noted = urllib.error.URLError(text) + noted.filename = url + return noted + + +def _retry_after_seconds(headers, now): + if headers is None or not hasattr(headers, "get"): + return None + raw = headers.get("Retry-After") + if raw is None: + return None + text = str(raw).strip() + if re.fullmatch(r"[0-9]+", text): + return int(text) + if re.fullmatch(r"[0-9]+\.[0-9]+", text): + return float(text) + try: + parsed = email.utils.parsedate_to_datetime(text) + except (TypeError, ValueError, OverflowError, OSError): + return None + if parsed is None: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + delay = (parsed - now).total_seconds() + return 0 if delay < 0 else delay + + +def _retry_delay(failed_attempt, retry_after, elapsed, jitter): + remaining = GET_RETRY_BUDGET_SECONDS - elapsed + if remaining <= 0: + return None + if retry_after is not None: + if retry_after >= remaining: + return None + return retry_after + if failed_attempt >= len(GET_RETRY_BACKOFF_SECONDS): + return None + delay = GET_RETRY_BACKOFF_SECONDS[failed_attempt] + jitter + if delay >= remaining: + return None + return delay + + class HistoryReader: - def __init__(self, opener=None): + def __init__(self, opener=None, *, pause=time.sleep, clock=time.monotonic, jitter=None, now=None): self.opener = opener or urllib.request.build_opener(NoRedirect()) self.requests = 0 self.total_bytes = 0 + self.pause = pause + self.clock = clock + self.jitter = _default_jitter if jitter is None else jitter + self.now = (lambda: datetime.now(timezone.utc)) if now is None else now def get_bytes(self, path): if path not in ("snapshots/index.json", "current-meta.json", "current.json") and not re.fullmatch( @@ -521,27 +596,75 @@ def get_bytes(self, path): raise ValueError("nonpublic history request path refused") if self.requests >= MAX_SNAPSHOTS + 3: raise ValueError("history request capacity exceeded") + # One logical Pages GET. Retries do not consume MAX_SNAPSHOTS + 3, so a + # full restore can still absorb a transient 503. This is HistoryReader, + # not publication.PublicReader; public_readback retries that other path. self.requests += 1 url = ORIGIN + path request = urllib.request.Request(url, headers={"User-Agent": "current-site-public-history", "Cache-Control": "no-cache", "Pragma": "no-cache", "Accept-Encoding": "identity"}) - with self.opener.open(request, timeout=20) as response: - if response.geturl() != url or response.status != 200: - raise ValueError("public history response identity mismatch") - encoding = response.headers.get("Content-Encoding", "identity") - if encoding not in ("", "identity"): - raise ValueError("encoded public history response refused") - length = response.headers.get("Content-Length") - if length is not None and (not length.isascii() or not length.isdecimal() or int(length) > MAX_FILE_BYTES): - raise ValueError("history response byte bound exceeded") - raw = response.read(MAX_FILE_BYTES + 1) + raw, length = self._read_exchange(request) self.total_bytes += len(raw) if (len(raw) > MAX_FILE_BYTES or (length is not None and len(raw) != int(length)) or self.total_bytes > MAX_HISTORY_BYTES + 3 * MAX_FILE_BYTES): - raise ValueError("history response byte bound exceeded") + raise ValueError(f"history response byte bound exceeded [{url}]") return raw + def _read_exchange(self, outbound): + url = outbound.full_url + started = self.clock() + last_error = None + for attempt in range(GET_RETRY_ATTEMPTS): + elapsed = self.clock() - started + if attempt and elapsed >= GET_RETRY_BUDGET_SECONDS: + break + timeout = 20 if attempt == 0 else min(20, GET_RETRY_BUDGET_SECONDS - elapsed) + retry_after = None + try: + with self.opener.open(outbound, timeout=timeout) as response: + if response.geturl() != url: + raise ValueError(f"public history response identity mismatch [{url}]") + status = response.status + headers = response.headers + if status in TRANSIENT_HTTP: + last_error = urllib.error.HTTPError( + url, status, f"transient HTTP {status} [{url}]", headers, None) + retry_after = _retry_after_seconds(headers, self.now()) + elif status != 200: + raise ValueError(f"public history response identity mismatch [{url}]") + else: + encoding = headers.get("Content-Encoding", "identity") + if encoding not in ("", "identity"): + raise ValueError(f"encoded public history response refused [{url}]") + length = headers.get("Content-Length") + if length is not None and (not length.isascii() or not length.isdecimal() or int(length) > MAX_FILE_BYTES): + raise ValueError(f"history response byte bound exceeded [{url}]") + return response.read(MAX_FILE_BYTES + 1), length + except urllib.error.HTTPError as error: + _note_http_error(error, url) + if error.code not in TRANSIENT_HTTP or attempt + 1 == GET_RETRY_ATTEMPTS: + raise + last_error = error + retry_after = _retry_after_seconds(getattr(error, "headers", None), self.now()) + except (urllib.error.URLError, TimeoutError) as error: + last_error = _note_url_error(error, url) + if attempt + 1 == GET_RETRY_ATTEMPTS: + raise last_error from error + retry_after = None + if last_error is None: + raise ValueError(f"public history transient failure [{url}]") + if attempt + 1 == GET_RETRY_ATTEMPTS: + raise last_error + jitter = 0 if retry_after is not None else self.jitter() + delay = _retry_delay(attempt, retry_after, self.clock() - started, jitter) + if delay is None: + raise last_error + self.pause(delay) + if last_error is not None: + raise last_error + raise ValueError(f"public history transient retry budget exceeded [{url}]") + class StaticHTML(HTMLParser): TAGS = {"html", "head", "meta", "title", "link", "body", "main", "nav", "section", "h1", "h2", "h3", @@ -993,7 +1116,7 @@ def remote_metadata(reader, *, missing_allowed=False): except urllib.error.HTTPError as error: if error.code == 404 and missing_allowed: return None - raise ValueError("published history metadata unavailable") from error + raise ValueError("published history metadata unavailable: " + str(error)) from error meta = parse_json(raw) if not isinstance(meta, dict): raise ValueError("malformed published history metadata") @@ -1073,7 +1196,7 @@ def restore_history(site, reader=None, expected_manifest_sha=None): raw_manifest = reader.get_bytes("snapshots/index.json") except urllib.error.HTTPError as error: if error.code != 404: - raise ValueError("published history manifest unavailable") from error + raise ValueError("published history manifest unavailable: " + str(error)) from error if expected_manifest_sha is not None: raise ValueError("published manifest pin is unavailable; bootstrap refused") from error remote = remote_metadata(reader, missing_allowed=True) diff --git a/tools/current_site_publication.py b/tools/current_site_publication.py index af936d9d0..5bcdf988f 100644 --- a/tools/current_site_publication.py +++ b/tools/current_site_publication.py @@ -8,10 +8,12 @@ import argparse import base64 from datetime import datetime, timezone +import email.utils import hashlib import json import os from pathlib import Path +import random import re import subprocess import tempfile @@ -34,6 +36,11 @@ MAX_SNAPSHOTS = 512 REF_READBACK_ATTEMPTS = 4 REF_READBACK_SECONDS = 10 +TRANSIENT_HTTP = frozenset({429, 500, 502, 503, 504}) +GET_RETRY_ATTEMPTS = 4 +GET_RETRY_BUDGET_SECONDS = 30 +GET_RETRY_BACKOFF_SECONDS = (2, 4, 8) +GET_RETRY_JITTER_SECONDS = 0.25 HEX40 = re.compile(r"[0-9a-f]{40}\Z") HEX64 = re.compile(r"[0-9a-f]{64}\Z") PATHS = {"manifest": "snapshots/index.json", "metadata": "current-meta.json", "payload": "current.json"} @@ -208,16 +215,88 @@ def guard_request(method, path, body): raise ValueError("nonmetadata GitHub route refused") +def _default_jitter(): + return random.uniform(0.0, GET_RETRY_JITTER_SECONDS) + + +def _note_http_error(error, url): + marker = "[" + url + "]" + message = str(error.msg or "") + if marker not in message: + error.msg = (message + " " + marker).strip() + error.filename = url + return error + + +def _note_url_error(error, url): + marker = "[" + url + "]" + reason = getattr(error, "reason", error) + text = str(reason) + if marker not in text: + text = text + " " + marker + noted = urllib.error.URLError(text) + noted.filename = url + return noted + + +def _retry_after_seconds(headers, now): + if headers is None or not hasattr(headers, "get"): + return None + raw = headers.get("Retry-After") + if raw is None: + return None + text = str(raw).strip() + if re.fullmatch(r"[0-9]+", text): + return int(text) + if re.fullmatch(r"[0-9]+\.[0-9]+", text): + return float(text) + try: + parsed = email.utils.parsedate_to_datetime(text) + except (TypeError, ValueError, OverflowError, OSError): + return None + if parsed is None: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + delay = (parsed - now).total_seconds() + return 0 if delay < 0 else delay + + +def _retry_delay(failed_attempt, retry_after, elapsed, jitter): + remaining = GET_RETRY_BUDGET_SECONDS - elapsed + if remaining <= 0: + return None + if retry_after is not None: + if retry_after >= remaining: + return None + return retry_after + if failed_attempt >= len(GET_RETRY_BACKOFF_SECONDS): + return None + delay = GET_RETRY_BACKOFF_SECONDS[failed_attempt] + jitter + if delay >= remaining: + return None + return delay + + class GitHubAPI: - def __init__(self, *, token=None, opener=None): + def __init__(self, *, token=None, opener=None, pause=time.sleep, clock=time.monotonic, jitter=None, now=None): self.token = os.environ.get("GITHUB_TOKEN") if token is None else token self.opener = opener if opener is not None else urllib.request.build_opener(NoRedirect()) self.requests = self.total_bytes = 0 + self.pause = pause + self.clock = clock + self.jitter = _default_jitter if jitter is None else jitter + self.now = (lambda: datetime.now(timezone.utc)) if now is None else now def request(self, method, path, body=None): guard_request(method, path, body) if self.requests >= 64: raise ValueError("publication API request bound exceeded") + # One logical request. GET retries below do not consume the 64-request + # custody bound, so a full anchor read can still absorb a transient 503. + # PATCH and POST, including receipt ref writes, are never retried. + # PublicReader.get_bytes is a separate Pages path and is unchanged; + # public_readback already repeats that read. self.requests += 1 headers = {"Accept": "application/vnd.github+json", "User-Agent": "current-site-publication-receipt", "X-GitHub-Api-Version": "2022-11-28"} @@ -226,19 +305,65 @@ def request(self, method, path, body=None): raw_body = encode(body) if body is not None else None if raw_body is not None: headers["Content-Type"] = "application/json" - request = urllib.request.Request(API_ORIGIN + path, data=raw_body, headers=headers, method=method) - with self.opener.open(request, timeout=20) as response: - if response.geturl() != request.full_url or response.status not in (200, 201): - raise ValueError("publication API response identity mismatch") - raw = response.read(MAX_RECEIPT_BYTES + 1) + outbound = urllib.request.Request(API_ORIGIN + path, data=raw_body, headers=headers, method=method) + raw = self._read_exchange(method, outbound) self.total_bytes += len(raw) if len(raw) > MAX_RECEIPT_BYTES or self.total_bytes > 16 * MAX_RECEIPT_BYTES: - raise ValueError("publication API byte bound exceeded") + raise ValueError(f"publication API byte bound exceeded [{outbound.full_url}]") result = parse_json(raw) if not isinstance(result, dict): - raise ValueError("publication API object response required") + raise ValueError(f"publication API object response required [{outbound.full_url}]") return result + def _read_exchange(self, method, outbound): + url = outbound.full_url + started = self.clock() + attempts = GET_RETRY_ATTEMPTS if method == "GET" else 1 + last_error = None + for attempt in range(attempts): + elapsed = self.clock() - started + if attempt and elapsed >= GET_RETRY_BUDGET_SECONDS: + break + timeout = 20 if attempt == 0 else min(20, GET_RETRY_BUDGET_SECONDS - elapsed) + retry_after = None + try: + with self.opener.open(outbound, timeout=timeout) as response: + if response.geturl() != url: + raise ValueError(f"publication API response identity mismatch [{url}]") + status = response.status + headers = getattr(response, "headers", None) + if method == "GET" and status in TRANSIENT_HTTP: + last_error = urllib.error.HTTPError( + url, status, f"transient HTTP {status} [{url}]", headers, None) + retry_after = _retry_after_seconds(headers, self.now()) + elif status not in (200, 201): + raise ValueError(f"publication API response identity mismatch [{url}]") + else: + return response.read(MAX_RECEIPT_BYTES + 1) + except urllib.error.HTTPError as error: + _note_http_error(error, url) + if method != "GET" or error.code not in TRANSIENT_HTTP or attempt + 1 == attempts: + raise + last_error = error + retry_after = _retry_after_seconds(getattr(error, "headers", None), self.now()) + except (urllib.error.URLError, TimeoutError) as error: + last_error = _note_url_error(error, url) + if method != "GET" or attempt + 1 == attempts: + raise last_error from error + retry_after = None + if last_error is None: + raise ValueError(f"publication API transient failure [{url}]") + if attempt + 1 == attempts: + raise last_error + jitter = 0 if retry_after is not None else self.jitter() + delay = _retry_delay(attempt, retry_after, self.clock() - started, jitter) + if delay is None: + raise last_error + self.pause(delay) + if last_error is not None: + raise last_error + raise ValueError(f"publication API transient retry budget exceeded [{url}]") + class PublicReader: def __init__(self, *, opener=None, cache_key="", expected_sizes=None, deadline=None, clock=time.monotonic): @@ -297,7 +422,7 @@ def read_ref_head(api, *, allow_bootstrap=False): except urllib.error.HTTPError as error: if error.code == 404 and allow_bootstrap: return None - raise ValueError("publication receipt branch unavailable; bootstrap requires explicit initial 404") from error + raise ValueError("publication receipt branch unavailable; bootstrap requires explicit initial 404: " + str(error)) from error if ref.get("ref") != REF_NAME or not isinstance(ref.get("object"), dict) or ref["object"].get("type") != "commit": raise ValueError("invalid publication anchor ref") return hex_value(ref["object"].get("sha"), HEX40, "anchor ref commit")