diff --git a/tools/price-backfill/README.md b/tools/price-backfill/README.md index 2f3615b..7bea38f 100644 --- a/tools/price-backfill/README.md +++ b/tools/price-backfill/README.md @@ -39,7 +39,14 @@ Smoke test first with `--limit 50`. `--price-scale` (default 1.0), `--min-item-count` (1), `--max-age-days` (180), `--deviation-factor` (1.5), `--concurrency` (6), `--rate-delay` (0.15s), -`--redeploy-threshold` (25), `--sentinel-id` (4389). +`--redeploy-threshold` (25), `--sentinel-id` (4389), `--resweep-rounds` (2). + +## Transient fetch failures + +Over a long run some requests hit transient HTTP/network errors that survive the +per-request retries (`_get` retries 5x with exponential backoff). At the end of a run +the tool re-sweeps any `fetch-failed` items for `--resweep-rounds` bounded passes until +none remain, so a single run doesn't leave a gap of items that simply weren't reached. ## Redeploy detection diff --git a/tools/price-backfill/backfill.py b/tools/price-backfill/backfill.py index f81693c..1a5db8b 100644 --- a/tools/price-backfill/backfill.py +++ b/tools/price-backfill/backfill.py @@ -144,6 +144,11 @@ def select_recovery_ids(records, final_gen): if r.get("reason") == "no-data" and r.get("gen", 0) < final_gen] +def select_fetch_failed(records): + """Items whose fetch errored out (transient HTTP/network) — worth another try.""" + return [r["item"] for r in records if r.get("reason") == "fetch-failed"] + + def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--ids-csv", required=True) @@ -164,6 +169,8 @@ def main(): p.add_argument("--sentinel-id", type=int, default=4389, help="known-good item id used to confirm a real site redeploy") p.add_argument("--limit", type=int, default=0) + p.add_argument("--resweep-rounds", type=int, default=2, + help="extra passes to retry transient fetch-failed items") args = p.parse_args() out_sql = args.out_sql or args.existing_sql @@ -210,6 +217,16 @@ def run_pass(id_list): final_gen, len(recovery))) run_pass(recovery) + # Re-sweep transient fetch failures: HTTP/network errors that survived + # per-request retries are usually transient over a long run, so re-fetch + # them for a few bounded rounds until none remain. + for _ in range(args.resweep_rounds): + failed = select_fetch_failed(list(load_checkpoint(args.checkpoint).values())) + if not failed: + break + print("re-sweeping {} fetch-failed items".format(len(failed))) + run_pass(failed) + records = list(load_checkpoint(args.checkpoint).values()) kept, ndev = write_outputs(records, out_sql, args.skipped_csv, args.deviations_csv) print("wrote {} rows to {} | {} deviations | build_gen={}".format( diff --git a/tools/price-backfill/test_backfill.py b/tools/price-backfill/test_backfill.py index 5ee1f80..fe81e07 100644 --- a/tools/price-backfill/test_backfill.py +++ b/tools/price-backfill/test_backfill.py @@ -73,6 +73,17 @@ def test_select_recovery_ids(self): self.assertEqual(backfill.select_recovery_ids(records, 1), [1]) +class SelectFetchFailedTest(unittest.TestCase): + def test_selects_only_fetch_failed(self): + records = [ + {"item": 1, "reason": "fetch-failed"}, + {"item": 2, "reason": "no-data"}, + {"item": 3, "row": [3, 1, 1], "reason": None}, + {"item": 4, "reason": "fetch-failed"}, + ] + self.assertEqual(sorted(backfill.select_fetch_failed(records)), [1, 4]) + + class NoteResultTest(unittest.TestCase): """Redeploy is confirmed via a sentinel item, not raw 404 counting.""" diff --git a/tools/price-backfill/wowauctions.py b/tools/price-backfill/wowauctions.py index a694d2e..6b414dd 100644 --- a/tools/price-backfill/wowauctions.py +++ b/tools/price-backfill/wowauctions.py @@ -18,7 +18,7 @@ def _raw_get(url: str, timeout: int) -> str: return resp.read().decode("utf-8") -def _get(url: str, timeout: int = 20, retries: int = 3, backoff: float = 1.0) -> str: +def _get(url: str, timeout: int = 20, retries: int = 5, backoff: float = 1.0) -> str: """GET with exponential backoff on 429/5xx and network errors. 404 is raised immediately.""" attempt = 0 while True: