Harden price-backfill against transient fetch failures

tools/price-backfill/README.md

@@ -39,7 +39,14 @@ Smoke test first with `--limit 50`.
 
 `--price-scale` (default 1.0), `--min-item-count` (1), `--max-age-days` (180),
 `--deviation-factor` (1.5), `--concurrency` (6), `--rate-delay` (0.15s),
-`--redeploy-threshold` (25), `--sentinel-id` (4389).
+`--redeploy-threshold` (25), `--sentinel-id` (4389), `--resweep-rounds` (2).
+
+## Transient fetch failures
+
+Over a long run some requests hit transient HTTP/network errors that survive the
+per-request retries (`_get` retries 5x with exponential backoff). At the end of a run
+the tool re-sweeps any `fetch-failed` items for `--resweep-rounds` bounded passes until
+none remain, so a single run doesn't leave a gap of items that simply weren't reached.
 
 ## Redeploy detection
 

tools/price-backfill/backfill.py

@@ -144,6 +144,11 @@ def select_recovery_ids(records, final_gen):
             if r.get("reason") == "no-data" and r.get("gen", 0) < final_gen]
 
 
+def select_fetch_failed(records):
+    """Items whose fetch errored out (transient HTTP/network) — worth another try."""
+    return [r["item"] for r in records if r.get("reason") == "fetch-failed"]
+
+
 def main():
     p = argparse.ArgumentParser(description=__doc__)
     p.add_argument("--ids-csv", required=True)
@@ -164,6 +169,8 @@ def main():
     p.add_argument("--sentinel-id", type=int, default=4389,
                    help="known-good item id used to confirm a real site redeploy")
     p.add_argument("--limit", type=int, default=0)
+    p.add_argument("--resweep-rounds", type=int, default=2,
+                   help="extra passes to retry transient fetch-failed items")
     args = p.parse_args()
 
     out_sql = args.out_sql or args.existing_sql
@@ -210,6 +217,16 @@ def run_pass(id_list):
                     final_gen, len(recovery)))
                 run_pass(recovery)
 
+        # Re-sweep transient fetch failures: HTTP/network errors that survived
+        # per-request retries are usually transient over a long run, so re-fetch
+        # them for a few bounded rounds until none remain.
+        for _ in range(args.resweep_rounds):
+            failed = select_fetch_failed(list(load_checkpoint(args.checkpoint).values()))
+            if not failed:
+                break
+            print("re-sweeping {} fetch-failed items".format(len(failed)))
+            run_pass(failed)
+
     records = list(load_checkpoint(args.checkpoint).values())
     kept, ndev = write_outputs(records, out_sql, args.skipped_csv, args.deviations_csv)
     print("wrote {} rows to {} | {} deviations | build_gen={}".format(

tools/price-backfill/test_backfill.py

@@ -73,6 +73,17 @@ def test_select_recovery_ids(self):
         self.assertEqual(backfill.select_recovery_ids(records, 1), [1])
 
 
+class SelectFetchFailedTest(unittest.TestCase):
+    def test_selects_only_fetch_failed(self):
+        records = [
+            {"item": 1, "reason": "fetch-failed"},
+            {"item": 2, "reason": "no-data"},
+            {"item": 3, "row": [3, 1, 1], "reason": None},
+            {"item": 4, "reason": "fetch-failed"},
+        ]
+        self.assertEqual(sorted(backfill.select_fetch_failed(records)), [1, 4])
+
+
 class NoteResultTest(unittest.TestCase):
     """Redeploy is confirmed via a sentinel item, not raw 404 counting."""
 

tools/price-backfill/wowauctions.py

@@ -18,7 +18,7 @@ def _raw_get(url: str, timeout: int) -> str:
         return resp.read().decode("utf-8")
 
 
-def _get(url: str, timeout: int = 20, retries: int = 3, backoff: float = 1.0) -> str:
+def _get(url: str, timeout: int = 20, retries: int = 5, backoff: float = 1.0) -> str:
     """GET with exponential backoff on 429/5xx and network errors. 404 is raised immediately."""
     attempt = 0
     while True: