diff --git a/output_writer.py b/output_writer.py index 13b43c1..180f476 100644 --- a/output_writer.py +++ b/output_writer.py @@ -394,6 +394,33 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: EXIT_PARTIAL = 6 +# Exit code for a run that never GOT its pages: a navigation timeout, a dead +# or unauthenticated proxy, a DNS failure, or an edge answering with +# something that is not the page that was asked for. +# +# Distinct from EXIT_NO_PRODUCTS because those are opposite facts. Exit 4 is +# a statement about the CATALOGUE — "we asked, and the answer was nothing" — +# so handing it to a run that never reached the site tells a pipeline the +# listing is empty when nothing was read at all. +# +# 5 rather than a new number, and 5 rather than EXIT_PARTIAL: +# +# * this family's contract already reserves 5 for a transport failure +# (scraper_api_client has used it for a remote API error since it was +# written), so this needs no new code and no per-repo table for a caller +# driving more than one of these scrapers; +# * EXIT_PARTIAL (6) means "some rows were gathered and the output is +# incomplete". A run holding nothing writes no output at all, so a +# consumer that reads the file on a 6 finds either nothing or the +# PREVIOUS run's good data, which `save` deliberately does not +# overwrite. Exit 5 promises no file. +# +# Deliberately NOT applied when rows WERE gathered: a timeout on page 7 of +# 10 is a partial run (exit 6, output written), which is already right. This +# decides only what a run holding nothing reports. +EXIT_FETCH_FAILED = 5 + + def write_run_meta(out_prefix: str, meta: dict) -> str: """Write a run-metadata sidecar next to the output, return its path. @@ -589,7 +616,7 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, print(f"[!] Failed run: 0 of {pages_requested} page(s) were " f"fetched ({stop_reason}). This is NOT an empty result — " f"nothing was read from the site at all.") - return EXIT_PARTIAL + return EXIT_FETCH_FAILED return rc if not complete: print(f"[!] Partial run: stopped after {pages_completed} of " diff --git a/smoke_test.py b/smoke_test.py index 1793bf1..c3b0435 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -73,7 +73,7 @@ import page_flow import product_parser from diff_runs import diff_products -from output_writer import (Market, Product, save, finish_run, write_csv, +from output_writer import (EXIT_FETCH_FAILED, Market, Product, save, finish_run, write_csv, run_meta, dedupe_by_key, dedupe_by_sku, merge_pages, is_deeper, ROW_CLASS_BY_MODE, UNIQUE_BY_SKU_MODES, @@ -1213,13 +1213,13 @@ def run(rows, stop_reason, blocked=False, allow_empty=False): # pipeline branching on the exit code, which is what this family # says exit codes are for, would have recorded an empty catalogue. code, meta = run([], "page_load_timeout", allow_empty=True) - ok &= check("0 rows because nothing was FETCHED is exit 6, not 4", - code == EXIT_PARTIAL) + ok &= check("0 rows because nothing was FETCHED is exit 5, not 4", + code == EXIT_FETCH_FAILED) ok &= check("and the sidecar says failed, not complete", meta is not None and meta["status"] == "failed") code, meta = run([], "next_batch_refused", allow_empty=True) - ok &= check("a refused batch with no rows is exit 6 too", - code == EXIT_PARTIAL) + ok &= check("a refused batch with no rows is exit 5 too", + code == EXIT_FETCH_FAILED) code, meta = run([], "blocked_cloudflare", blocked=True, allow_empty=True) ok &= check("but a BLOCK still outranks both, at exit 3", code == EXIT_BLOCKED)