From 4202c49331852c2b3bc14b7d43c8cf6ea01ef656 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 21:26:40 +0000 Subject: [PATCH] Report exit 5, not 4, when nothing was gathered and the run did not finish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit 4 is a claim about the CATALOGUE — "we asked, and the answer was nothing" — so handing it to a run that never reached the site tells a pipeline the listing is empty when nothing was read at all. Measured across the family on 2026-09-21 by CALLING each repo's finish_run rather than grepping for the fix: 20 repos still returned 4, two returned 5, and seven had already fixed it using EXIT_PARTIAL with written reasoning. Three camps, none aware of the other two. This unifies on 5, taking the rule from the exit-6 camp and the code from the exit-5 camp. The rule is `if not complete` rather than a named list of stop reasons: a list cannot cover a reason nobody has added to it yet, so a new stop_reason would fall silently through to "the catalogue is empty" — the defect all three camps set out to fix. The code is 5 rather than 6 because EXIT_PARTIAL means "some rows were gathered and the output is incomplete", and a run holding nothing writes no output, so a consumer reading the file on a 6 finds either nothing or the PREVIOUS run's good data that save() deliberately leaves in place. 5 promises no file, and the family contract already reserves it for a transport failure. Rows gathered and THEN a failure is still exit 6, unchanged. Verified by invoking finish_run after the patch and requiring 5, with the repo's own suite green. Checks that pinned the old answer were rewritten in the same commit. Co-Authored-By: Claude Opus 5 (1M context) --- output_writer.py | 52 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/output_writer.py b/output_writer.py index 720caf7..272448e 100644 --- a/output_writer.py +++ b/output_writer.py @@ -310,6 +310,33 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: EXIT_PARTIAL = 6 +# Exit code for a run that never GOT its pages: a navigation timeout, a dead +# or unauthenticated proxy, a DNS failure, or an edge answering with +# something that is not the page that was asked for. +# +# Distinct from EXIT_NO_PRODUCTS because those are opposite facts. Exit 4 is +# a statement about the CATALOGUE — "we asked, and the answer was nothing" — +# so handing it to a run that never reached the site tells a pipeline the +# listing is empty when nothing was read at all. +# +# 5 rather than a new number, and 5 rather than EXIT_PARTIAL: +# +# * this family's contract already reserves 5 for a transport failure +# (scraper_api_client has used it for a remote API error since it was +# written), so this needs no new code and no per-repo table for a caller +# driving more than one of these scrapers; +# * EXIT_PARTIAL (6) means "some rows were gathered and the output is +# incomplete". A run holding nothing writes no output at all, so a +# consumer that reads the file on a 6 finds either nothing or the +# PREVIOUS run's good data, which `save` deliberately does not +# overwrite. Exit 5 promises no file. +# +# Deliberately NOT applied when rows WERE gathered: a timeout on page 7 of +# 10 is a partial run (exit 6, output written), which is already right. This +# decides only what a run holding nothing reports. +EXIT_FETCH_FAILED = 5 + + def write_run_meta(out_prefix: str, meta: dict) -> str: """Write a run-metadata sidecar next to the output, return its path. @@ -480,9 +507,28 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, extra=extra)) if not rows: - # Nothing gathered at all: a challenge outranks "empty result", - # because it says something stood between the run and the content. - return EXIT_BLOCKED if blocked else rc + # Nothing gathered at all, and WHY decides the code. The three + # outcomes are different facts and a pipeline branches on them + # (blocked is not empty is not "never reached"): + # + # blocked something stood between the run and the content + # did not complete we never got the pages — a dead proxy, a load + # timeout, an edge serving something else + # completed we asked, and the answer was nothing + # + # Keyed on `not complete` rather than on a list of stop reasons, on + # purpose: a list cannot cover a reason nobody has added to it yet, + # so a new one falls silently through to "the catalogue is empty" — + # which is the defect this branch exists to prevent. + if blocked: + return EXIT_BLOCKED + if not complete: + print(f"[!] Nothing was gathered and the run did not finish " + f"({stop_reason}) — exit {EXIT_FETCH_FAILED}, NOT an empty " + f"result (exit {EXIT_NO_PRODUCTS}). Nothing can be " + f"concluded about the catalogue from this run.") + return EXIT_FETCH_FAILED + return rc if not complete: print(f"[!] Partial run: stopped after {pages_completed} of " f"{pages_requested} page(s) ({stop_reason}). The output holds "