Report parser failures and blocks honestly, cap paid solves per page, write output atomically

.dockerignore

@@ -14,4 +14,4 @@ __pycache__/
 .venv*/
 README.md
 CHANGELOG.md
-CLAUDE.md
+*.md

.github/workflows/canary.yml

@@ -1,10 +1,16 @@
 name: canary
 
-# One real run daily against a known-good FILTERED CATEGORY URL
-# (CLAUDE.md §11). Requests >=3 pages -- with one page, pagination is
-# never exercised, which is exactly how a silent-single-page bug would
-# survive undetected. Uploads artefacts on every run so a red canary can
-# be diagnosed from the log alone.
+# One real run daily against a known-good search URL, >=3 pages, so
+# pagination is exercised and not only page 1. Artefacts are uploaded on
+# every run, so a red canary can be diagnosed from the run alone.
+#
+# This canary ends green ONLY when the scraper read real listings from
+# zimmo.be. zimmo.be refuses GitHub's datacentre addresses (Cloudflare
+# 403, measured 2026-09-23 and 2026-10-13) and serves only a residential
+# exit (Belgian or German worked; a US exit was refused) with a headful
+# browser, so the run needs the ZIMMO_PROXY repository secret. Without
+# it the job FAILS with an error naming the secret: a skipped or
+# "refused, as expected" run is not a test of the site.
 
 on:
   schedule:
@@ -18,40 +24,22 @@ on:
       - ".github/workflows/canary.yml"
 
 jobs:
-  # `secrets` is not available in a job-level `if`, so presence is tested
-  # here and handed on as an output. The canary job below is then SKIPPED
-  # (shown as skipped, not as a green run) when no ZIMMO_PROXY is set: a
-  # run from the runner's own address is refused by Cloudflare every time
-  # (README, "What you need"), so running it would test nothing and
-  # reporting it as a pass would be a claim nobody measured.
-  gate:
-    runs-on: ubuntu-22.04
-    env:
-      HAS_ZIMMO_PROXY: ${{ secrets.ZIMMO_PROXY != '' }}
-    outputs:
-      has_proxy: ${{ steps.g.outputs.has_proxy }}
-    steps:
-      - id: g
-        run: |
-          echo "has_proxy=$HAS_ZIMMO_PROXY" >> "$GITHUB_OUTPUT"
-          if [ "$HAS_ZIMMO_PROXY" != "true" ]; then
-            echo "::notice::canary SKIPPED, not passed: no ZIMMO_PROXY secret is set, and zimmo.be refuses this runner's address. The site needs a residential exit plus a headful browser (README, 'What you need'). Add a residential proxy as the ZIMMO_PROXY repository secret to make this canary test the scraper daily."
-          fi
-
   canary:
-    needs: gate
-    if: needs.gate.outputs.has_proxy == 'true'
     # jammy, pinned, for the same reason tests.yml's engine-smoke job
     # pins it: playwright==1.44.0 cannot install its system deps on
     # Ubuntu 24.04, and the Dockerfile's base image is v1.44.0-jammy.
     runs-on: ubuntu-22.04
-    # What zimmo.be lets through, measured 2026-09-23 (README, "What you
-    # need"): a residential exit AND a visible (headful) browser. Headless
-    # Chromium is refused even from a Belgian home address, and a
-    # datacentre address is refused either way.
-    env:
-      HAS_ZIMMO_PROXY: "true"
     steps:
+      - name: Require the ZIMMO_PROXY secret
+        env:
+          ZIMMO_PROXY: ${{ secrets.ZIMMO_PROXY }}
+        run: |
+          if [ -z "$ZIMMO_PROXY" ]; then
+            echo "::error::ZIMMO_PROXY is missing or empty, so this canary cannot reach zimmo.be (it refuses GitHub runners' addresses). Set the ZIMMO_PROXY repository secret to a residential proxy URL (credentials, host and port) with a Belgian or German exit. This run tested nothing."
+            exit 1
+          fi
+          echo "[canary] ZIMMO_PROXY is set (value masked)."
+
       - uses: actions/checkout@v4
 
       - uses: actions/setup-python@v5
@@ -66,9 +54,8 @@ jobs:
 
       - name: Run against a known-good filtered category, >=3 pages (headful, under xvfb)
         env:
-          # The proxy goes through the ENVIRONMENT, never a command line
-          # (CLAUDE.md §3, §11). An unset secret arrives as an empty
-          # string, which env_config.py treats as unset.
+          # The proxy goes through the ENVIRONMENT, never a command line,
+          # so it cannot appear in the process list or the job log.
           ZIMMO_PROXY: ${{ secrets.ZIMMO_PROXY }}
         run: |
           set +e
@@ -91,7 +78,7 @@ jobs:
 
           code = int(os.environ.get("exit_code", "1"))
           if code != 0:
-              print(f"::error::engine exit code {code} (0 ok, 3 blocked, 4 zero rows, 5 content never obtained, 6 partial)")
+              print(f"::error::engine exit code {code} (0 ok, 3 blocked, 4 zero rows, 5 content never obtained or parser found nothing on a rendered page, 6 partial)")
               sys.exit(1)
 
           meta = json.load(open("canary_run.meta.json"))

.github/workflows/tests.yml

@@ -1,6 +1,6 @@
 name: tests
 
-# Offline only (CLAUDE.md §11) — nothing here touches zimmo.be or a
+# Offline only — nothing here touches zimmo.be or a
 # credential. See canary.yml for the one real live run.
 on:
   push:
@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         # Oldest and newest supported Python — claiming 3.9 without
-        # testing it is how a walrus operator ships (CLAUDE.md §11).
+        # testing it is how a walrus operator ships.
         python-version: ["3.9", "3.12"]
     steps:
       - uses: actions/checkout@v4
@@ -107,7 +107,7 @@ jobs:
           python-version: "3.12"
 
       # One venv per engine, installed from its OWN requirements file
-      # (CLAUDE.md §11) — a single shared environment lets pip silently
+      # — a single shared environment lets pip silently
       # resolve the family's mutually-unsatisfiable pins by reaching
       # for whatever it can, which stays green against a version
       # nobody ships.
@@ -141,7 +141,7 @@ jobs:
   docker-build:
     runs-on: ubuntu-latest
     # "Build the Docker image in CI. Nothing else does, which is how
-    # all three [sibling] repos shipped a broken one." (CLAUDE.md §11)
+    # all three [sibling] repos shipped a broken one."
     # Four steps, each catching a DISTINCT failure a manual COPY-list
     # review cannot: the image builds; its entrypoint actually runs;
     # Chromium actually launches (not just `playwright install
@@ -173,7 +173,7 @@ jobs:
         run: |
           docker create --name zimmo-check zimmo-scraper:ci --help
           FOUND=0
-          for forbidden in .env tests sample_output.json sample_output.csv README.md CLAUDE.md; do
+          for forbidden in .env tests sample_output.json sample_output.csv README.md CHANGELOG.md CONTRIBUTING.md; do
             if docker cp zimmo-check:/app/$forbidden /tmp/should-not-exist 2>/dev/null; then
               echo "FORBIDDEN: /app/$forbidden exists inside the image"
               rm -rf /tmp/should-not-exist

.gitignore

@@ -1,5 +1,5 @@
 # Scraper output — large, machine-specific, and stale by the time anyone
-# reads it. Sample files are the one deliberate exception (CLAUDE.md §2).
+# reads it. Sample files are the one deliberate exception.
 *.json
 *.csv
 !sample_output.json
@@ -11,7 +11,7 @@ zimmo_listings.*
 
 # Local diagnostics / live-test working directory — and ANY run directory by
 # kind, not one chosen name (a sibling committed a raw dump through `live/`),
-# plus paged dumps like X.page3 (family template §24, §26).
+# plus paged dumps like X.page3.
 live_results/
 live/
 out/
@@ -69,6 +69,6 @@ secrets*
 
 # Every spelling of a working .env, not just `.env`: a `.env.bak` or
 # `.env.local` is exactly how a live key gets one `git add -A` from a commit
-# (family template §22). The negation keeps the documented example tracked.
+#. The negation keeps the documented example tracked.
 .env*
 !.env.example

CHANGELOG.md

@@ -7,6 +7,75 @@ manage. A patch release means "fixes" — not that every flag is frozen,
 so a behaviour-changing default landing in a patch is stated plainly
 here rather than treated as a violation of the format.
 
+## [Unreleased]
+
+> **Exit codes.** A page whose listing grid rendered but parsed to zero
+> rows is now a parser failure: exit 5 when no row was recovered, exit 6
+> (`partial`, `parser_failed_pages` in the sidecar) when other pages had
+> rows. It used to be reported as an empty search, exit 4. A run with no
+> rows where one page was served empty and another failed is now exit 5,
+> not 4, and the Scraper API path now answers 4 for a served empty page
+> and 3 for a Cloudflare challenge (both were 5).
+>
+> **Rejected flag values (exit 2).** `--retries`, `--delay` and
+> `--retry-delay` below 0, and `--pages`, `--concurrency` and
+> `--proxy-block-retries` below 1, are argparse errors in all three
+> engines (`--retries -1` used to run no navigation at all and report exit
+> 5; `--retries 0` still means one attempt). `--proxy`/`--proxy-file`
+> together with `--cdp-endpoint` (from flags or `.env`) is refused instead
+> of the proxy being silently ignored. `--captcha-api` is removed: nothing
+> ever read it.
+>
+> **CSV cells and file modes.** In the CSV only, a text cell starting with
+> `=`, `+`, `-`, `@`, a tab, CR or LF gets a leading apostrophe so a
+> spreadsheet does not run it as a formula; numbers and the JSON are
+> unchanged, and the sidecar counts the escaped cells in
+> `csv_cells_escaped`. Output files keep an existing file's mode; a new one
+> gets what the umask allows, as before.
+
+### Fixed
+
+- Output files (`.json`, `.csv`, `.meta.json`) are written to a temporary
+  file in the same directory and moved into place only when complete, so a
+  failed or interrupted write leaves the previous good file intact instead
+  of truncated.
+- One page buys at most one paid captcha solve, however many attempts,
+  proxy rotations or challenge markers it goes through. The allowance is
+  checked before each purchase in all three engines; a missing key or an
+  unsolvable Managed Challenge costs nothing and does not use it up.
+- `currency` is null when nothing states one (it defaulted to `EUR` on the
+  embedded-JSON and JSON-LD paths, which today's pages do not use).
+- The credential scan now also reads `.json`, `.csv`, `.html` and `.cfg`
+  files, recognises a virtualenv by its `pyvenv.cfg` under any name,
+  judges the `sample_output` exemption by file name rather than by the
+  whole path, and refuses to report a tree clean when it read no file.
+- README: the Scraping Browser credential lifetime is no longer stated as
+  "about a day"; an HTTP 401 on connect is described as credentials no
+  longer accepted. The Docker image is documented as needing
+  `--cdp-endpoint` or a display, and the Dockerfile no longer claims a CI
+  step that did not exist (the offline suite now checks its COPY list).
+- Comments, docs and tests no longer cite private maintainer notes; each
+  rule is stated in place. The offline suite fails if such a citation
+  returns.
+
+### Changed
+
+- The canary no longer skips green: without a usable `ZIMMO_PROXY` secret
+  (missing or empty) it fails with an error naming it, because zimmo.be
+  refuses GitHub's runners and a run that never reached the site tested
+  nothing. Any nonzero exit (block, zero rows, parser failure, partial)
+  is red.
+
+### Not changed
+
+- Challenge markers (`turnstile`, `cf-turnstile-response`) that a Scraping
+  Browser session may inject: changing them needs real served and refused
+  captures over `--cdp-endpoint`, which this change did not take.
+- The family flags `--fingerprint`, `--fp-*`, `--locale` and `--mode`, and
+  whether to keep `fingerprint_client.py`: an owner decision.
+- The three per-engine fetch loops (no shared-loop refactor); version,
+  tag and release of 1.3.0.
+
 ## [1.3.0] — a published row schema, three columns removed, a canary that can say "not run" (2026-10-13)
 
 > **Output change.** `rating`, `review_count` and `in_stock` are gone from
@@ -411,7 +480,7 @@ fixed, and each has its own regression test in `smoke_test.py`.
   confirmed live via the proxy provider's own usage dashboard, not
   just this project's own output.
 
-## [0.3.0] — External audit fixes (codex.md, 2026-09-14)
+## [0.3.0] — External audit fixes (2026-09-14)
 
 An independent audit ran this project against the live site and its own
 test suite and found ten confirmed defects, several severe. Every
@@ -573,7 +642,7 @@ patched on the audit's word alone — see each item for how.
   match the family default. JSON-LD parsing now also handles `@graph`
   and an `image` field shaped as an `ImageObject` dict. `SELECTORS`'
   key renamed from an ad-hoc `item_link_css_approx` to the family's own
-  `item_link` convention (CLAUDE.md §5).
+  `item_link` convention.
 - `env_config.py`: a value containing `{...}` (a vendor's own braced
   placeholder, e.g. `ws://{login}-zone-...`) is now treated as unset,
   not just a fixed literal list — a copied `.env.example` following the

CONTRIBUTING.md

@@ -2,8 +2,9 @@
 
 Thanks for considering a contribution. This repo is part of the
 [2scraper](https://github.com/2scraper) family — several single-site
-scrapers sharing one architecture (`~/2scraper/CLAUDE.md`, not committed
-to this repo, describes it in full for maintainers).
+scrapers sharing one architecture: shared modules with no site knowledge
+(`output_writer.py`, `env_config.py`, `proxy_pool.py`, `captcha_solver.py`)
+and the site itself confined to `product_parser.py`.
 
 ## Before you start
 

Dockerfile

@@ -1,10 +1,12 @@
 # zimmo-scraper — Playwright edition, containerised.
 #
-# Explicit COPY list, not `COPY . .` (CLAUDE.md §11): the image should
-# carry no test suite, no fixtures and no stray .env. The list below is
-# checked against playwright_scraper.py's own import graph by this
-# repo's docker-build CI job (via a real `ast` walk, not a manual
-# review) -- keep the two in sync by hand.
+# Explicit COPY list, not `COPY . .`: the image should
+# carry no test suite, no fixtures and no stray .env. smoke_test.py
+# checks the list below against playwright_scraper.py's own local import
+# graph (an `ast` walk), so a missing module fails the offline suite.
+#
+# The default run is headless, and zimmo.be refuses headless Chromium:
+# use the image with --cdp-endpoint, or give it a display and --headful.
 
 FROM mcr.microsoft.com/playwright/python:v1.44.0-jammy
 
@@ -18,7 +20,7 @@ RUN pip install --no-cache-dir -r requirements.txt -r requirements-playwright.tx
 COPY env_config.py output_writer.py product_parser.py captcha_solver.py \
      proxy_pool.py playwright_scraper.py ./
 
-# The exact invocation a missing module breaks (CLAUDE.md §11) -- CI's
+# The exact invocation a missing module breaks -- CI's
 # docker-build job actually runs this, rather than trusting that the
 # image "should" work.
 ENTRYPOINT ["python3", "playwright_scraper.py"]