One paid solve per page, atomic outputs, and matching exit codes in every engine

.github/canary_check.py

@@ -14,17 +14,17 @@
 drifts silently; this one is imported by the offline suite, which asserts it
 against output_writer's constants.
 
-Two callers, deliberately the same implementation (CLAUDE.md §17: one
-implementation invoked from both CI and the offline suite, not a workflow
-reimplementing a narrower copy of a shipped script):
+Two callers, deliberately the same implementation (one implementation
+invoked from both CI and the offline suite, not a workflow reimplementing a
+narrower copy of a shipped script):
 
     python3 .github/canary_check.py --exit-code 3 --allow-block
     python3 .github/canary_check.py --data canary_run
 
 `--allow-block` is what separates the two canary signals. From a GitHub
 runner — a datacentre address — Akamai refusing us says something about the
 runner's IP, not about Farfetch or about this code, and a check that is red
-every morning teaches everyone to ignore checks (CLAUDE.md §11). So the free
+every morning teaches everyone to ignore checks. So the free
 no-proxy signal treats a block as a SKIP and stays green, while the
 proxy-backed signal treats the same code as a failure, because from a
 residential exit a block IS news.
@@ -85,7 +85,7 @@
 # run, so three pages clear this comfortably. Deliberately far below the real
 # figure rather than tracking it: a floor guards against a near-empty render,
 # and a threshold that tracks a number which legitimately varies between runs
-# is a check that goes stale on its own (CLAUDE.md §17, check 4).
+# is a check that goes stale on its own.
 PAGES_REQUESTED = 3
 MIN_PRODUCTS = 120
 MIN_PRICED_SHARE = 0.9
@@ -102,10 +102,9 @@ def check_exit_code(code: int, allow_block: bool) -> int:
         print(f"::notice::{meaning} This canary runs from a GitHub runner, "
               f"which is a datacentre address, and no proxy secret is set — so "
               f"this says the runner's IP was refused, NOT that the site or "
-              f"this code changed. Reported as a skip rather than a failure, "
-              f"because a check that is red every morning is a check nobody "
-              f"reads. Set the FARFETCH_PROXY secret for a signal that can "
-              f"tell the two apart.")
+              f"this code changed. This job is documented as 'datacentre "
+              f"refusal expected'; the production-like job, which needs the "
+              f"FARFETCH_PROXY secret, is the one that tests real data.")
         return 0
 
     if not fails:

.github/ci_checks.py

@@ -65,7 +65,20 @@
 HEX32 = re.compile(r"\b[0-9a-f]{32}\b")
 HEX32_ALLOWED = ("sha", "hash", "nonce", "example", "md5", "digest", "checksum")
 
-SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example")
+# Data files too: a captured fixture or a generated sample is where a key or
+# a session token arrives with a page, and those were never read before.
+SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example",
+                    ".json", ".csv", ".html", ".toml", ".cfg")
+
+
+def _is_env_backup(path):
+    """`.env.bak`, `.env.local`, `.env.save` … — a renamed working copy.
+
+    Not `.env` itself, which is where a key is supposed to live, and not
+    `.env.example`, which the suffix rule already covers. A backup has no
+    useful suffix, so a suffix list alone never read one.
+    """
+    return path.name.startswith(".env.") and path.name != ".env.example"
 
 
 # A directory holding `pyvenv.cfg` is a virtualenv, whatever it is called.
@@ -76,7 +89,7 @@
 # into pip's vendored code and flags a 32-hex string in `_elffile.py` as
 # key-shaped. Correct about the string, wrong about the file — and it is the
 # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people
-# have to argue with is one they learn to suppress (CLAUDE.md §22).
+# have to argue with is one they learn to suppress.
 #
 # Deliberately NOT a narrowing of what gets scanned: an untracked file is
 # still read, because a key pasted into a scratch file beside the scripts is
@@ -105,10 +118,15 @@ def _is_in_virtualenv(path):
 
 def scanned_files():
     for path in sorted(REPO.rglob("*")):
-        if not path.is_file() or path.suffix not in SCANNED_SUFFIXES:
+        if not path.is_file():
+            continue
+        if path.suffix not in SCANNED_SUFFIXES and not _is_env_backup(path):
             continue
+        # Relative parts only. The absolute path includes every directory
+        # ABOVE the clone, so a clone that happens to live under a directory
+        # called `venv` skipped every file and reported clean.
         if any(part in {".git", "__pycache__", ".venv", "venv"}
-               for part in path.parts):
+               for part in path.relative_to(REPO).parts):
             continue
         if _is_in_virtualenv(path):
             continue
@@ -200,6 +218,10 @@ def secret_check():
                 failed.append(f"{rel}:{lineno} contains {match[:6]}… — a "
                               f"32-char hex string, the shape of a 2captcha key")
 
+    if not scanned:
+        # "Nothing credential-shaped" about nothing at all is not a pass.
+        failed.append("0 files scanned — the scan read nothing, so it cannot "
+                      "say the repository is clean")
     if not failed:
         print(f"ok       {scanned} files scanned, nothing credential-shaped")
     return failed

.github/workflows/canary.yml

@@ -14,21 +14,22 @@ name: canary
 # listed a code (124) no engine returns and had no entry for the codes they
 # do. A check that is red every morning is a check everyone learns to ignore.
 #
-#   reachability-no-proxy  — free, no secret, every day. Answers "can an
-#                            ordinary browser on a datacentre IP still get
-#                            this page, and does the parser still work on it?"
-#                            A BLOCK here is reported as a skip, not a
-#                            failure: it says something about the runner's
-#                            address, not about Farfetch. Anything else —
+#   reachability-no-proxy  — free, no secret, every day, and DOCUMENTED AS
+#                            "datacentre refusal expected": farfetch.com
+#                            refused the GitHub runner (exit 3) on every one
+#                            of the last 10 runs before 2026-10-15. A block
+#                            here is a notice, not a failure. Anything else —
 #                            0 products, a fetch failure, a partial run —
-#                            still fails, because those are real news.
+#                            still fails. This job is NOT the data test.
 #
-#   production-like        — the authoritative signal. Runs only when the
-#                            FARFETCH_PROXY secret is set, and is strict about
-#                            everything including a block, because from a
-#                            residential exit a block IS news. Skips with a
-#                            notice when the secret is absent (CLAUDE.md §11:
-#                            a canary that cannot pass must SKIP, not fail).
+#   production-like        — THE data test. Needs the FARFETCH_PROXY secret
+#                            (a residential exit) and is strict about
+#                            everything, a block included. When the secret is
+#                            absent the job FAILS with an error naming it:
+#                            a canary that never reaches the site must not
+#                            show green. Until 2026-10-15 it skipped green
+#                            instead, so the badge was green for a month
+#                            while no run reached the site.
 #
 # The proxy secret is passed through the ENVIRONMENT, never as a --proxy
 # argument. env_config.py already reads FARFETCH_PROXY, an unset secret
@@ -46,7 +47,7 @@ on:
 
 jobs:
   reachability-no-proxy:
-    name: reachability (no proxy, datacentre IP)
+    name: reachability (no proxy, datacentre refusal expected)
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
@@ -131,9 +132,15 @@ jobs:
     env:
       HAVE_PROXY: ${{ secrets.FARFETCH_PROXY != '' }}
     steps:
-      - name: Explain the skip
+      - name: Require the proxy secret
+        # Fails, not skips: without a residential exit this canary cannot
+        # reach the site, and a job that never reaches the site must not
+        # end green. The reachability job is no substitute for this one —
+        # from a datacentre address it is refused every day.
         if: env.HAVE_PROXY != 'true'
-        run: echo "::notice::Skipped - no FARFETCH_PROXY secret is set, so there is no residential exit to run from. farfetch.com refuses GitHub's datacentre addresses, so running this job without one would be red every morning while saying nothing about the site. The reachability job above still runs for free. Set the secret to turn this signal on; it is read from the environment by env_config.py and never reaches a command line."
+        run: |
+          echo "::error::The FARFETCH_PROXY secret is not set, so this canary cannot reach farfetch.com: the site refuses GitHub's datacentre addresses. Add a residential proxy URL as the repository secret FARFETCH_PROXY (read from the environment by env_config.py, never put on a command line)."
+          exit 1
 
       - uses: actions/checkout@v7
         if: env.HAVE_PROXY == 'true'
@@ -157,8 +164,8 @@ jobs:
         env:
           # Read by env_config.py, exactly as a local .env would be. NOT
           # passed as --proxy: a credential in argv is readable by anything
-          # that can run `ps` and is the rule this repo states in §3 of its
-          # own docs.
+          # that can run `ps`, which is why the README tells users to keep
+          # credentials in .env.
           FARFETCH_PROXY: ${{ secrets.FARFETCH_PROXY }}
         run: |
           set +e

.gitignore

@@ -19,6 +19,16 @@ test_farfetch*.*
 # --- the live-test harness's working directory ---------------------------
 live_results/
 live_results.zip
+# By KIND of directory, not by one chosen name: a raw capture reaches
+# `git add -A` through whatever directory nobody listed.
+live/
+captures/
+dumps/
+scratch/
+# `--dump-html X --pages N` writes X.page2, X.page3 …, which `*.json` does not
+# cover when X ends in .json, and a bare page capture at the repo root.
+*.page[0-9]*
+/*.html
 
 # --- debug artifacts the diagnostics drop next to themselves -------------
 # check_signup_captcha*.py write these on purpose, for manual inspection.
@@ -66,7 +76,6 @@ secrets*
 # recoverable only because that repo was private at the time. On 2026-09-17
 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC
 # repository, one `git add -A` from publication. Craigslist was fixed then
-# and the fix never reached anyone else, which is §16's rule unapplied: when
-# you fix one, check the siblings.
+# and the fix never reached anyone else: when you fix one, check the siblings.
 .env*
 !.env.example

CHANGELOG.md

@@ -8,8 +8,76 @@ a promise that every flag and exit code is contractually frozen.
 
 ## [Unreleased]
 
+> **Behaviour changes.** (1) The pyppeteer and Selenium engines now report an
+> HTTP 4xx/5xx page whose body names no known bot-check vendor as exit 5
+> (content never obtained) instead of exit 4 (empty category), and a page that
+> links to products but parses to none as exit 5 (`stop_reason: parse_drift`,
+> exit 6 if earlier pages were gathered) — the same codes the Playwright
+> engine already gave. (2) The CSV output now prefixes a string cell that
+> begins with `=`, `+`, `-`, `@`, a tab or a line break with an apostrophe,
+> so a spreadsheet reads it as text; JSON is unchanged and the sidecar
+> records how many cells were escaped as `csv_cells_escaped`. (3) A refused
+> `--cdp-endpoint` connect now exits 5 with a one-line, credential-free
+> message instead of a traceback (exit 1).
+> (4) The canary's production-like job now FAILS with an error naming
+> `FARFETCH_PROXY` when that secret is absent, instead of skipping green. The
+> no-proxy job was refused (exit 3) on every one of the last 10 runs, so for a
+> month the badge was green while no run reached the site. The canary will
+> stay red until the secret is set.
+
 ### Fixed
 
+- **One paid captcha solve per page, across every retry and exit rotation.**
+  The Playwright engine called its captcha handler once per block-retry with
+  nothing counting purchases, so with a 3-exit proxy pool and
+  `--solve-captcha always` one blocked page bought three solves (reproduced
+  with the solver stubbed). A per-page `SolveBudget` is now checked before
+  every purchase in all three engines and is not reset by a rotation.
+- **Output files are written atomically.** `write_json`, `write_csv` and the
+  `.meta.json` sidecar went through `open(path, "w")`, which truncates first:
+  a `json.dump` failing part-way turned a 433-byte good file into 12 bytes of
+  invalid JSON. They now write a temp file beside the target, fsync it and
+  rename it over the target. A new file gets the same mode `open()` would
+  have given it (`0o666 & ~umask`); an existing file keeps its own.
+- **A failed `--cdp-endpoint` connect no longer prints the endpoint.**
+  Playwright's connect error repeats the endpoint, `user:pass` included, in
+  its message and call log, and nothing caught it. It is now caught, every
+  occurrence redacted, a 401 explained as wrong or expired credentials, and a
+  `profile_locked`/HTTP 500 answer retried up to three times, 3 s apart. The
+  pyppeteer connect is bounded at 30 s (it had no timeout).
+- **pyppeteer keeps the HTTP status of the page it loaded, and Selenium reads
+  it from the browser** (`PerformanceNavigationTiming.responseStatus`), so the
+  mirrors classify an error page the way Playwright does. Both now also name
+  parser drift.
+- **The suite's skip report printed empty whatever happened.** `main()`
+  rebound `_skips` and `_skipped_engines` locally, shadowing the records the
+  sections wrote to, so the `SKIPPED_ENGINES:` line the engine-smoke CI job
+  reads could never name a skipped engine. A check now asserts every engine
+  whose driver is absent is listed.
+- **The credential scan reported clean after reading nothing.** Exclusions
+  were tested against the ABSOLUTE path, so a clone under any directory
+  called `venv` scanned 0 files and printed "nothing credential-shaped". They
+  are now tested on the path relative to the repo, a scan of 0 files fails,
+  and JSON/CSV/HTML/TOML/CFG files and renamed `.env.*` backups are read too.
+- **A value still holding a `{…}` slot from the documented format** (the
+  Scraping Browser endpoint or proxy shape in `.env.example`) now reads as
+  unset instead of being sent as a real credential, and an empty value no
+  longer logs a "still set to the placeholder" warning.
+- **The DOM fallback no longer defaults a missing currency to USD.** A tile
+  price whose currency the reader could not attribute was written as `USD`;
+  it is now `null`. Currency comes only from structured data, a written ISO
+  code, or a prefixed or bare symbol on the tile. Every price shape the
+  fallback recognises today carries one of those, so current output is
+  unchanged; the default could only have fired on an unrecognised shape.
+- `.gitignore` covers the capture shapes the engines write themselves
+  (`X.page2`, `X.page3` from `--dump-html`), a bare `.html` at the repo root,
+  and `live/`, `captures/`, `dumps/`, `scratch/`.
+- The README exit-code table and the `stop_reason_for` docstring said parser
+  drift exits 4; it exits 5 (6 with earlier pages gathered).
+- Comments, docs and the v0.4.3 notes no longer point at private notes or
+  bare section numbers that no reader of this repository can open; a suite
+  check keeps them out.
+
 - **Scraper API: `waitFor` is now sent as a JSON object.** Measured
   2026-09-23 against `scraper.2captcha.com/tasks/sync`: the JSON-encoded
   string this client sent (the form older docs described) is refused with
@@ -372,7 +440,7 @@ PR. Of P2, three of four.
 
 - Offline suite: **319 checks**, up from 232; `pytest` reports 321 results. New coverage includes the
   Dockerfile's COPY list against the entrypoint's import graph — a check
-  CLAUDE.md §10 calls for after every repo in this family shipped an image
+  worth having after every repo in this family shipped an image
   that died on every invocation, and which did not exist here. It failed on
   its first run, catching a module added in this same batch.
 
@@ -382,9 +450,9 @@ PR. Of P2, three of four.
 
 - **`fingerprint_client.py` could not read the key from `.env`.** `--key`
   defaulted to `os.environ.get("TWOCAPTCHA_KEY")` and only that, so a key put
-  in `.env` — exactly as §3, the README and `.env.example` instruct — worked
+  in `.env` — exactly as the README and `.env.example` instruct — worked
   for every engine and failed HERE with "No API key". A documented mechanism
-  not applied on one path, which is the shape of half the defects §16 lists.
+  not applied on one path.
 
   It now reads through `env_config.env_value`, calling `load_env()` itself
   because this is a standalone entry point that no engine has necessarily run
@@ -396,7 +464,7 @@ PR. Of P2, three of four.
   "still set to the placeholder from .env.example" instead.
 
   Found on a sibling repo's first live `--fingerprint` run, then checked
-  across the family before patching, per §16: five repos had it and one had
+  across the family before patching: five repos had it and one had
   already fixed it. Pinned by a check verified to fail on the old code —
   including that the help string does not interpolate its default, which is
   one substring away from printing a live credential to anyone who types

README.md

@@ -282,8 +282,8 @@ A contract, not decoration — the harness and any pipeline can branch on these.
 | `1` | Unhandled error |
 | `2` | Bad usage |
 | `3` | **Blocked before parsing** — a challenge page, or an outright refusal |
-| `4` | Fetched the page fine, parsed **0 products** |
-| `5` | **Never got the page** — navigation timeout, dead proxy, 4xx/5xx, or a Scraper API error |
+| `4` | Fetched the page fine, parsed **0 products**, and the page links to no products either |
+| `5` | **No content obtained** — navigation timeout, dead proxy, 4xx/5xx, a failed `--cdp-endpoint` connect, a Scraper API error, or a page that links to products and parsed to none (`stop_reason: parse_drift` — a parser problem, not a network one) |
 | `6` | **Partial run** — products written, but the page loop stopped early |
 | `124` | Selenium only: chromedriver could not be **started** within the watchdog |
 

captcha_solver.py

@@ -589,6 +589,37 @@ def solve_recaptcha(challenge: CaptchaChallenge, twocaptcha_api_key: Optional[st
     return solver(twocaptcha_api_key, challenge, min_score=min_score)
 
 
+# How many paid solves one page may buy, across every attempt and every exit
+# rotation. One: a challenge that survives a solved token is not one this run
+# can pass, and a second solve is a second charge for the same answer.
+SOLVES_PER_PAGE = 1
+
+
+class SolveBudget:
+    """The one allowance every paid solve on a page goes through.
+
+    Created once per PAGE, outside the retry and rotation loops, and checked
+    BEFORE each purchase. Before it existed the Playwright engine called its
+    captcha handler once per block-retry with nothing counting, so with a
+    3-exit proxy pool one page bought three solves (reproduced with the
+    solver stubbed). It deliberately does NOT reset on rotation: a fresh exit
+    is a reason to re-fetch, not a fresh allowance to spend.
+    """
+
+    def __init__(self, limit: int = SOLVES_PER_PAGE):
+        self.limit = limit
+        self.spent = 0
+
+    def take(self) -> bool:
+        """Reserve one solve. False, with a log line, once the page is spent."""
+        if self.spent >= self.limit:
+            logger.info("Not buying another solve for this page: %d of %d "
+                        "already bought.", self.spent, self.limit)
+            return False
+        self.spent += 1
+        return True
+
+
 INJECT_TOKEN_JS = """
 (token) => {
   let el = document.getElementById('g-recaptcha-response');