Exit 5 for content never obtained, exit 2 for usage errors, CSV formula guard, umask-respecting outputs

.github/ci_checks.py

@@ -103,7 +103,8 @@ def sample_check(prefix="sample_output"):
 
 
 def secret_check():
-    """Delegate to tools/scan_secrets.py — ONE scanner, not two (§21)."""
+    """Delegate to tools/scan_secrets.py — ONE scanner, not two, so CI and
+    the git hooks cannot disagree about what is a secret."""
     result = subprocess.run(
         [sys.executable, str(REPO / "tools" / "scan_secrets.py"), "--worktree"],
         capture_output=True, text=True, cwd=REPO)
@@ -155,15 +156,19 @@ def run_check(prefix="sample_output"):
     # each fail quietly in their own way on this site:
     #   price        — the grid painted before its prices did
     #   region_slug  — every item URL states it; empty means the URL was lost
-    #   seller_name  — 49 of 50 cards named a seller when measured
+    #   sku/url/title — the row's identity; a card read without them is a
+    #                   parser fault, whatever the category is selling
     priced = sum(1 for r in rows if r.get("price") is not None)
     if priced < 0.9 * len(rows):
         failed.append(f"only {priced}/{len(rows)} rows carry a price")
     if not all(r.get("region_slug") for r in rows):
         failed.append("a row has no region_slug")
-    named = sum(1 for r in rows if r.get("seller_name"))
-    if named < 0.5 * len(rows):
-        failed.append(f"only {named}/{len(rows)} rows name a seller")
+    # Not a seller-name share: how many private vs. company sellers a category
+    # shows is merchandising, not the parser.
+    for column in ("sku", "url", "title"):
+        empty = sum(1 for r in rows if not r.get(column))
+        if empty:
+            failed.append(f"{empty}/{len(rows)} rows have no {column}")
     inverted = [r for r in rows if r.get("original_price") is not None
                 and r.get("price") is not None and r["original_price"] <= r["price"]]
     if inverted:

.github/workflows/canary.yml

@@ -18,9 +18,11 @@ name: canary
 # different exit is rejected (measured).
 #
 # Needs: secrets TWOCAPTCHA_KEY and AVITO_PROXY (the base gateway login), and
-# repository variables AVITO_BROWSER_ACCOUNT_ID / AVITO_BROWSER_PROFILE_ID.
-# Without them the job goes GREEN with a ::notice:: — a check that is always
-# red teaches everyone to ignore checks.
+# repository variable AVITO_BROWSER_ACCOUNT_ID (AVITO_BROWSER_PROFILE_ID is
+# optional). Without them the job FAILS, naming what is missing: a green
+# canary must mean avito.ru was scraped, and a skip that reads green said
+# nothing about the site for weeks. A blocked run (exit 3), a run that never
+# reached the site (5) or zero rows is red too.
 on:
   schedule:
     - cron: "23 5 * * *"
@@ -33,30 +35,31 @@ jobs:
     env:
       # `secrets` is not available in a job-level `if`; presence goes through
       # job env and is read in each step's `if`.
-      READY: ${{ secrets.TWOCAPTCHA_KEY != '' && secrets.AVITO_PROXY != '' && vars.AVITO_BROWSER_ACCOUNT_ID != '' }}
+      HAS_TWOCAPTCHA_KEY: ${{ secrets.TWOCAPTCHA_KEY != '' }}
+      HAS_AVITO_PROXY: ${{ secrets.AVITO_PROXY != '' }}
+      HAS_ACCOUNT_ID: ${{ vars.AVITO_BROWSER_ACCOUNT_ID != '' }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
 
-      - name: Explain the skip
-        if: env.READY != 'true'
+      - name: Require the live-run credentials
+        # A GitHub runner is not a Russian residential address, so there is no
+        # live path without these. Missing ones fail the job by name.
         run: |
-          echo "::warning::LIVE RUN SKIPPED — this green run proves the workflow is healthy, NOT that avito.ru was scraped."
-          echo "::notice::Needs secrets TWOCAPTCHA_KEY and AVITO_PROXY and the"
-          echo "::notice::variable AVITO_BROWSER_ACCOUNT_ID. A GitHub runner is not a"
-          echo "::notice::Russian residential address; a live run from here without the"
-          echo "::notice::Scraping Browser would be red every day regardless of the code."
+          missing=0
+          [ "$HAS_TWOCAPTCHA_KEY" = "true" ] || { echo "::error::secret TWOCAPTCHA_KEY is not set — the canary cannot reach avito.ru"; missing=1; }
+          [ "$HAS_AVITO_PROXY" = "true" ] || { echo "::error::secret AVITO_PROXY (the base Russian gateway login) is not set"; missing=1; }
+          [ "$HAS_ACCOUNT_ID" = "true" ] || { echo "::error::repository variable AVITO_BROWSER_ACCOUNT_ID is not set"; missing=1; }
+          exit $missing
 
       - name: Install
-        if: env.READY == 'true'
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt -r requirements-playwright.txt
 
       - name: Build a fresh endpoint and its matching solver proxy
-        if: env.READY == 'true'
         # Credentials through the ENVIRONMENT, never argv. The tool writes
         # AVITO_CDP_ENDPOINT and the minted AVITO_PROXY into a 0600 .env and
         # prints both masked.
@@ -71,7 +74,6 @@ jobs:
             --mint-session --session-minutes 60 --write-env
 
       - name: Live run
-        if: env.READY == 'true'
         # AVITO_PROXY is deliberately NOT exported here: an exported variable
         # outranks .env, and the base login would replace the minted session
         # the endpoint was built with.
@@ -87,10 +89,13 @@ jobs:
           set -e
 
       - name: Assert the data, not just the exit code
-        if: env.READY == 'true'
         # THE SHIPPED CHECK, not an inline copy (a 2scraper family rule).
         run: |
           echo "exit code from the run: $rc"
+          if [ "$rc" != "0" ]; then
+            echo "::error::the live run exited $rc (3 blocked, 4 no rows, 5 content never obtained, 6 partial) — the canary did not get a complete scrape"
+            exit 1
+          fi
           python3 .github/ci_checks.py --run-check --sample-check --output-prefix canary
 
       - name: Remove the minted credentials

.github/workflows/tests.yml

@@ -64,7 +64,7 @@ jobs:
         run: python3 .github/ci_checks.py --wheel-check
 
       - name: No credentials committed
-        # THE SHIPPED CHECK — one scanner, shared with the git hooks (§21).
+        # THE SHIPPED CHECK — one scanner, shared with the git hooks.
         run: python3 .github/ci_checks.py --secret-check
 
   engine-smoke:

.gitignore

@@ -26,6 +26,9 @@ canary*.meta.json
 *_debug.png
 *.html.page*
 *_p[0-9]*.html
+# --dump-html with any extension writes <root>_p<N><ext>
+*_p[0-9]*.txt
+*_p[0-9]*.htm
 
 # --- raw research captures ---------------------------------------------------
 recon/

CHANGELOG.md

@@ -6,6 +6,72 @@ follow SemVer as closely as a CLI toolkit can: a **patch** means fixes, and a
 fix may change a default when the old one was wrong — such a change leads its
 entry in a blockquote.
 
+## [Unreleased]
+
+> **Exit codes and file modes changed.** A run that ends with zero rows for
+> any reason other than a listing read to its end now exits **5** (content
+> never obtained) instead of **4** — an item page with no item on it, a
+> seller profile that never parsed, a failed page. A wrong host, a
+> robots-disallowed URL, the wrong `--mode`, two URLs outside item mode and
+> an unreadable `--from` file now exit **2** instead of **1**. Output files
+> were created `0600`; they now follow the umask, and an existing file keeps
+> its mode. Selenium with a typed `--cdp-endpoint` now exits **2** instead of
+> **5**. **At most one paid captcha solve per page per run:** a page that
+> stays walled used to buy one solve per exit (2 with the default
+> `--proxy-block-retries 1`, 4 with 3); rotating to a fresh exit still
+> re-fetches the page, but no longer buys a second token for it.
+
+### Fixed
+
+* **Zero rows read as an empty catalogue.** `finish_run` returned 4 for
+  every zero-row run that was not blocked, so a removed item or a page that
+  failed looked like "nothing listed". Only the stop reasons that mean the
+  listing was read to its end keep 4; any other one, including one added
+  later, is 5.
+* **Usage errors exited 1**, which the exit table calls a crash, and a
+  missing `--from` file ended in a traceback. They are now argparse errors
+  (exit 2), checked before any browser starts.
+* **Outputs were owner-only.** The atomic writer's temp file is created
+  `0600` and the rename kept it. New files now get `0o666 & ~umask`; an
+  existing file keeps its own mode. Applies to the rows, the sidecar and the
+  attempt sidecar.
+* **Selenium and the Scraping Browser.** An endpoint from `.env` (shared by
+  all three engines) made every Selenium run exit 5 "transport_error". It is
+  now ignored with a notice; a typed `--cdp-endpoint` is a usage error.
+* **The secret scan read tracked files only.** `tools/scan_secrets.py
+  --worktree` (and `ci_checks.py --secret-check`) now also reads untracked
+  files that are not ignored, skips any directory holding `pyvenv.cfg`, prints
+  how many files it read, and fails when it read none instead of reporting
+  "clean".
+* **Paid solves are capped per page.** The allowance
+  (`page_flow.SOLVES_PER_PAGE = 1`) is checked before each purchase, keyed by
+  the page's URL, and not reset by a retry or by rotating to a fresh exit. A
+  refused purchase is logged and counted as `solves_refused_by_budget` in
+  `transport_facts`. The Scraping Browser's own auto-solve is not counted.
+* **pyppeteer's Scraping Browser connect is bounded** (30 s); a rejected
+  handshake could leave it waiting forever.
+
+### Added
+
+* **CSV formula guard.** Text cells starting with `=`, `+`, `-`, `@`, tab, CR
+  or LF get a leading `'` in the CSV only (after lists and dicts are joined);
+  numbers are untouched and the JSON keeps the site's text. The sidecar
+  records `csv_cells_escaped`.
+
+### Changed
+
+* **The canary no longer goes green without scraping.** Without
+  `TWOCAPTCHA_KEY`, `AVITO_PROXY` or `AVITO_BROWSER_ACCOUNT_ID` it fails,
+  naming the missing one, instead of taking a green skip branch; a live run
+  that does not exit 0 is red. Its data check no longer requires half the rows
+  to name a seller (the private/company mix is the category's, not the
+  parser's) and instead requires `sku`, `url` and `title` on every row.
+* `requirements-playwright.txt` asks for `playwright>=1.44`, as the
+  `[playwright]` extra already did.
+* `.gitignore` covers `--dump-html` names with a `.txt` or `.htm` extension.
+* Comments and docstrings no longer point at internal documents or at other
+  repositories; each states its reason here.
+
 ## [1.0.2] — 2026-10-10
 
 Fixes from a review of v1.0.1, each reproduced against v1.0.1 before it

CONTRIBUTING.md

@@ -71,7 +71,8 @@ wrong:
 - **The wall is recognised by its `<form>`**, not by the class name, which the
   wall's own script also mentions.
 - **Exit codes are a contract**: `0` ok, `1` crash, `2` bad usage, `3`
-  blocked, `4` zero rows, `5` transport error, `6` partial. A transport fault
+  blocked, `4` zero rows from a listing read to its end, `5` content never obtained
+  (transport, or zero rows for any other reason), `6` partial. A transport fault
   is never `3`.
 - **A run that finds nothing writes nothing.** `--allow-empty` is the opt-out.
 - **No phone, no IMEI, no private seller's placeholder name** in any row.

README.md

@@ -18,12 +18,14 @@ actually needs.
 [![engines](https://img.shields.io/badge/engines-Playwright%20%7C%20Selenium%20%7C%20pyppeteer%20%7C%20CDP-informational)](#engines)
 [![needs a Russian exit](https://img.shields.io/badge/needs-a%20Russian%20exit%20%2B%20a%20browser-orange)](#the-one-thing-you-actually-need)
 
-> **What the canary badge means.** The daily canary scrapes avito.ru only when
-> the repository holds the secrets for a Russian exit and a captcha key. This
-> public repository does not, so the canary takes its documented SKIP branch:
-> a green badge here says the workflow is healthy, **not** that avito.ru was
-> scraped today. Each skipped run says so in a warning. The live measurements
-> are in [Measured results](#measured-results), with their dates.
+> **What the canary badge means.** The daily canary scrapes three pages of
+> avito.ru through the Scraping Browser API on a Russian exit and is green only
+> when that run is complete and its rows pass the data checks. It needs the
+> secrets `TWOCAPTCHA_KEY` and `AVITO_PROXY` and the variable
+> `AVITO_BROWSER_ACCOUNT_ID`; without them it is **red**, naming what is
+> missing, rather than green without having scraped anything. A blocked run,
+> one that never reached the site, or zero rows is red too. The live
+> measurements are in [Measured results](#measured-results), with their dates.
 
 ---
 
@@ -252,7 +254,7 @@ you operate applies to that data — in Russia, Federal Law 152-FZ.
 |---|---|---|
 | `playwright_scraper.py` | yes — **recommended** | yes |
 | `puppeteer_scraper.py` | yes | yes |
-| `selenium_scraper.py` | **no** — chromedriver's `debuggerAddress` has nowhere to put the endpoint's password | yes |
+| `selenium_scraper.py` | **no** — chromedriver's `debuggerAddress` has nowhere to put the endpoint's password (a typed `--cdp-endpoint` exits 2; `AVITO_CDP_ENDPOINT` from `.env` is ignored with a notice) | yes |
 
 All three share one CLI and one run loop (`browser_bridge.py`), so they agree
 on exit codes and on when to wait, solve or rotate. Live-verified on
@@ -277,17 +279,27 @@ pyppeteer is effectively unmaintained; it is here for parity.
 |---|---|
 | `0` | Complete |
 | `1` | Crash — an unexpected exception: a bug, not the site and not the transport (traceback printed, credentials masked) |
-| `2` | Bad usage — wrong host, wrong `--mode` for the URL, a robots-disallowed URL |
+| `2` | Bad usage — wrong host, wrong `--mode` for the URL, a robots-disallowed URL, an unreadable `--from` file |
 | `3` | Blocked — the firewall stayed up after every rung |
-| `4` | Zero results — including a search with no matches |
-| `5` | Transport error — **our** plumbing: a navigation timeout, a dead exit, `profile_locked`, `proxy_error` |
+| `4` | Zero results — the listing was read to its end and is empty, including a search with no matches |
+| `5` | Content never obtained — **our** plumbing (a navigation timeout, a dead exit, `profile_locked`, `proxy_error`), or zero rows for any reason other than a listing read to its end: a failed page, an item page with no item on it, a seller profile that never parsed |
 | `6` | Partial — some pages fetched, then stopped |
 
-`3` and `5` mean different things: `5` says the site was never reached, so
-the answer is "run it again" or "rebuild the endpoint", not the firewall.
+A page buys at most one paid captcha solve per run
+(`page_flow.SOLVES_PER_PAGE`): a fresh exit re-fetches a page that stayed
+walled, but does not buy a second token for it.
+
+`3` and `5` mean different things: `5` says the content was never obtained,
+so the answer is "run it again", "rebuild the endpoint" or "check the URL",
+not the firewall. `4` is only for a run that read the listing to its end.
 
 A run that finds nothing writes nothing — last night's output is not replaced
-with `[]` (`--allow-empty` is the opt-out). `<out>.meta.json` describes the
+with `[]` (`--allow-empty` is the opt-out). Output files get the mode
+`open()` would give them under your umask, and an existing file keeps its
+own. In the CSV, a text cell starting with `=`, `+`, `-`, `@`, a tab or a
+line break is prefixed with `'` so a spreadsheet does not run it as a
+formula; the JSON keeps the site's text, and `csv_cells_escaped` in the
+sidecar counts the cells that differ. `<out>.meta.json` describes the
 data beside it; `<out>.latest_attempt.meta.json` is written on EVERY ending — success, block, transport error or crash, with `exit_code` and `error_class` — and
 records `transport_facts`: walls met, which rung cleared each, every solve
 with its time and cost, rotations, and auto-solve events.