Keep good output on a failed write, survive blocks without a proxy, stop on a 404

.env.example

@@ -130,8 +130,10 @@ BBB_PROXY=http://{user}:{password}@na.proxy.2captcha.com:2334
 #                                        only one that is Cloudflare-gated.
 #
 # `--text` and `--location` build a search URL for you, so this is optional
-# even for a search.
-BBB_URL=https://www.bbb.org/search?find_country=USA&find_text=restaurants&find_loc=New+York%2C+NY
+# even for a search, and it is commented out so that a copied example does
+# not become a default nobody chose. A URL set here is a default only: query
+# flags typed on the command line replace it.
+# BBB_URL=https://www.bbb.org/search?find_country=USA&find_text=restaurants&find_loc=New+York%2C+NY
 
 # Deliberately NOT here: an output prefix. `--out` already carries a
 # non-empty default, so the loader — which only fills UNSET values — would

.github/ci_checks.py

@@ -81,14 +81,27 @@
 HEX32 = re.compile(r"\b[0-9a-f]{32}\b")
 # Contexts in which a 32-hex string is plainly not a key. Site-specific
 # entries belong here only once a committed fixture actually carries one —
-# CLAUDE.md §16: a family-wide entry copied without checking is how dead
-# code spreads. Kept as a CONTEXT allowlist rather than by loosening the
+# a family-wide entry copied without checking is how dead code spreads. Kept as a CONTEXT allowlist rather than by loosening the
 # pattern: a bare 32-hex string anywhere else still fails, which is the
 # point of the check.
 HEX32_ALLOWED = ("sha", "hash", "nonce", "example", "md5", "digest",
                  "checksum")
 
-SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example")
+SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example",
+                    ".json", ".csv", ".html", ".toml", ".cfg", ".ini", ".sh")
+
+# Directories never worth reading, matched against the path RELATIVE to the
+# repository root. Matching the absolute path skipped everything once the
+# clone itself lived under a directory with one of these names: a clone
+# under `~/venv/…` scanned 0 files and reported "nothing credential-shaped".
+SKIPPED_DIRS = {".git", "__pycache__", ".venv", "venv"}
+
+
+def _is_env_file(path):
+    """`.env`, `.env.bak`, `.env.local`, ... — the copies people make of the
+    one file that holds a key. Their suffix is anything at all, so they are
+    recognised by name rather than by suffix."""
+    return path.name == ".env" or path.name.startswith(".env.")
 
 
 # A directory holding `pyvenv.cfg` is a virtualenv, whatever it is called.
@@ -99,7 +112,7 @@
 # into pip's vendored code and flags a 32-hex string in `_elffile.py` as
 # key-shaped. Correct about the string, wrong about the file — and it is the
 # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people
-# have to argue with is one they learn to suppress (CLAUDE.md §22).
+# have to argue with is one they learn to suppress.
 #
 # Deliberately NOT a narrowing of what gets scanned: an untracked file is
 # still read, because a key pasted into a scratch file beside the scripts is
@@ -128,10 +141,12 @@ def _is_in_virtualenv(path):
 
 def scanned_files():
     for path in sorted(REPO.rglob("*")):
-        if not path.is_file() or path.suffix not in SCANNED_SUFFIXES:
+        if not path.is_file():
+            continue
+        if path.suffix not in SCANNED_SUFFIXES and not _is_env_file(path):
             continue
-        if any(part in {".git", "__pycache__", ".venv", "venv"}
-               for part in path.parts):
+        if any(part in SKIPPED_DIRS
+               for part in path.relative_to(REPO).parts[:-1]):
             continue
         if _is_in_virtualenv(path):
             continue
@@ -197,9 +212,8 @@ def sample_check():
         failed.append("sample_output.csv header differs from output_writer.Business")
 
     if not failed:
-        discounted = sum(1 for r in rows if r.get("original_price"))
         print(f"ok       {len(rows)} rows, {len(expected)} columns, "
-              f"{discounted} discounted, schema matches")
+              f"schema matches")
     return failed
 
 
@@ -223,6 +237,10 @@ def secret_check():
                 failed.append(f"{rel}:{lineno} contains {match[:6]}… — a "
                               f"32-char hex string, the shape of a 2captcha key")
 
+    if scanned == 0:
+        # A scan that read nothing has proved nothing, and must not say "ok".
+        failed.append("the credential scan read 0 files — the exclusions "
+                      "matched the whole repository")
     if not failed:
         print(f"ok       {scanned} files scanned, nothing credential-shaped")
     return failed

.github/workflows/canary.yml

@@ -12,9 +12,11 @@
 # exactly the case this proves.
 #
 # The PROFILE path is different — it fetches a Cloudflare-gated page — so that
-# half is guarded on a secret and SKIPS with a ::notice:: when the secret is
-# absent. A check that is always red teaches everyone to ignore checks
-# (CLAUDE.md §11).
+# half needs a Scraping Browser endpoint. Without the secret it FAILS with an
+# ::error:: naming the secret rather than going green: a skipped job did not
+# test the site, and a green badge over an untested path is a claim nobody
+# measured. Exit 3 (blocked), exit 5 (never reached the page) and zero rows
+# are failures too.
 #
 #     gh secret set BBB_CDP_ENDPOINT --repo 2scraper/bbb-scraper
 #
@@ -52,7 +54,7 @@ jobs:
 
       # >= 3 pages, because with one page pagination is never exercised —
       # which is exactly how a silent single-page bug survived in a sibling
-      # repo (§11).
+      # repo.
       - name: Run
         id: run
         run: |
@@ -87,6 +89,9 @@ jobs:
 
           meta = json.load(open("canary.meta.json"))
           rows = json.load(open("canary.json"))
+          if not rows:
+              print("::error::The listing run wrote zero rows.")
+              sys.exit(1)
           print(json.dumps(meta, indent=1))
 
           failures = []
@@ -173,20 +178,19 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Explain the skip
+      - name: Require the secret
         if: env.HAVE_CDP != 'true'
         run: |
-          echo "::notice::Skipping the profile run: no BBB_CDP_ENDPOINT secret is set."
+          echo "::error::Missing secret BBB_CDP_ENDPOINT — the profile canary cannot test the site."
           echo "A business profile page is Cloudflare-gated, and a GitHub runner is a"
-          echo "datacenter address, so this half cannot pass without a residential exit."
-          echo "The LISTING job above needs no credentials and is the real canary."
+          echo "datacenter address, so this job needs a Scraping Browser endpoint:"
           echo ""
           echo "    gh secret set BBB_CDP_ENDPOINT --repo 2scraper/bbb-scraper"
           {
-            echo "### Profile canary skipped"
-            echo "No \`BBB_CDP_ENDPOINT\` secret. The listing canary covers the parser;"
-            echo "this job covers the gated page only."
+            echo "### Profile canary did not run"
+            echo "Missing secret \`BBB_CDP_ENDPOINT\`. Nothing about the gated page was tested."
           } >> "$GITHUB_STEP_SUMMARY"
+          exit 1
 
       - uses: actions/setup-python@v5
         if: env.HAVE_CDP == 'true'
@@ -207,7 +211,7 @@ jobs:
           # Through the ENVIRONMENT, not a --flag: env_config.py already reads
           # exactly this name, an unset secret arrives as an empty string
           # which the loader treats as unset, and the credential never reaches
-          # a command line (§3, §11).
+          # a command line.
           BBB_CDP_ENDPOINT: ${{ secrets.BBB_CDP_ENDPOINT }}
         run: |
           set +e
@@ -224,22 +228,26 @@ jobs:
           python - <<'PY'
           import json, os, sys
           code = int(os.environ["EXIT_CODE"])
+          # Every non-zero exit is red: a run that was blocked or never
+          # reached the page tested nothing, and a green job would say it had.
           if code == 3:
-              # Access, not a regression — and not a failure of this repo.
-              print("::warning::Blocked (exit 3). The Scraping Browser profile "
-                    "was refused or has expired. Nothing was tested; this is "
-                    "access rather than a parser regression.")
-              sys.exit(0)
+              print("::error::Blocked (exit 3). The page was refused through "
+                    "the Scraping Browser endpoint; nothing was tested.")
+              sys.exit(1)
           if code == 5:
-              print("::warning::Remote API error (exit 5). If the log says "
-                    "'profile_locked', another run still holds this pid — give "
-                    "CI its own. Otherwise the secret has expired.")
-              sys.exit(0)
+              print("::error::Nothing was obtained (exit 5). Read the log for "
+                    "the endpoint's own answer: 'profile_locked' means another "
+                    "run holds this pid; a 401 means the credentials were "
+                    "refused.")
+              sys.exit(1)
           if code != 0:
               print(f"::error::The profile run exited {code}.")
               sys.exit(1)
 
           rows = json.load(open("canary_profile.json"))
+          if not rows:
+              print("::error::The profile run wrote zero rows.")
+              sys.exit(1)
           row = rows[0]
           failures = []
           if len(rows) != 1:

.gitignore

@@ -24,6 +24,24 @@ live_results.zip
 *_debug.html
 *_debug.png
 *.html.page*
+# `--dump-html X --pages 3` writes X, X.page2, X.page3 whatever X is called,
+# and `*.json` does not cover `canary_dump.json.page3`.
+*.page[0-9]*
+# A raw page saved at the top of the repo while debugging. The landing page
+# is the one HTML file that is committed.
+/*.html
+!/landing.html
+# Working directories for live runs, by KIND rather than by one name: a
+# 1.5 MB dump reached `git add -A` in a sibling through `live/`, which no
+# pattern listed.
+live/
+out/
+output/
+runs/
+results/
+tmp/
+scratch/
+dumps/
 
 # --- captures kept for parser work ---------------------------------------
 # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py);
@@ -72,8 +90,6 @@ secrets*
 # a `.env.hold` made while testing that was committed WITH A LIVE KEY in it,
 # recoverable only because that repo was private at the time. On 2026-09-17
 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC
-# repository, one `git add -A` from publication. Craigslist was fixed then
-# and the fix never reached anyone else, which is §16's rule unapplied: when
-# you fix one, check the siblings.
+# repository, one `git add -A` from publication.
 .env*
 !.env.example

CHANGELOG.md

@@ -24,8 +24,59 @@ When it does, the release notes lead with it.
 > in `pages_failed` with `stop_reason: "unreadable_response"`. A pipeline
 > that branched on exit 0 was reading an unread page as the end of a listing.
 
+> **Behaviour change:** a URL BBB answers with its own 404 page now stops at
+> once with `stop_reason: "not_found"` (exit 5 when nothing was gathered).
+> It used to wait 30 s, re-fetch, and end as `unreadable_response`.
+
+> **Behaviour change:** negative values for `--pages`, `--delay`,
+> `--retry-delay`, `--proxy-block-retries` and `--concurrency` (and 0 for
+> `--pages` and `--concurrency`) are now usage errors (exit 2) in every
+> engine, and so is a negative `--retries`/`--retry-delay` in
+> `scraper_api_client.py`.
+
+> **Behaviour change:** CSV cells that begin with `=`, `+`, `-`, `@`, a tab
+> or a line break are now prefixed with an apostrophe so a spreadsheet does
+> not run them as formulas. The JSON output is unchanged; the sidecar says
+> how many cells differ in `csv_cells_escaped`. Output files are now written
+> through a temporary file and renamed, keeping the mode `open()` would give
+> a new file and the existing mode of a file being replaced.
+
 ### Fixed
 
+- **The Selenium and pyppeteer engines crashed on a block with no proxy.**
+  With no `--proxy`, a blocked page (and, in pyppeteer, a dead exit) reached
+  `pool.current`/`pool.advance` on None: a traceback, exit 1, and every row
+  from earlier pages lost. Both now re-fetch on the same path, as the
+  Playwright engine always did, and report exit 3.
+- **An interrupted write destroyed the previous good output.** `out.json`,
+  `out.csv` and `out.meta.json` were truncated before being written, so a
+  failure halfway left a short or empty file. They are now written to a
+  temporary file beside the target, synced and renamed over it.
+- **The HTTP status never reached the classifier.** Playwright and
+  pyppeteer now pass the response status, so BBB's 404 is terminal and a
+  marker-less 403 is a block. Selenium has no status to pass; BBB's own 404
+  title is recognised in the body for it.
+- **A `BBB_URL` from `.env` blocked `--text`/`--location`.** A URL from the
+  environment is now a default that the query flags replace; a TYPED
+  `--url` with query flags is still refused. `.env.example` ships `BBB_URL`
+  commented out.
+- **The credential scan judged exclusions on the absolute path**, so a
+  clone under a directory named `venv` scanned 0 files and reported clean.
+  Exclusions are now relative to the repository, a scan of 0 files fails,
+  and `.env*` copies plus `.json`, `.csv` and `.html` files are read too.
+- **Dump and capture files were not ignored**: `X.page3` from
+  `--dump-html`, a raw `.html` at the repository root, and `live/`,
+  `out/`-style working directories.
+- **The profile canary went green without testing anything.** With no
+  `BBB_CDP_ENDPOINT` secret it skipped with a notice, and with one it
+  passed on exit 3 or 5. It now fails with an error naming the missing
+  secret, and every non-zero exit or empty result is red. The listing
+  canary needs no secret and stays scheduled; it now also fails explicitly
+  on zero rows.
+- `--solve-captcha` help, a docstring and the README's exit-code table
+  described refusals and exit 5 inaccurately; corrected. Comments no longer
+  point at documents a reader of this repository cannot open.
+
 - **`--retries 0` fetched nothing and called it a result.** The load loop is
   `range(1, retries + 1)`, so 0 never runs it and the engine judged
   `about:blank`. Three third-party audits of this family ran exactly that and

README.md

@@ -240,7 +240,7 @@ Nothing here integrates a competitor.
 | 2 | bad usage |
 | 3 | blocked — Cloudflare, distinct from an empty result |
 | 4 | zero businesses — the query matched nothing |
-| 5 | remote API error (the Scraping Browser or Scraper API) |
+| 5 | nothing was obtained — a remote API error (the Scraping Browser or Scraper API), a navigation timeout, a dead proxy, a page that could not be read, or a URL BBB answers with its own 404 |
 | 6 | partial — some pages came back and some did not |
 
 **A run that finds nothing writes nothing**, so a failure never replaces last

captcha_solver.py

@@ -732,4 +732,4 @@ class CaptchaUnsolvable(RuntimeError):
 # what the address has been doing — a narrow list is how a challenge gets
 # reported as an empty page months later. A solver for a challenge this site
 # has never been observed to serve is dead code; a DETECTOR for one is cheap
-# insurance. See the family note in CLAUDE.md.
+# insurance.

diff_runs.py

@@ -65,7 +65,7 @@
 # What is worth watching on a business directory, and nothing else.
 #
 # A price monitor's fields are absent because BBB has no prices — porting
-# them would be dead code that looks load-bearing (CLAUDE.md §4). What
+# them would be dead code that looks load-bearing. What
 # changes here is a business's STANDING and its CONTACT DETAILS, and those
 # are the two things anyone diffs a directory for.
 #
@@ -96,7 +96,7 @@
 # against a profile run would report each of them as a change on every row —
 # and none of it would be about the business. When the two rows disagree on
 # `data_source`, those fields are reported separately as `source_changed`
-# rather than as changes, which is this family's rule (§8: a difference that
+# rather than as changes, which is this family's rule (a difference that
 # comes with a provenance difference says something about our own two
 # snapshots, not about the site).
 PROFILE_ONLY_FIELDS = (

env_config.py

@@ -66,8 +66,8 @@
 # A literal-only check reported both of those as CONFIGURED, so `cp
 # .env.example .env` followed by a run connected to cb.2captcha.com with the
 # string `{login}-zone-…` as its username and got a 401 — a confusing failure
-# a long way from its cause, which is precisely what §3's rule exists to
-# prevent. Any `{…}` left in a value means the line was never filled in.
+# a long way from its cause, which is precisely what keeping
+# credentials in `.env` was meant to prevent. Any `{…}` left in a value means the line was never filled in.
 _BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}")
 
 _loaded_from = None

fingerprint_client.py

@@ -331,7 +331,8 @@ def main() -> int:
     # loader. It used to read `os.environ` alone, which meant a key put in
     # `.env` exactly as the README and .env.example instruct worked for every
     # engine and failed HERE with "No API key" — a documented mechanism not
-    # applied on one path, which is the shape of half the defects §16 lists.
+    # applied on one path, which is the shape of many defects copied
+    # code has carried in this family.
     #
     # `.env` has to be LOADED before it can be read: `env_value` looks at
     # os.environ, and `load_env` is what fills that from the file. Calling it