One captcha solve per page, OPENSEA_URL never overrides --collection, atomic writes

.dockerignore

@@ -1,6 +1,7 @@
 .git/
 .github/
 .env
+.env*
 *.log
 *.key
 secrets*

.env.example

@@ -25,9 +25,9 @@
 TWOCAPTCHA_KEY=your_2captcha_api_key_here
 
 # The Scraping Browser API endpoint, for --cdp-endpoint. THE SHAPE, not a
-# working one: a profile's credentials live about a day, so a pasted endpoint
-# is stale by tomorrow and produces an auth error a long way from its cause.
-# Get a fresh one from the 2Captcha dashboard.
+# working one: a profile's credentials expire (how long they last varies), so
+# a pasted endpoint eventually produces a 401 a long way from its cause. A 401
+# means the endpoint or profile needs refreshing from the 2Captcha dashboard.
 #
 #   country-xx  picks the exit country
 #   pid-…       is a profile with persistent cookies, ONE live connection
@@ -42,8 +42,10 @@ OPENSEA_CDP_ENDPOINT=ws://{login}-zone-scraping_browser-country-us-pid-{profileI
 OPENSEA_PROXY=http://{user}:{password}@us.proxy.2captcha.com:2334
 
 # A default --url, so a bare `python3 playwright_scraper.py` has somewhere to
-# go. Deliberately NOT mapped onto --out or --sort: env_config only fills
-# values that are unset, so mapping a variable onto a flag with a non-empty
-# default would make it silently inert — a setting that looks configurable
-# and is not.
-OPENSEA_URL=https://opensea.io/collection/boredapeyachtclub
+# go. Commented out on purpose: uncomment it only if you want that default.
+# It never outranks --collection, --category or --mode collections, and a
+# typed --url always wins over it. Deliberately NOT mapped onto --out or
+# --sort: env_config only fills values that are unset, so mapping a variable
+# onto a flag with a non-empty default would make it silently inert — a
+# setting that looks configurable and is not.
+# OPENSEA_URL=https://opensea.io/collection/boredapeyachtclub

.github/ci_checks.py

@@ -77,7 +77,7 @@
     # pasted while debugging. An allowlist entry would have switched the
     # check off exactly there. A sibling repo's copy of this script FAILED ON
     # ITS OWN MAIN for the opposite reason: its fixtures were neither
-    # allowlisted nor written this way (§17).
+    # allowlisted nor written this way.
     "{login}", "{user}", "password}@", "***", "u:p@h",
     "login:password@host:port",     # the shape a refusal message prints
     "user:secret@",                 # the proxy-pool masking fixtures
@@ -115,7 +115,7 @@
 # into pip's vendored code and flags a 32-hex string in `_elffile.py` as
 # key-shaped. Correct about the string, wrong about the file — and it is the
 # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people
-# have to argue with is one they learn to suppress (CLAUDE.md §22).
+# have to argue with is one they learn to suppress.
 #
 # Deliberately NOT a narrowing of what gets scanned: an untracked file is
 # still read, because a key pasted into a scratch file beside the scripts is
@@ -147,12 +147,16 @@ def scanned_files():
         if not path.is_file() or path.suffix not in SCANNED_SUFFIXES:
             continue
         # Any virtualenv, however it is named. `.venv-pup` exists here
-        # because the three engines' pins are mutually unsatisfiable (§6) and
+        # because the three engines' pins are mutually unsatisfiable and
         # each one needs its own environment — and a vendored dependency's
         # test data is full of hex strings that are not keys.
+        # RELATIVE to the repository, never the absolute path: `path.parts`
+        # includes every ancestor, so a clone made under a directory called
+        # `live`, `captures` or `venv-work` skipped every file and printed
+        # "nothing credential-shaped" having read nothing.
         if any(part in {".git", "__pycache__", "captures", "live"}
                or part.startswith((".venv", "venv", "node_modules"))
-               for part in path.parts):
+               for part in path.relative_to(REPO).parts):
             continue
         if _is_in_virtualenv(path):
             continue
@@ -218,9 +222,9 @@ def sample_check():
         failed.append("sample_output.csv header differs from output_writer.Product")
 
     if not failed:
-        discounted = sum(1 for r in rows if r.get("original_price"))
+        priced = sum(1 for r in rows if r.get("price") is not None)
         print(f"ok       {len(rows)} rows, {len(expected)} columns, "
-              f"{discounted} discounted, schema matches")
+              f"{priced} priced, schema matches")
     return failed
 
 
@@ -244,6 +248,11 @@ def secret_check():
                 failed.append(f"{rel}:{lineno} contains {match[:6]}… — a "
                               f"32-char hex string, the shape of a 2captcha key")
 
+    if not scanned:
+        # A scan that read nothing has proved nothing, and must not say
+        # "nothing credential-shaped" as if it had.
+        failed.append("0 files scanned — the exclusions skipped the whole "
+                      "repository, so nothing was checked")
     if not failed:
         print(f"ok       {scanned} files scanned, nothing credential-shaped")
     return failed

.github/repo-metadata.yml

@@ -3,7 +3,7 @@
 # GitHub's description, topics and homepage are the first thing anyone reads,
 # and they are the one surface the offline suite could not see: it scans
 # FILES, and a description lives behind an API. That gap is not hypothetical.
-# A sibling repo's description shipped carrying the very wording §12 bans for
+# A sibling repo's description shipped carrying the very wording this repo bans for
 # the Scraping Browser API, copied out of the family template, and reached
 # two public repos before anyone noticed — because the suite scans repo files
 # and a GitHub description is not a file.

.github/workflows/canary.yml

@@ -6,9 +6,9 @@ name: canary
 #
 # THIS CANARY NEEDS NO SECRETS, AND THAT IS THE POINT
 # ---------------------------------------------------
-# §11 says a canary that cannot pass must SKIP rather than go red every day,
+# A canary that cannot pass must SKIP rather than go red every day,
 # because a check that is always red teaches everyone to ignore checks. The
-# other half of that rule is §21's: a canary that CAN pass without a
+# other half of that rule: a canary that CAN pass without a
 # credential should never be gated on one.
 #
 # This one can. Measured from one Hetzner datacentre address in Helsinki
@@ -29,17 +29,20 @@ name: canary
 #
 # It goes through the ENVIRONMENT rather than a commented-out --proxy line,
 # because env_config.py already reads exactly that variable, a credential
-# must never reach a command line (§3), and an unset secret arrives as an
+# must never reach a command line, and an unset secret arrives as an
 # empty string which the loader treats as "not configured". One line is
 # correct with and without it.
 #
 # WHAT FAILS THE BADGE AND WHAT ONLY WARNS
 # ----------------------------------------
-# A block (exit 3) WARNS, and says in the step summary that the site-side
-# check did not happen today, so a warning cannot read as a pass. Parsing
-# zero rows from a page the site plainly served (exit 4), a pagination walk
-# that stops short, or a coverage floor being missed DOES fail — those are
-# the regressions this exists to catch.
+# Everything that means the site was NOT tested today fails the job: a block
+# (exit 3), content never obtained (exit 5), zero rows (exit 4) in any mode,
+# and an unexpected exit. A green badge has to mean "OpenSea was read today
+# and the parser worked", and a warning on a green run reads as a pass. A
+# pagination walk that stops short or a row/field floor being missed fails
+# too — those are the regressions this exists to catch. The floors test the
+# PARSER (row counts a busy collection always fills, non-null core fields,
+# the shape of the walk), not market levels that can move overnight.
 
 on:
   schedule:
@@ -102,8 +105,8 @@ jobs:
           case "${{ steps.items.outputs.exit_code }}" in
             0) echo "::notice::Items walk complete." ;;
             3)
-              echo "::warning::Blocked before parsing (exit 3) from a bare GitHub runner. Nothing refused this scraper from a datacentre address while it was written, so this says something about the runner rather than about this code — but it also means the site-side check did NOT happen today."
-              echo "A runner address was refused. The parser was not exercised. Set an \`OPENSEA_PROXY\` secret to give this canary an exit whose request rate it controls." >> "$GITHUB_STEP_SUMMARY" ;;
+              echo "::error::Blocked before parsing (exit 3) from a GitHub runner, so the site was NOT tested today. Nothing refused this scraper from a datacentre address while it was written, so look at the dump in the artefacts; an OPENSEA_PROXY secret gives the canary another exit if runner addresses start being refused."
+              exit 1 ;;
             4)
               echo "::error::Ran fine and parsed 0 rows (exit 4) from a collection that holds ten thousand items. The inlined urql state or the GraphQL query has most likely moved — this is exactly the regression this canary exists to catch. The dump in the artefacts tells you which."
               exit 1 ;;
@@ -127,7 +130,7 @@ jobs:
           fail = []
 
           # THE PAGINATION CHECK. With one page a dead page walk is invisible,
-          # which is the family's most expensive bug (§7). Here the walk is
+          # which is the family's most expensive bug. Here the walk is
           # cursor-driven, so "did page 2 use page 1's cursor" is the thing
           # that has to keep working.
           if meta["pages_completed"] < 3:
@@ -182,12 +185,14 @@ jobs:
               fail.append("the walk found %d priced rows and OpenSea says "
                           "%d items are listed — the two are supposed to "
                           "agree" % (priced, listed))
-          if priced < 100:
-              fail.append("only %d priced rows; a collection with a live "
-                          "floor should carry far more" % priced)
+          # At least one priced row: the floor exists on any collection with
+          # a listing. How MANY are listed is market state, and the count is
+          # already checked against OpenSea's own listedItemCount above.
+          if priced < 1:
+              fail.append("no priced rows; the price walk read nothing")
 
           # A price is a TOKEN amount and the token is named. A defaulted
-          # currency, or an empty-string one, is the §4 failure — and so is a
+          # currency, or an empty-string one, is a guess presented as a fact — and so is a
           # currency on a row that has no price.
           if any(r.get("price") is not None and not r.get("currency")
                  for r in rows):
@@ -212,14 +217,14 @@ jobs:
                           "or its endpoint" % sorted(views))
 
           # `position` restarts at 1 on every page, so the column is
-          # worthless without `page` beside it (§18).
+          # worthless without `page` beside it.
           pairs = {(r.get("page"), r.get("position")) for r in rows}
           if len(pairs) != len(rows):
               fail.append("page+position is not unique across the run")
           if len({r["sku"] for r in rows}) != len(rows):
               fail.append("two rows share a sku")
 
-          # §21's arithmetic: the sidecar must say what the collection holds
+          # The sidecar must say what the collection holds
           # beside what the run read, or `complete` is lying by omission.
           totals = meta.get("collection_totals") or {}
           if not totals.get("total_supply"):
@@ -253,8 +258,8 @@ jobs:
           case "${{ steps.ranking.outputs.exit_code }}" in
             0) echo "::notice::Ranking run complete." ;;
             3)
-              echo "::warning::Blocked on the ranking run (exit 3)."
-              echo "The ranking run was refused; that mode was not exercised today." >> "$GITHUB_STEP_SUMMARY" ;;
+              echo "::error::Blocked on the ranking run (exit 3); that mode was not tested today."
+              exit 1 ;;
             4)
               echo "::error::0 rows from /collections, which renders fifty collections."
               exit 1 ;;
@@ -333,11 +338,11 @@ jobs:
           case "${{ steps.activity.outputs.exit_code }}" in
             0) echo "::notice::Activity run complete." ;;
             3)
-              echo "::warning::Blocked on the activity run (exit 3)."
-              echo "The activity run was refused; that mode was not exercised today." >> "$GITHUB_STEP_SUMMARY" ;;
+              echo "::error::Blocked on the activity run (exit 3); that mode was not tested today."
+              exit 1 ;;
             4)
-              echo "::warning::0 rows from a sales feed. A quiet collection can legitimately have no sales in its recent feed, so this warns rather than failing — but check the dump if it repeats."
-              echo "The activity feed came back empty. That is legitimate on a quiet collection and worth looking at if it happens two days running." >> "$GITHUB_STEP_SUMMARY" ;;
+              echo "::error::0 rows from the sales feed of Pudgy Penguins, which sells every day (300 sales over three pages on 2026-10-15). An empty feed here means the parser or the query broke, not a quiet market."
+              exit 1 ;;
             *)
               echo "::error::Unexpected exit ${{ steps.activity.outputs.exit_code }}."
               exit 1 ;;

.github/workflows/tests.yml

@@ -81,50 +81,10 @@ jobs:
           EOF
 
       - name: Committed sample output is real
-        # The README links to these two files, so a push without them ships
-        # broken links. And a sample invented by hand drifts from the schema and
-        # teaches readers the wrong field names, so these are cut from a real
-        # run rather than written by hand.
-        run: |
-          python - <<'EOF'
-          import csv, json, sys
-          from pathlib import Path
-
-          MARKERS = ("sample-product-", "example brand", "sample product",
-                     "product description text", "lorem ipsum")
-
-          fail = []
-          for name in ("sample_output.json", "sample_output.csv"):
-              if not Path(name).is_file():
-                  fail.append(f"{name} is missing — regenerate it from a real run")
-
-          if not fail:
-              rows = json.loads(Path("sample_output.json").read_text())
-              if not rows:
-                  fail.append("sample_output.json is empty")
-              blob = json.dumps(rows).lower()
-              hit = [m for m in MARKERS if m in blob]
-              if hit:
-                  fail.append(f"sample_output.json looks fabricated: {hit}")
-
-              # Column set must match what the writers actually produce.
-              from output_writer import Product
-              from dataclasses import asdict
-              expected = list(asdict(Product()).keys())
-              for i, row in enumerate(rows):
-                  if list(row.keys()) != expected:
-                      fail.append(f"row {i} columns drifted from output_writer.Product")
-                      break
-
-              with open("sample_output.csv", newline="", encoding="utf-8") as f:
-                  header = next(csv.reader(f))
-              if header != expected:
-                  fail.append("sample_output.csv header drifted from output_writer.Product")
-
-          for line in fail:
-              print("FAILED:", line)
-          sys.exit(1 if fail else 0)
-          EOF
+        # The README links to these two files, and a sample invented by hand
+        # drifts from the schema. One implementation, in ci_checks.py, rather
+        # than a second copy inlined here that imports the row class itself.
+        run: python3 .github/ci_checks.py --sample-check
 
       - name: Config reporter runs with no .env present
         run: python env_config.py

.gitignore

@@ -40,6 +40,24 @@ _*.py
 *_debug.html
 *_debug.png
 *.html.page*
+# --dump-html with any other name and --pages > 1 writes NAME.pageN.
+*.page[0-9]*
+
+# --- working directories, by KIND rather than by one remembered name -----
+# A raw capture reaches `git add -A` through whichever directory nobody had
+# listed, so every usual name for a scratch or output directory is here, plus
+# any HTML dropped at the repository root.
+/out/
+/output/
+/runs/
+/run/
+/results/
+/result/
+/tmp/
+/scratch/
+/dumps/
+/dump/
+/*.html
 
 # --- captures kept for parser work ---------------------------------------
 # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py);
@@ -89,7 +107,7 @@ secrets*
 # recoverable only because that repo was private at the time. On 2026-09-17
 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC
 # repository, one `git add -A` from publication. Craigslist was fixed then
-# and the fix never reached anyone else, which is §16's rule unapplied: when
+# and the fix never reached anyone else, which is the rule unapplied: when
 # you fix one, check the siblings.
 .env*
 !.env.example
@@ -98,6 +116,6 @@ secrets*
 live/
 
 # One virtualenv per engine — the three engines' pins are mutually
-# unsatisfiable (§6), so a reader following the README ends up with several.
+# unsatisfiable, so a reader following the README ends up with several.
 .venv-pup/
 .venv-*/