diff --git a/.dockerignore b/.dockerignore index c4f1d30..dfc48d4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ .git/ .github/ .env +.env* *.log *.key secrets* diff --git a/.env.example b/.env.example index 2331071..be4e4a6 100644 --- a/.env.example +++ b/.env.example @@ -25,9 +25,9 @@ TWOCAPTCHA_KEY=your_2captcha_api_key_here # The Scraping Browser API endpoint, for --cdp-endpoint. THE SHAPE, not a -# working one: a profile's credentials live about a day, so a pasted endpoint -# is stale by tomorrow and produces an auth error a long way from its cause. -# Get a fresh one from the 2Captcha dashboard. +# working one: a profile's credentials expire (how long they last varies), so +# a pasted endpoint eventually produces a 401 a long way from its cause. A 401 +# means the endpoint or profile needs refreshing from the 2Captcha dashboard. # # country-xx picks the exit country # pid-… is a profile with persistent cookies, ONE live connection @@ -42,8 +42,10 @@ OPENSEA_CDP_ENDPOINT=ws://{login}-zone-scraping_browser-country-us-pid-{profileI OPENSEA_PROXY=http://{user}:{password}@us.proxy.2captcha.com:2334 # A default --url, so a bare `python3 playwright_scraper.py` has somewhere to -# go. Deliberately NOT mapped onto --out or --sort: env_config only fills -# values that are unset, so mapping a variable onto a flag with a non-empty -# default would make it silently inert — a setting that looks configurable -# and is not. -OPENSEA_URL=https://opensea.io/collection/boredapeyachtclub +# go. Commented out on purpose: uncomment it only if you want that default. +# It never outranks --collection, --category or --mode collections, and a +# typed --url always wins over it. Deliberately NOT mapped onto --out or +# --sort: env_config only fills values that are unset, so mapping a variable +# onto a flag with a non-empty default would make it silently inert — a +# setting that looks configurable and is not. +# OPENSEA_URL=https://opensea.io/collection/boredapeyachtclub diff --git a/.github/ci_checks.py b/.github/ci_checks.py index b353b30..0faa674 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -77,7 +77,7 @@ # pasted while debugging. An allowlist entry would have switched the # check off exactly there. A sibling repo's copy of this script FAILED ON # ITS OWN MAIN for the opposite reason: its fixtures were neither - # allowlisted nor written this way (§17). + # allowlisted nor written this way. "{login}", "{user}", "password}@", "***", "u:p@h", "login:password@host:port", # the shape a refusal message prints "user:secret@", # the proxy-pool masking fixtures @@ -115,7 +115,7 @@ # into pip's vendored code and flags a 32-hex string in `_elffile.py` as # key-shaped. Correct about the string, wrong about the file — and it is the # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people -# have to argue with is one they learn to suppress (CLAUDE.md §22). +# have to argue with is one they learn to suppress. # # Deliberately NOT a narrowing of what gets scanned: an untracked file is # still read, because a key pasted into a scratch file beside the scripts is @@ -147,12 +147,16 @@ def scanned_files(): if not path.is_file() or path.suffix not in SCANNED_SUFFIXES: continue # Any virtualenv, however it is named. `.venv-pup` exists here - # because the three engines' pins are mutually unsatisfiable (§6) and + # because the three engines' pins are mutually unsatisfiable and # each one needs its own environment — and a vendored dependency's # test data is full of hex strings that are not keys. + # RELATIVE to the repository, never the absolute path: `path.parts` + # includes every ancestor, so a clone made under a directory called + # `live`, `captures` or `venv-work` skipped every file and printed + # "nothing credential-shaped" having read nothing. if any(part in {".git", "__pycache__", "captures", "live"} or part.startswith((".venv", "venv", "node_modules")) - for part in path.parts): + for part in path.relative_to(REPO).parts): continue if _is_in_virtualenv(path): continue @@ -218,9 +222,9 @@ def sample_check(): failed.append("sample_output.csv header differs from output_writer.Product") if not failed: - discounted = sum(1 for r in rows if r.get("original_price")) + priced = sum(1 for r in rows if r.get("price") is not None) print(f"ok {len(rows)} rows, {len(expected)} columns, " - f"{discounted} discounted, schema matches") + f"{priced} priced, schema matches") return failed @@ -244,6 +248,11 @@ def secret_check(): failed.append(f"{rel}:{lineno} contains {match[:6]}… — a " f"32-char hex string, the shape of a 2captcha key") + if not scanned: + # A scan that read nothing has proved nothing, and must not say + # "nothing credential-shaped" as if it had. + failed.append("0 files scanned — the exclusions skipped the whole " + "repository, so nothing was checked") if not failed: print(f"ok {scanned} files scanned, nothing credential-shaped") return failed diff --git a/.github/repo-metadata.yml b/.github/repo-metadata.yml index 8a24e1a..5acd256 100644 --- a/.github/repo-metadata.yml +++ b/.github/repo-metadata.yml @@ -3,7 +3,7 @@ # GitHub's description, topics and homepage are the first thing anyone reads, # and they are the one surface the offline suite could not see: it scans # FILES, and a description lives behind an API. That gap is not hypothetical. -# A sibling repo's description shipped carrying the very wording §12 bans for +# A sibling repo's description shipped carrying the very wording this repo bans for # the Scraping Browser API, copied out of the family template, and reached # two public repos before anyone noticed — because the suite scans repo files # and a GitHub description is not a file. diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index e1468dc..20f1f32 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -6,9 +6,9 @@ name: canary # # THIS CANARY NEEDS NO SECRETS, AND THAT IS THE POINT # --------------------------------------------------- -# §11 says a canary that cannot pass must SKIP rather than go red every day, +# A canary that cannot pass must SKIP rather than go red every day, # because a check that is always red teaches everyone to ignore checks. The -# other half of that rule is §21's: a canary that CAN pass without a +# other half of that rule: a canary that CAN pass without a # credential should never be gated on one. # # This one can. Measured from one Hetzner datacentre address in Helsinki @@ -29,17 +29,20 @@ name: canary # # It goes through the ENVIRONMENT rather than a commented-out --proxy line, # because env_config.py already reads exactly that variable, a credential -# must never reach a command line (§3), and an unset secret arrives as an +# must never reach a command line, and an unset secret arrives as an # empty string which the loader treats as "not configured". One line is # correct with and without it. # # WHAT FAILS THE BADGE AND WHAT ONLY WARNS # ---------------------------------------- -# A block (exit 3) WARNS, and says in the step summary that the site-side -# check did not happen today, so a warning cannot read as a pass. Parsing -# zero rows from a page the site plainly served (exit 4), a pagination walk -# that stops short, or a coverage floor being missed DOES fail — those are -# the regressions this exists to catch. +# Everything that means the site was NOT tested today fails the job: a block +# (exit 3), content never obtained (exit 5), zero rows (exit 4) in any mode, +# and an unexpected exit. A green badge has to mean "OpenSea was read today +# and the parser worked", and a warning on a green run reads as a pass. A +# pagination walk that stops short or a row/field floor being missed fails +# too — those are the regressions this exists to catch. The floors test the +# PARSER (row counts a busy collection always fills, non-null core fields, +# the shape of the walk), not market levels that can move overnight. on: schedule: @@ -102,8 +105,8 @@ jobs: case "${{ steps.items.outputs.exit_code }}" in 0) echo "::notice::Items walk complete." ;; 3) - echo "::warning::Blocked before parsing (exit 3) from a bare GitHub runner. Nothing refused this scraper from a datacentre address while it was written, so this says something about the runner rather than about this code — but it also means the site-side check did NOT happen today." - echo "A runner address was refused. The parser was not exercised. Set an \`OPENSEA_PROXY\` secret to give this canary an exit whose request rate it controls." >> "$GITHUB_STEP_SUMMARY" ;; + echo "::error::Blocked before parsing (exit 3) from a GitHub runner, so the site was NOT tested today. Nothing refused this scraper from a datacentre address while it was written, so look at the dump in the artefacts; an OPENSEA_PROXY secret gives the canary another exit if runner addresses start being refused." + exit 1 ;; 4) echo "::error::Ran fine and parsed 0 rows (exit 4) from a collection that holds ten thousand items. The inlined urql state or the GraphQL query has most likely moved — this is exactly the regression this canary exists to catch. The dump in the artefacts tells you which." exit 1 ;; @@ -127,7 +130,7 @@ jobs: fail = [] # THE PAGINATION CHECK. With one page a dead page walk is invisible, - # which is the family's most expensive bug (§7). Here the walk is + # which is the family's most expensive bug. Here the walk is # cursor-driven, so "did page 2 use page 1's cursor" is the thing # that has to keep working. if meta["pages_completed"] < 3: @@ -182,12 +185,14 @@ jobs: fail.append("the walk found %d priced rows and OpenSea says " "%d items are listed — the two are supposed to " "agree" % (priced, listed)) - if priced < 100: - fail.append("only %d priced rows; a collection with a live " - "floor should carry far more" % priced) + # At least one priced row: the floor exists on any collection with + # a listing. How MANY are listed is market state, and the count is + # already checked against OpenSea's own listedItemCount above. + if priced < 1: + fail.append("no priced rows; the price walk read nothing") # A price is a TOKEN amount and the token is named. A defaulted - # currency, or an empty-string one, is the §4 failure — and so is a + # currency, or an empty-string one, is a guess presented as a fact — and so is a # currency on a row that has no price. if any(r.get("price") is not None and not r.get("currency") for r in rows): @@ -212,14 +217,14 @@ jobs: "or its endpoint" % sorted(views)) # `position` restarts at 1 on every page, so the column is - # worthless without `page` beside it (§18). + # worthless without `page` beside it. pairs = {(r.get("page"), r.get("position")) for r in rows} if len(pairs) != len(rows): fail.append("page+position is not unique across the run") if len({r["sku"] for r in rows}) != len(rows): fail.append("two rows share a sku") - # §21's arithmetic: the sidecar must say what the collection holds + # The sidecar must say what the collection holds # beside what the run read, or `complete` is lying by omission. totals = meta.get("collection_totals") or {} if not totals.get("total_supply"): @@ -253,8 +258,8 @@ jobs: case "${{ steps.ranking.outputs.exit_code }}" in 0) echo "::notice::Ranking run complete." ;; 3) - echo "::warning::Blocked on the ranking run (exit 3)." - echo "The ranking run was refused; that mode was not exercised today." >> "$GITHUB_STEP_SUMMARY" ;; + echo "::error::Blocked on the ranking run (exit 3); that mode was not tested today." + exit 1 ;; 4) echo "::error::0 rows from /collections, which renders fifty collections." exit 1 ;; @@ -333,11 +338,11 @@ jobs: case "${{ steps.activity.outputs.exit_code }}" in 0) echo "::notice::Activity run complete." ;; 3) - echo "::warning::Blocked on the activity run (exit 3)." - echo "The activity run was refused; that mode was not exercised today." >> "$GITHUB_STEP_SUMMARY" ;; + echo "::error::Blocked on the activity run (exit 3); that mode was not tested today." + exit 1 ;; 4) - echo "::warning::0 rows from a sales feed. A quiet collection can legitimately have no sales in its recent feed, so this warns rather than failing — but check the dump if it repeats." - echo "The activity feed came back empty. That is legitimate on a quiet collection and worth looking at if it happens two days running." >> "$GITHUB_STEP_SUMMARY" ;; + echo "::error::0 rows from the sales feed of Pudgy Penguins, which sells every day (300 sales over three pages on 2026-10-15). An empty feed here means the parser or the query broke, not a quiet market." + exit 1 ;; *) echo "::error::Unexpected exit ${{ steps.activity.outputs.exit_code }}." exit 1 ;; diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 62d0ada..bb3a181 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -81,50 +81,10 @@ jobs: EOF - name: Committed sample output is real - # The README links to these two files, so a push without them ships - # broken links. And a sample invented by hand drifts from the schema and - # teaches readers the wrong field names, so these are cut from a real - # run rather than written by hand. - run: | - python - <<'EOF' - import csv, json, sys - from pathlib import Path - - MARKERS = ("sample-product-", "example brand", "sample product", - "product description text", "lorem ipsum") - - fail = [] - for name in ("sample_output.json", "sample_output.csv"): - if not Path(name).is_file(): - fail.append(f"{name} is missing — regenerate it from a real run") - - if not fail: - rows = json.loads(Path("sample_output.json").read_text()) - if not rows: - fail.append("sample_output.json is empty") - blob = json.dumps(rows).lower() - hit = [m for m in MARKERS if m in blob] - if hit: - fail.append(f"sample_output.json looks fabricated: {hit}") - - # Column set must match what the writers actually produce. - from output_writer import Product - from dataclasses import asdict - expected = list(asdict(Product()).keys()) - for i, row in enumerate(rows): - if list(row.keys()) != expected: - fail.append(f"row {i} columns drifted from output_writer.Product") - break - - with open("sample_output.csv", newline="", encoding="utf-8") as f: - header = next(csv.reader(f)) - if header != expected: - fail.append("sample_output.csv header drifted from output_writer.Product") - - for line in fail: - print("FAILED:", line) - sys.exit(1 if fail else 0) - EOF + # The README links to these two files, and a sample invented by hand + # drifts from the schema. One implementation, in ci_checks.py, rather + # than a second copy inlined here that imports the row class itself. + run: python3 .github/ci_checks.py --sample-check - name: Config reporter runs with no .env present run: python env_config.py diff --git a/.gitignore b/.gitignore index c2024d0..cabddb7 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,24 @@ _*.py *_debug.html *_debug.png *.html.page* +# --dump-html with any other name and --pages > 1 writes NAME.pageN. +*.page[0-9]* + +# --- working directories, by KIND rather than by one remembered name ----- +# A raw capture reaches `git add -A` through whichever directory nobody had +# listed, so every usual name for a scratch or output directory is here, plus +# any HTML dropped at the repository root. +/out/ +/output/ +/runs/ +/run/ +/results/ +/result/ +/tmp/ +/scratch/ +/dumps/ +/dump/ +/*.html # --- captures kept for parser work --------------------------------------- # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py); @@ -89,7 +107,7 @@ secrets* # recoverable only because that repo was private at the time. On 2026-09-17 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC # repository, one `git add -A` from publication. Craigslist was fixed then -# and the fix never reached anyone else, which is §16's rule unapplied: when +# and the fix never reached anyone else, which is the rule unapplied: when # you fix one, check the siblings. .env* !.env.example @@ -98,6 +116,6 @@ secrets* live/ # One virtualenv per engine — the three engines' pins are mutually -# unsatisfiable (§6), so a reader following the README ends up with several. +# unsatisfiable, so a reader following the README ends up with several. .venv-pup/ .venv-*/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2242f47..489d49e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,19 @@ was written, because anyone can read it with `git show v0.1.0:CHANGELOG.md`. ## [Unreleased] +> **Behaviour changes in this batch, read before upgrading a script:** +> `OPENSEA_URL` no longer outranks `--collection`, `--category` or +> `--mode collections` (it used to make `--collection pudgypenguins` scrape +> the environment's collection instead), and `.env.example` now ships it +> commented out. `--pages 0`, `--limit 0`, `--concurrency 0` and negative +> `--delay`, `--retry-delay` and `--proxy-block-retries` are usage errors +> (exit 2) instead of being accepted; `scraper_api_client.py --retries -1` +> too. A typed `--url` that names a different collection from +> `--collection` is refused. CSV cells that begin with `=`, `+`, `-`, `@`, a +> tab or a line break now get a leading apostrophe (CSV only; the JSON is +> unchanged) and the sidecar counts them in `csv_cells_escaped`. The +> canary now fails, rather than warning, when OpenSea was not actually read. + > **If you script this scraper: `--retries 0` is now a usage error (exit 2).** > It used to be accepted and to mean "never fetch the page" — the load loop > is `range(1, retries + 1)` — so a run passing it classified the empty @@ -23,6 +36,47 @@ was written, because anyone can read it with `git show v0.1.0:CHANGELOG.md`. ### Fixed +- **A page that kept coming back as a challenge could buy up to five + captcha solves.** Each engine checks for a challenge before and after + classifying the page, and re-fetches up to three more times; only the + second check was counted, so the per-page limit of one did not hold + (5 solver calls against a limit of 1, reproduced with the solver stubbed, + in all three engines). Both checks now draw on one per-page allowance, + spent immediately before the purchase and never reset by a retry or a + rotation. A suite check drives each engine with its driver stubbed. +- **`OPENSEA_URL` silently replaced `--collection`.** With the URL + `.env.example` used to ship, `--collection pudgypenguins` scraped Bored + Apes and `--mode collections` was refused as a mismatch. What was typed is + now captured before the environment is applied. +- **Output files are written atomically.** `out.json`, `out.csv` and + `out.meta.json` are written to a temporary file beside the target and + renamed over it, so a crash, a full disk or a Ctrl-C mid-write leaves the + previous good file in place (before: an error injected inside `json.dump` + left 12 bytes of invalid JSON). A new file gets the mode `open()` would + have given it; an existing file keeps its own. +- **`diff_runs.py` compared item columns in every mode**, so a collections + run whose volume went from 100 to 5,000, or an activity event whose type + changed, reported 0 changed. The compared columns are now per mode, the + mode comes from the sidecar (or the rows), and its help and docstring + describe OpenSea rather than another site. +- **The credential scan read nothing when the clone sat under a directory + called `live`, `captures` or `venv-…`**, because its exclusions were + matched against the absolute path, and still printed "nothing + credential-shaped". Exclusions are now relative to the repository, and a + scan that read 0 files fails. +- Numeric flags reject values below their floor with an error naming the + flag (see the blockquote above). +- `.gitignore` covers `--dump-html NAME --pages N` dumps (`NAME.pageN`), the + usual scratch and output directory names and a root-level `*.html`; + `.dockerignore` excludes every `.env*`. +- The docs no longer say Scraping Browser credentials "live about a day": + how long they last varies, and a 401 is the sign to refresh them. +- Shipped files no longer cite section numbers of an internal document; the + rules are stated in this repository's own words, and a suite check keeps + it that way. +- CI's sample-output step calls `.github/ci_checks.py --sample-check` instead + of a second inline copy of the same check. + - **Two of the three modes had silently degraded, and the run still said `complete`.** Between 2026-09-17 and 2026-10-13 OpenSea stopped inlining its state on the ranking and on the activity feed; both now render @@ -76,7 +130,7 @@ was written, because anyone can read it with `git show v0.1.0:CHANGELOG.md`. rows deep, and pages 4 to 10 carried zero priced items. OpenSea fixed their end. The behaviour is unchanged, because a floor scrape should still stop where the prices stop, but it is now keyed on a page with no priced - row (§7: a terminating condition is a property of the catalogue, not of + row (a terminating condition is a property of the catalogue, not of the transport). Every copy of the old claim in the code and the docs has been corrected rather than quietly dropped. @@ -119,7 +173,7 @@ was written, because anyone can read it with `git show v0.1.0:CHANGELOG.md`. with a plain client. - **The proxy path, run for the first time**, which was the last credentialled - path §16 asks for and the one that had never been exercised. Through a + path left to verify and the one that had never been exercised. Through a 2Captcha residential exit (the global mix, which came out in RU rather than the EU its hostname suggests): Playwright 250 rows over 3 pages with one timeout retried and recovered; pyppeteer 50 rows through @@ -132,8 +186,8 @@ was written, because anyone can read it with `git show v0.1.0:CHANGELOG.md`. The check worth having a real credential for: **the password appears in no process command line**, verified with `ps` mid-run. The browser is given `--proxy-server=http://eu.proxy.2captcha.com:2334` — host and port only. - That is §3's central promise, checked against a live secret instead of a - fixture. + That is the central promise (a secret never reaches argv), checked + against a live secret instead of a fixture. The data did not move: 250 of 250 rows identical to the Finnish datacentre and US Scraping Browser runs, on 12 stable columns each. Four unrelated diff --git a/README.md b/README.md index 3fab830..dbcfbaa 100644 --- a/README.md +++ b/README.md @@ -320,7 +320,7 @@ came out in **RU** (`AS24955`) rather than the EU its hostname suggests: | pool rotation | a dead exit first, the live one second: `ERR_PROXY_CONNECTION_FAILED` recognised as a proxy fault rather than a timeout, rotated, fresh browser, 150 rows, exit 0 | | **the password in any process command line** | **absent** — checked with `ps` mid-run against a real secret; the browser was given `--proxy-server=http://eu.proxy.2captcha.com:2334`, host and port only | -That last row is the one worth having a real credential for. §3 promises a +That last row is the one worth having a real credential for. This repo promises a secret never reaches `argv`, where anything that can run `ps` could read it; Playwright is handed the credentials through its own proxy fields instead. It is now checked with a live password rather than a fixture. diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 58fce73..53b881f 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -214,9 +214,9 @@ one. Without that rule the run would connect to `cb.2captcha.com` with the literal string `{login}-zone-…` as its username and get a 401 a long way from its cause. -Scraping Browser profile credentials live about a day, so a `ws://` endpoint -you pasted yesterday is stale today — get a fresh one from the dashboard -rather than debugging the old one. +Scraping Browser profile credentials expire, and how long they last varies. +A 401 on connect means the endpoint or its profile needs refreshing — get a +fresh one from the dashboard rather than debugging the old one. --- diff --git a/captcha_solver.py b/captcha_solver.py index b0b9d8d..2ee8135 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -25,7 +25,7 @@ and is not a credential. It appears NOWHERE on the site today: 0 occurrences across twelve captures, the homepage, and the `/login` and `/signup` routes (both of which 404 — signing in is a modal). So it was either stale or never -measured, and it is not evidence that a challenge is configured (§13). +measured, and it is not evidence that a challenge is configured. What the absence means is "no challenge was met", NOT "no challenge can be". The site sits behind Cloudflare — `server: cloudflare` and a `cf-ray` header on @@ -33,7 +33,7 @@ which 2Captcha solves with `TurnstileTaskProxyless`. The sentence this family is never entitled to write is that a captcha here cannot be solved; the one it is entitled to is "this repo implements X", which is what the -list below is (§19). +list below is. IMPLEMENTED HERE ---------------- @@ -829,8 +829,8 @@ def solve_recaptcha(challenge: CaptchaChallenge, twocaptcha_api_key: Optional[st api_version: str = "v2", min_score: float = 0.7) -> str: """Public entry point: solve `challenge` through 2captcha and return the token. - There used to be a `use_antidetect` branch here, behind a CLI flag of the - same name, that POSTed to a hardcoded local "solve-captcha" endpoint. + There used to be a second branch here, behind a CLI flag of its own, + that POSTed to a hardcoded local "solve-captcha" endpoint. It was removed before publishing: that endpoint was a placeholder for a product that does not exist under that name, so the flag could not work for anyone who set it, and the real "the browser solves it for you" path is diff --git a/diff_runs.py b/diff_runs.py index cc8297d..870c5e8 100644 --- a/diff_runs.py +++ b/diff_runs.py @@ -3,40 +3,45 @@ diff_runs.py ------------- Compares two output files from this project (JSON, as written by -output_writer.save) and reports what changed between them, keyed on `sku` — -the identifier the README tells people to diff on for tracking an answer's -reception over time. +output_writer.save) and reports what changed between them, keyed on `sku`: +an NFT's chain/contract/token id in an items run, a collection's slug in a +collections run, an event's id in an activity run. - python3 diff_runs.py --old ml.2026-09-01.json \\ - --new ml.2026-09-07.json + python3 diff_runs.py --old bayc.2026-09-01.json \\ + --new bayc.2026-09-07.json Typical use is a scheduled re-run of one of the engines, kept under a dated filename, diffed against the previous one: - python3 playwright_scraper.py --url "$URL" --out "ml_$(date +%F)" - python3 diff_runs.py --old "ml_$(ls -t ml_*.json | sed -n 2p)" \\ - --new "ml_$(date +%F).json" --out diff.json - -Four buckets, each keyed on sku — here the market's slug: - - added — sku present in --new, absent from --old - removed — sku present in --old, absent from --new (deleted or - collapsed, or just off this particular feed run) - changed — sku present in both, with a different price, order book, - volume, liquidity, state or question - source_changed — sku present in both with a different value, but also a - different `data_source`. That is the bucket this site - needs most: `spread`, the volume windows and the on-chain - ids are published by an EVENT page and by nothing else, so - a `markets` run diffed against an `events` run would - report every one of them as having appeared from nowhere. - Reported separately because it says something about our - own two snapshots rather than about the site — and - --fail-on-change deliberately ignores it. - -A market this project's parser could not recover a sku for (None) cannot be + python3 playwright_scraper.py --collection boredapeyachtclub \\ + --out "bayc_$(date +%F)" + python3 diff_runs.py --old "$(ls -t bayc_*.json | sed -n 2p)" \\ + --new "bayc_$(date +%F).json" --out diff.json + +WHICH COLUMNS are compared depends on the run's MODE, because the three modes +write three different row classes (`Item`, `Collection`, `Activity`) that +share only the family prefix. The mode is read from the `.meta.json` +sidecar, or from the rows' own columns when there is no sidecar. + +Buckets, each keyed on sku: + + added — sku present in --new, absent from --old + removed — sku present in --old, absent from --new (sold out of + the listed walk, delisted, out of this ranking window, + or simply beyond the pages this run fetched) + changed — sku present in both, with a different tracked value + source_changed — sku present in both with a different value, but also a + different `data_source`: one run read the page's own + state or the endpoint, the other fell back to the DOM, + which carries no prices. Reported separately because it + says something about our own two snapshots rather than + about the site, and --fail-on-change ignores it. + within_tolerance — only numeric columns moved, and by less than + --price-tolerance-pct. + +A row this project's parser could not recover a sku for (None) cannot be matched across runs at all, so it is counted and reported separately rather -than silently folded into "added"/"removed", which would be wrong on its face. +than silently folded into "added"/"removed". """ import argparse @@ -48,63 +53,82 @@ from output_writer import UNIQUE_BY_SKU_MODES -# WHAT COUNTS AS A CHANGE ON THIS SITE, and why each of these and not more. +# WHAT COUNTS AS A CHANGE ON THIS SITE, per mode. # -# These names are CHECKED against the row class by the offline suite, and -# that check exists because this file arrives from a sibling repo tracking -# ITS columns. Nothing fails when they are wrong: `diff_runs.py` simply -# compares eight fields that are absent from every row of both runs and -# reports "no changes" for ever. That is this family's most common bug shape -# — a tool doing less than it says while reporting success (§16) — and it -# survives a green suite because the suite tests the DIFF's mechanics rather -# than its subject. +# These names are CHECKED against each mode's row class by the offline suite. +# Nothing fails at runtime when they are wrong: comparing a column that is +# absent from every row of both runs simply reports "no changes" for ever. # -# price / currency the whole point of an items run: what the -# cheapest standing listing costs, in the token it -# is denominated in. `currency` rides along -# because 6.8 ETH and 6.8 WETH are not the same -# price, and a listing can be re-made in another -# token without the number moving. -# best_offer the other side of the book. An offer moving -# while the ask holds still is an event a monitor -# wants and no `price` column shows. -# last_sale / last_sale_at a SALE happened — the most consequential thing -# that can occur to an NFT, and it moves neither -# the listing nor the offer by itself. -# owner_address it changed hands. Tracked as the address and not -# the username, because a handle is display text -# and a person can rename themselves without -# anything having happened (§20: join on the id, -# never on a label). +# items: +# price / currency what the cheapest standing listing costs, in the +# token it is denominated in (6.8 ETH and 6.8 WETH +# are not the same price). +# best_offer the other side of the book. +# last_sale / last_sale_at a sale happened. +# owner_address it changed hands. The address and not the +# username, because a handle is display text. # listing_marketplace OpenSea aggregates, so a listing moving from -# `opensea` to `blur` is a real change in where -# the item is for sale. -# is_delisted OpenSea hiding an item — a reported theft, -# usually — which removes it from the market -# without changing any price. -# rarity_rank / title both change what the row MEANS rather than what -# it costs. A rank moves when a collection is -# re-ranked; a name moves when its metadata is -# updated. +# `opensea` to `blur` is a real change. +# is_delisted OpenSea hiding an item (usually a reported theft). +# rarity_rank / title what the row MEANS rather than what it costs. # -# Deliberately NOT tracked: `price_usd` and the other USD columns, which move -# with the token every minute and would report a change on every row of every -# run; `scraped_at` (it differs by construction); and `page`/`position`, -# which are properties of the ORDERING rather than of the item — and two runs +# collections: +# price / top_offer the floor and the best collection offer. +# volume / sales / owners activity in the run's window (`volume_window` +# is the same for both runs: two runs under +# different timeframes are refused below). +# floor_change / rank how the collection moved and where it now sits. +# total_supply / verified / title: what the collection IS. +# +# activity: +# An event is a past fact and should not change once recorded; every +# column that describes it is tracked anyway, so a difference (a re-indexed +# event, a parser change) is reported rather than hidden. +# +# Deliberately NOT tracked in any mode: the USD columns, which move with the +# token every minute; `scraped_at` (it differs by construction); and +# `page`/`position`, which are properties of the ORDERING — and two runs # under different orderings are refused outright, below. -TRACKED_FIELDS = ("price", "currency", "best_offer", "best_offer_currency", - "last_sale", "last_sale_at", "owner_address", - "listing_marketplace", "is_delisted", "rarity_rank", - "title") - -# The subset whose presence depends on WHICH ROUTE built the row. Nothing on -# this site actually qualifies — the inlined state and the endpoint return -# the same objects, measured 49 of 50 rows identical across 18 stable columns -# — except the DOM fallback, which carries no prices at all. A run that fell -# back to the anchors would otherwise report every price as having vanished, -# so `data_source` differing turns a difference into `source_changed` rather -# than into a change (§8). -COUNT_FIELDS = ("price", "best_offer", "last_sale", "rarity_rank") +TRACKED_FIELDS_BY_MODE = { + "items": ("price", "currency", "best_offer", "best_offer_currency", + "last_sale", "last_sale_at", "owner_address", + "listing_marketplace", "is_delisted", "rarity_rank", "title"), + "collections": ("price", "currency", "top_offer", "top_offer_currency", + "floor_change", "volume", "sales", "owners", + "total_supply", "rank", "verified", "title"), + "activity": ("event_type", "event_time", "price", "currency", "quantity", + "from_address", "to_address", "token_id", "title"), +} + +# The numeric subset per mode: what --price-tolerance-pct may absorb, and +# what a DOM-fallback row lacks (it carries no prices), so a `data_source` +# difference turns a difference here into `source_changed`. +COUNT_FIELDS_BY_MODE = { + "items": ("price", "best_offer", "last_sale", "rarity_rank"), + "collections": ("price", "top_offer", "volume", "sales", "owners", + "floor_change"), + "activity": ("price", "quantity"), +} + +# The items lists under their historical names, for anything importing them. +TRACKED_FIELDS = TRACKED_FIELDS_BY_MODE["items"] +COUNT_FIELDS = COUNT_FIELDS_BY_MODE["items"] + + +def mode_of_rows(rows: List[dict]) -> Optional[str]: + """The mode a run's rows were written in, read from their own columns. + + Used only when there is no sidecar to say so: `event_type` exists only on + an activity row and `volume` only on a collections row. + """ + for r in rows: + if "event_type" in r: + return "activity" + if "volume" in r or "top_offer" in r: + return "collections" + if "best_offer" in r or "rarity_rank" in r: + return "items" + return None def _load(path: str) -> List[dict]: @@ -132,27 +156,23 @@ def _by_sku(products: List[dict]) -> Tuple[Dict[str, dict], int]: def _within_tolerance(before: dict, after: dict, changes: dict, - tolerance_pct: float) -> bool: - """True if every differing count field moved by less than `tolerance_pct`. - -Unlike in most of this family, this flag has a real use here and the - reason is worth stating. A prediction market's price moves CONTINUOUSLY: - two runs of the same command minutes apart will differ on most rows by a - tick, and a monitor alerted on every tick is a monitor nobody reads. A - monitor watching for a real repricing wants a threshold; a monitor - watching for a market CLOSING wants `closed`, which is not a count field - and is never absorbed by this. - - It still DEFAULTS TO ZERO, because the default should report what - happened rather than decide for the reader what was interesting. - - A move is judged on the LARGEST relative change among the count fields, - so a genuine collapse in one price is not hidden by a tolerance applied - field-by-field. + tolerance_pct: float, + count_fields: Tuple[str, ...] = COUNT_FIELDS) -> bool: + """True if every differing numeric field moved by less than `tolerance_pct`. + + A floor price, a collection's volume and its owner count move all the + time, so a monitor alerted on every small move is a monitor nobody reads. + A monitor watching for an item changing hands wants `owner_address`, + which is not numeric and is never absorbed by this. + + It DEFAULTS TO ZERO, because the default should report what happened + rather than decide for the reader what was interesting. Every numeric + field must be inside the tolerance, so a genuine collapse in one is not + hidden by small moves in the others. """ if tolerance_pct <= 0: return False - for field in COUNT_FIELDS: + for field in count_fields: if field not in changes: continue was, now = before.get(field), after.get(field) @@ -166,12 +186,18 @@ def _within_tolerance(before: dict, after: dict, changes: dict, def diff_products(old: List[dict], new: List[dict], - price_tolerance_pct: float = 0.0) -> dict: - """The four buckets. Named `diff_products` for the family's call shape. + price_tolerance_pct: float = 0.0, + mode: Optional[str] = None) -> dict: + """The buckets. Named `diff_products` for the family's call shape. - `price_tolerance_pct` keeps the family's parameter name; on this site it - is a COUNT tolerance — see `_within_tolerance`. + `mode` picks the columns compared; when it is not given it is read from + the rows themselves (see `mode_of_rows`), falling back to items. """ + mode = mode or mode_of_rows(new) or mode_of_rows(old) or "items" + if mode not in TRACKED_FIELDS_BY_MODE: + raise ValueError(f"diff_runs cannot compare a {mode!r} run") + tracked = TRACKED_FIELDS_BY_MODE[mode] + count_fields = COUNT_FIELDS_BY_MODE[mode] old_by_sku, old_unmatchable = _by_sku(old) new_by_sku, new_unmatchable = _by_sku(new) @@ -183,71 +209,44 @@ def diff_products(old: List[dict], new: List[dict], before, after = old_by_sku[sku], new_by_sku[sku] field_changes = { field: {"old": before.get(field), "new": after.get(field)} - for field in TRACKED_FIELDS + for field in tracked if before.get(field) != after.get(field) } if not field_changes: continue - # THE TWO RUNS READ DIFFERENT VIEWS, which is not a change in the - # answer — and on this site this is the bucket that matters most. - # - # Upvotes, views, shares, comments and the question answer count come - # from the payload the view carried, which an archive or author page - # carries and a topic page does not. So a row read off a topic feed - # has no spread and carries its EVENT's volume, and the same market - # read off its own event page has a spread and the MARKET's volume. - # Diffing the two would report every one of them as having appeared - # from nowhere or collapsed by two orders of magnitude. - # - # TWO columns decide "which view", not one, and the second is the - # subtle one. `data_source` is `flight` on a listing row AND on an - # event-page row — the payload is the payload — so a `markets` run - # diffed against an `events` run agrees on it while disagreeing about - # what `volume` MEANS. `volume_scope` is the column that says which, - # and a run where it moved is a run that changed its view. Without it - # the most likely diff anyone will actually run here — yesterday's - # listing against today's deep walk — reports a 199,000,000 -> - # 19,000,000 "collapse" on rows where nothing happened at all. - # - # `--fail-on-change` ignores this bucket for the same reason it - # ignores a tolerance move: it says which view we read, not what - # changed on the site. + # THE TWO RUNS READ DIFFERENT ROUTES. A row built from the page's + # own state and one built from the endpoint carry the same objects; + # a row from the DOM fallback carries no prices at all. So when + # `data_source` differs, a difference in a numeric column is about + # our two snapshots rather than the site, and goes to its own bucket. sources = (before.get("data_source"), after.get("data_source")) - scopes = (before.get("volume_scope"), after.get("volume_scope")) - view_fields = COUNT_FIELDS + ("volume_scope",) - if (sources[0] != sources[1] or scopes[0] != scopes[1]) and any( - f in field_changes for f in view_fields): + if sources[0] != sources[1] and any( + f in field_changes for f in count_fields): view_part = {f: v for f, v in field_changes.items() - if f in view_fields} + if f in count_fields} other_part = {f: v for f, v in field_changes.items() - if f not in view_fields} + if f not in count_fields} source_changed.append({ "sku": sku, "title": after.get("title"), "data_source": {"old": sources[0], "new": sources[1]}, - "volume_scope": {"old": scopes[0], "new": scopes[1]}, "changes": view_part, }) field_changes = other_part if not field_changes: continue - # There is no lifecycle bucket on this site, and its absence is a - # measurement rather than an omission. A sibling repo needs one - # because an auction closing moves a bid kind and the amount beside - # it in one event; an answer has no such state machine. What it does - # have -- being deleted or collapsed -- makes it vanish from the - # feed, which is the `removed` bucket. The `lifecycle` key is still - # emitted, always empty, so a consumer written against the family - # diff shape does not have to branch. - - # A counter ticking rather than a real move -- see - # `_within_tolerance`. Only when the ONLY differences are count - # fields: a title or a body length changing alongside is a real - # change whatever the size of the move. - if (all(f in COUNT_FIELDS for f in field_changes) + # There is no lifecycle bucket on this site; the `lifecycle` key is + # still emitted, always empty, so a consumer written against the + # family diff shape does not have to branch. + + # A small numeric move rather than an event -- see + # `_within_tolerance`. Only when the ONLY differences are numeric: + # a title or an owner changing alongside is a real change whatever + # the size of the move. + if (all(f in count_fields for f in field_changes) and _within_tolerance(before, after, field_changes, - price_tolerance_pct)): + price_tolerance_pct, count_fields)): within_tolerance.append({"sku": sku, "title": after.get("title"), "changes": field_changes}) continue @@ -256,6 +255,7 @@ def diff_products(old: List[dict], new: List[dict], "changes": field_changes}) return { + "mode": mode, "added": added, "removed": removed, "changed": changed, @@ -271,15 +271,15 @@ def _print_summary(result: dict) -> None: print(f"[+] {len(result['added'])} added, {len(result['removed'])} removed, " f"{len(result['changed'])} changed, " f"{len(result['source_changed'])} not comparable (the two runs read " - f"different views), " - f"{len(result.get('within_tolerance', []))} within the count " - f"tolerance.") + f"different routes), " + f"{len(result.get('within_tolerance', []))} within the " + f"tolerance ({result.get('mode')} run).") for p in result["added"]: print(f" + {p.get('sku')} {p.get('title')} " - f"{p.get('price')} in {p.get('event_slug')}") + f"{p.get('price')} {p.get('currency') or ''}".rstrip()) for p in result["removed"]: print(f" - {p.get('sku')} {p.get('title')} " - f"{p.get('price')} in {p.get('event_slug')}") + f"{p.get('price')} {p.get('currency') or ''}".rstrip()) for c in result["changed"]: deltas = ", ".join(f"{f}: {v['old']!r} -> {v['new']!r}" for f, v in c["changes"].items()) @@ -288,18 +288,15 @@ def _print_summary(result: dict) -> None: moves = ", ".join( f"{f}: {v['old']} -> {v['new']}" for f, v in c["changes"].items()) print(f" ~ {c['sku']} {c['title']} {moves} [within --price-" - f"tolerance-pct: a live counter ticking, not an event]") + f"tolerance-pct: a small move, not an event]") for c in result["source_changed"]: src = c["data_source"] deltas = ", ".join(f"{f}: {v['old']!r} -> {v['new']!r}" for f, v in c["changes"].items()) - scope = c.get("volume_scope") or {} print(f" ? {c['sku']} {c['title']} {deltas} " - f"[data_source {src['old']!r} -> {src['new']!r}, volume_scope " - f"{scope.get('old')!r} -> {scope.get('new')!r}: the two runs " - f"read different views of the same market, so this is not a " - f"site-side change. A listing row has no spread and carries " - f"its EVENT's volume; an event-page row has both of its own]") + f"[data_source {src['old']!r} -> {src['new']!r}: the two runs " + f"read different routes, and a DOM-fallback row carries no " + f"prices, so this is not a site-side change]") unmatchable = result["unmatchable_old"] + result["unmatchable_new"] if unmatchable: print(f"[!] {unmatchable} row(s) across both files had no sku or a " @@ -341,7 +338,7 @@ def _check_comparable(args) -> bool: # 282 of 9,998 on one collection — while `--sort created` walks the # collection itself. Two runs under different orderings are two different # SAMPLES, so every row the other happens not to hold reads as `added` or - # `removed` (§21: the ordering is a column, not a sidecar footnote). + # `removed` (the ordering is a column, not a sidecar footnote). orderings = {} for label, path in (("--old", args.old), ("--new", args.new)): status, meta = _run_status(path) @@ -356,12 +353,12 @@ def _check_comparable(args) -> bool: if any(ordering): orderings[label] = ordering if mode and mode not in UNIQUE_BY_SKU_MODES: - # This tool's whole premise is one row per `sku`, diffed on - # A mode that produces many rows per sku would give a diff - # whose every line is an artefact of two rows sharing an id, so - # it is refused outright rather than answered. Both of this - # repo's current modes qualify; the check is here so that adding - # one that does not is caught rather than discovered. + # This tool's whole premise is one row per `sku`. A mode that + # produces many rows per sku would give a diff whose every line + # is an artefact of two rows sharing an id, so it is refused + # outright rather than answered. All three current modes + # qualify; the check is here so that adding one that does not is + # caught rather than discovered. problems.append( f"{label} ({path}) is a {mode!r} run, which is not one row " f"per sku. This tool diffs one row per sku, so there is " @@ -388,24 +385,11 @@ def _check_comparable(args) -> bool: f"one sample happens not to contain would read as `added` or " f"`removed`.") - # WHICH ADDRESS ANSWERED, and deliberately NOT which language. - # - # `source` is the host that served a row. On this site it is - # `opensea.io` on every row — the seven locales are PATHS under - # one host only. A run whose rows carry more than one is - # NOT normal here and is worth saying out loud — so a mixed run is - # never refused. What IS worth saying is when two runs each landed - # consistently on a DIFFERENT single host, because then `added` and - # `removed` would be describing the address rather than the catalogue. - # - # There is deliberately no language check, and the reason is worth - # writing down because the sibling repo this file came from has one. - # There, `language` meant which of twenty-four language SITES a row came - # from, and those are different catalogues. Here it is the site's own - # `detectedLanguage` for the STORY — one measured English tag's day - # archive carried 123 `en` rows and 5 `id` ones — so refusing a pair - # whose languages differ would refuse two perfectly comparable runs of - # the same tag. + # WHICH ADDRESS ANSWERED. `source` is the host that served a row; on + # this site it is `opensea.io` on every row, because the locales are + # PATHS under one host. Two runs that each landed on a DIFFERENT single + # host would make `added` and `removed` describe the address rather than + # the catalogue, so that pair is refused. sources = {} for label, path in (("--old", args.old), ("--new", args.new)): try: @@ -417,17 +401,15 @@ def _check_comparable(args) -> bool: sources[label] = hosts.pop() if len(set(sources.values())) > 1: problems.append( - f"the two runs landed on different hosts ({sources}). This site " - f"serves one story from several addresses, so this is usually a " - f"different URL rather than a different catalogue — but added " + f"the two runs landed on different hosts ({sources}), so added " f"and removed would describe the address change rather than " - f"anything about the markets.") + f"anything on OpenSea.") if not problems: return True - # A generic headline, because the reasons below are no longer only about - # completeness: a mode mismatch and a reviews run are refused too, and a + # A generic headline, because the reasons below are not only about + # completeness: a mode or ordering mismatch is refused too, and a # message naming the wrong reason sends the reader looking in the wrong # place. print("[!] Refusing to diff these two runs:") @@ -448,15 +430,13 @@ def parse_args(): help="Write the full diff as JSON to this path too.") p.add_argument("--price-tolerance-pct", type=float, default=0.0, metavar="PCT", - help="Treat a counter move smaller than PCT%% as a live " - "counter ticking rather than an event: reported " - "separately and ignored by --fail-on-change. Default 0 " - "— report every tick. Unlike in most of this family " - "the flag has a real use here: this site's own " - "clap counts are live, so a monitor watching for an " - "answer taking off wants a threshold, while one " - "watching for an edit wants text_chars, which this " - "never absorbs.") + help="Treat a move smaller than PCT%% in the numeric " + "columns (a floor price, an offer, a collection's " + "volume, sales or owners) as noise rather than an " + "event: reported separately and ignored by " + "--fail-on-change. Default 0 — report every move. " + "A change in a non-numeric column (an owner, a " + "marketplace, a title) is never absorbed.") p.add_argument("--fail-on-change", action="store_true", help="Exit 1 if anything was added, removed or changed — " "for a cron job that should only notify on a real diff.") @@ -479,7 +459,17 @@ def main() -> int: print(f"[!] Could not read one of the input files: {e}") return 2 - result = diff_products(old, new, price_tolerance_pct=args.price_tolerance_pct) + mode = None + for path in (args.new, args.old): + _, meta = _run_status(path) + if meta and meta.get("mode"): + mode = meta["mode"] + break + if mode is not None and mode not in TRACKED_FIELDS_BY_MODE: + print(f"[!] This tool cannot compare a {mode!r} run.") + return 2 + result = diff_products(old, new, price_tolerance_pct=args.price_tolerance_pct, + mode=mode) _print_summary(result) if args.out: @@ -488,12 +478,9 @@ def main() -> int: print(f"[+] Full diff written to {args.out}") # Neither `source_changed` nor `within_tolerance` is a reason to fail. - # The first means our two runs read different views of the same answer; - # the second means a live counter ticked. Neither says anything about the - # site, and alerting on either would train whoever reads the alert to - # ignore it. - # Neither `source_changed` nor `within_tolerance` is a reason to fail — - # see their comments above. + # The first means our two runs read different routes; the second means a + # number moved by less than the reader asked to hear about. Alerting on + # either would train whoever reads the alert to ignore it. if args.fail_on_change and (result["added"] or result["removed"] or result["changed"]): return 1 return 0 diff --git a/env_config.py b/env_config.py index a75434e..294d6dc 100644 --- a/env_config.py +++ b/env_config.py @@ -67,7 +67,7 @@ # A literal-only check reported both of those as CONFIGURED, so `cp # .env.example .env` followed by a run connected to cb.2captcha.com with the # string `{login}-zone-…` as its username and got a 401 — a confusing failure -# a long way from its cause, which is precisely what §3's rule exists to +# a long way from its cause, which is precisely what this rule exists to # prevent. Any `{…}` left in a value means the line was never filled in. _BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}") @@ -204,7 +204,7 @@ def _dests_given_on_argv(dests, argv=None): headful path and its Scraping Browser path are both first-class, and switching between them is routine rather than exceptional. - Scanning argv is the smallest fix that keeps §3's precedence intact -- + Scanning argv is the smallest fix that keeps the documented precedence intact -- explicit flag, then exported variable, then `.env`, then default -- with "explicit" now meaning what the caller typed rather than what happens to be truthy. diff --git a/fingerprint_client.py b/fingerprint_client.py index 7037176..adda808 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -113,7 +113,7 @@ def _cache_path(cache_dir: str, params: dict, generate: bool, fingerprint 5393493 from disk and raised nothing, because a real key had fetched the same parameters earlier. A user whose fingerprint subscription lapsed would see `--fingerprint` keep working on their own - machine and 401 on a fresh one (§16: a path that looks like it works). + machine and 401 on a fresh one (a path that looks like it works). Keying on the key costs one extra fetch per key and makes a 401 a 401. """ @@ -264,7 +264,7 @@ def playwright_context_kwargs(fp: dict) -> dict: # 2026-09-11: a fingerprint stating `deviceScaleFactor: 1.25` produced a # browser reporting `window.devicePixelRatio === 1`, so the identity # contradicted itself on an axis any fingerprinter reads for free. Same - # shape as the three defects §16 lists -- a key the API returns that + # shape as several defects copied fingerprint code carried -- a key the API returns that # nothing applies -- and found the same way, by comparing the live # browser against the fingerprint rather than by reading the code. scale = (fp.get("screen") or {}).get("deviceScaleFactor") @@ -345,7 +345,7 @@ def main() -> int: # loader. It used to read `os.environ` alone, which meant a key put in # `.env` exactly as README and .env.example instruct worked for every # engine and failed here with "No API key" — a documented mechanism not - # applied on one path, which is the shape of half the defects §16 lists. + # applied on one path, which is a common shape of defect in copied code. # `.env` has to be LOADED before it can be read: `env_value` looks at # os.environ, which `load_env` is what fills from the file. Calling it # here rather than relying on an engine having called it is the whole diff --git a/make_fixtures.py b/make_fixtures.py index 959ce04..f9584cf 100644 --- a/make_fixtures.py +++ b/make_fixtures.py @@ -40,7 +40,7 @@ capture, and written into the fixture file. A hand-typed expectation is a guess about the site; a computed one is a record of it. -WHAT IS NOT VERBATIM, and why (§10) +WHAT IS NOT VERBATIM, and why ------------------------------------ Two kinds of field are replaced with obvious placeholders before anything else happens: @@ -52,7 +52,7 @@ capture of a busy collection carries dozens of them. The site showing them on its own page is one thing; this repository republishing them is a separate act, and the checks need the STRUCTURE of an owner rather than the person -(§10). The scrub runs on the ORIGINAL before the trim, so what the tests +The scrub runs on the ORIGINAL before the trim, so what the tests compare against is the scrubbed page and the two cannot disagree. WALLET ADDRESSES ARE KEPT, deliberately. An address is the row's id, it is @@ -112,7 +112,7 @@ "items", None), # THE SAME collection page, fetched over the 2Captcha Scraping Browser. # It is here because a guard is only as good as the fixture it runs - # against (§21): the check that `cf-turnstile` must never be a marker + # against: the check that `cf-turnstile` must never be a marker # would otherwise run only against pages this repo fetched with a local # browser, which carry no extension injection at all — and would pass for # the wrong reason. This one carries the injection, measured. @@ -127,7 +127,7 @@ # --------------------------------------------------------------------------- -# The scrub (§10) +# The scrub # --------------------------------------------------------------------------- # Applied to the ORIGINAL capture, before anything is trimmed or measured, so # the expectations written into the fixture file describe the scrubbed page @@ -222,7 +222,7 @@ def _strip_furniture(html: str) -> str: # --------------------------------------------------------------------------- -# The leak scan (§10) +# The leak scan # --------------------------------------------------------------------------- # PATTERNS rather than the literals of a capture that happened to be clean: # the next capture is the one that will carry something. @@ -296,7 +296,7 @@ def _expectations(rows: list, html: str, url: str, mode: str) -> dict: Values, not coverage: a column can be 100% populated and entirely wrong, and a sibling repo shipped a `review_count` of 445279961 on every row - with a coverage check reading 100% (§10). + with a coverage check reading 100%. """ first = asdict(rows[0]) if rows else {} first.pop("scraped_at", None) @@ -365,7 +365,7 @@ def build(name: str, filename: str, url: str, mode: str, keep) -> dict: # Real RESPONSES from the endpoint, for the shapes a page capture cannot # show. Captured 2026-10-13 by sending the request that produces each, and -# stored verbatim — §26's rule, because a fixture assembled by hand can +# stored verbatim, because a fixture assembled by hand can # claim what no response ever says. A third-party audit asked for exactly # these on the same day. # diff --git a/output_writer.py b/output_writer.py index 9fcf4de..ec97c51 100644 --- a/output_writer.py +++ b/output_writer.py @@ -15,8 +15,7 @@ a sale, a listing, an offer, a transfer or a mint Unlike most of this family, these are three genuinely different KINDS of -thing rather than three views of one, so they get three dataclasses (§9's -allowance, which amazon-scraper spends on `Review`). What they share is the +thing rather than three views of one, so they get three dataclasses. What they share is the family prefix — `source`, `scraped_at`, `url`, `sku`, `title` — byte- identical and in the same order in all three, so one column name works across the family and across the modes. `diff_runs.py` refuses to compare @@ -73,11 +72,14 @@ actually denominated in — and never a defaulted "USD". The dollar figure the site computes beside it is `price_usd`, in its own column, so a consumer can tell an exact on-chain amount from a conversion that moves with the market. -§4's rule reads the same here as it does with fiat: the structured source +The rule reads the same here as it does with fiat: the structured source names the currency, so it is a fact and is never overwritten from the DOM. """ +import contextlib import csv import json +import os +import tempfile from dataclasses import dataclass, asdict, field, fields from datetime import datetime, timezone from typing import Optional, List, Set, Sequence, Any, Type @@ -116,7 +118,7 @@ class Item: currency: Optional[str] = None # The site's own dollar conversion of `price`, which moves with the token # even when the listing does not. A separate column precisely so the two - # are never confused (§8: never present a guess as a fact — this one is + # are never confused (never present a guess as a fact — this one is # the site's guess, clearly labelled). price_usd: Optional[float] = None # The highest standing bid, same three-column shape. An offer is what @@ -162,21 +164,21 @@ class Item: # flags the account. Both are the site's assertion, carried through # rather than acted on. # - # NEITHER HAS EVER BEEN SEEN TRUE, and §20 says to write that down rather + # NEITHER HAS EVER BEEN SEEN TRUE, and that is written down here rather # than let a column look verified because it is always populated. # Measured 2026-09-17: 3,400 items — 1,000 under the price ordering # across ten collections, and 2,400 under the created ordering across # four — came back `False` on both, with not one True. # # The price-ordered half of that proves nothing on its own and is the - # §20 mistake made deliberately, then corrected: an item with a live + # mistake made deliberately, then corrected: an item with a live # listing is by definition not hidden, so looking for a delisting among # listed items is looking where it cannot be. The created-ordered scan is # the one that counts, and it found none either. # # So the open question, unresolved: either OpenSea excludes hidden items # from `collectionItems` altogether — which would make this column a - # constant and, by §9, one that should not exist — or they are simply + # constant, and a column that is always the same should not exist — or they are simply # rarer than 2,400 items of four blue-chip collections. A `False` here is # therefore LESS PROVEN than a `True` would be, and a consumer should not # read it as "OpenSea has checked and cleared this item". @@ -186,7 +188,7 @@ class Item: # --- provenance -------------------------------------------------------- # WHICH route built this row. Never a guess presented as a fact, and # `diff_runs.py` reports a difference that comes with a `data_source` - # difference as `source_changed` rather than as a change (§8). + # difference as `source_changed` rather than as a change. # # ssr the page's own server-rendered state — page 1 of a run # graphql the site's own GraphQL endpoint, which is what pages 2..N @@ -368,7 +370,7 @@ def merge_pages(groups: Sequence[Any], key: str = "sku") -> tuple: Pages really are sequential on this site — page N+1's cursor comes out of page N — so this costs nothing and keeps the family's contract: dedupe that mutates a running set inside the loop makes the OUTPUT depend on - which page finished first (§8). + which page finished first. The upgrade-in-place rule the family carries is inert here — see `is_deeper`, which is False on every mode this repo has, because no run @@ -420,12 +422,88 @@ def _csv_value(v: Any) -> Any: return v +@contextlib.contextmanager +def _atomic(path: str, newline: Optional[str] = None): + """Write to a temporary file beside `path`, then rename over it. + + `open(path, "w")` truncates before a single byte is written, so a crash, + a kill, a full disk or a Ctrl-C halfway through `json.dump` leaves a + SHORTER file where a complete one was, and the previous good run is + destroyed by the attempt to replace it. Reproduced here before this + existed: an error injected inside `json.dump` left `out.json` 12 bytes of + invalid JSON, and `out.csv` 0 bytes. The sidecar matters most, because it + is the file a consumer branches on. + + * the temp file goes in the target's OWN directory, because `os.replace` + is atomic only within one filesystem; + * `fsync` before the rename makes the bytes durable, not merely visible; + * `newline` is passed through because the csv module needs `newline=""`; + * `NamedTemporaryFile` creates at 0600 and `os.replace` keeps that mode, + so an existing target keeps ITS mode and a new one gets exactly what + `open()` would have given it (`0o666 & ~umask`) — never a flat 0644, + which would narrow 0664 under umask 002. + + Each file is replaced atomically on its own; the JSON, CSV and sidecar + are not replaced as one set. + """ + directory = os.path.dirname(os.path.abspath(path)) or "." + handle = tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", newline=newline, dir=directory, + prefix=os.path.basename(path) + ".", suffix=".tmp", delete=False) + try: + with handle: + yield handle + handle.flush() + os.fsync(handle.fileno()) + try: + mode = os.stat(path).st_mode & 0o777 + except OSError: + umask = os.umask(0) + os.umask(umask) + mode = 0o666 & ~umask + os.chmod(handle.name, mode) + os.replace(handle.name, path) + except BaseException: + # Including KeyboardInterrupt: an interrupted run must not leave + # `out.json.ab12cd.tmp` behind beside the good file. + try: + os.unlink(handle.name) + except OSError: + pass + raise + + +# A spreadsheet treats a cell beginning with one of these as a FORMULA rather +# than as text, and every string in these rows was written by whoever named +# the collection or the NFT. The tab and the two newlines are here because +# some readers strip a leading one and then execute whatever followed it. +CSV_FORMULA_LEADS = ("=", "+", "-", "@", "\t", "\r", "\n") + +# Excel, Google Sheets and LibreOffice read a leading apostrophe as "the rest +# of this cell is text" and do not display it. A consumer parsing the file +# with the `csv` module DOES see it, which is why the count goes into the +# sidecar as `csv_cells_escaped`. +CSV_FORMULA_ESCAPE = "'" + + +def _csv_escape(v: Any): + """Neutralise a formula-shaped cell. Returns (value, was_escaped). + + Only STRINGS are touched: escaping a number would turn `-5.0` into text + and break every sum over the column (`floor_change` is negative on any + falling collection). CSV only — the JSON output keeps the site's bytes. + """ + if isinstance(v, str) and v.startswith(CSV_FORMULA_LEADS): + return CSV_FORMULA_ESCAPE + v, True + return v, False + + def write_json(rows: Sequence[Any], path: str) -> None: - with open(path, "w", encoding="utf-8") as f: + with _atomic(path) as f: json.dump([asdict(r) for r in rows], f, ensure_ascii=False, indent=2) -def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: +def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> int: # An empty result still gets the header row. A zero-byte file makes a # consumer fail on read (no columns to parse) instead of reading a valid # table with zero rows — and "an empty result is still a well-formed @@ -433,12 +511,23 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: # # The header comes from `row_cls`, not from the first row, so an empty # run still writes the columns of the mode that produced it. + # + # Returns how many cells were formula-escaped. fieldnames = [f.name for f in fields(row_cls)] - with open(path, "w", encoding="utf-8", newline="") as f: + escaped = 0 + with _atomic(path, newline="") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() for r in rows: - writer.writerow({k: _csv_value(v) for k, v in asdict(r).items()}) + row = {} + for k, v in asdict(r).items(): + # AFTER the list serialiser: a joined list is text too, and + # its first element can be formula-shaped. + value, was_escaped = _csv_escape(_csv_value(v)) + escaped += was_escaped + row[k] = value + writer.writerow(row) + return escaped # Exit code used when a run completes but produced nothing. Distinct from 1 @@ -462,7 +551,7 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: # Nothing was refused while this repo was built — see the README for the # addresses and the dates — so this path is implemented and unexercised # here, which is a fact about this repo's testing rather than a claim about -# the site or about what a solver can do (§19). +# the site or about what a solver can do. EXIT_BLOCKED = 3 # Exit code for a run that gathered SOME rows and then stopped early — a @@ -523,7 +612,7 @@ def write_run_meta(out_prefix: str, meta: dict) -> str: both complete, and between runs of different `mode`. """ path = f"{out_prefix}.meta.json" - with open(path, "w", encoding="utf-8") as f: + with _atomic(path) as f: json.dump(meta, f, ensure_ascii=False, indent=2) print(f"[+] Wrote run metadata -> {path} (status={meta.get('status')})") return path @@ -589,7 +678,8 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, def save(rows: Sequence[Any], out_prefix: str, fmt: str, - allow_empty: bool = False, row_cls: Type = Product) -> int: + allow_empty: bool = False, row_cls: Type = Product, + stats: Optional[dict] = None) -> int: """Write JSON/CSV and return a process exit code. Returns 0 when rows were written, EXIT_NO_PRODUCTS when there were none. @@ -616,8 +706,18 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, write_json(rows, f"{out_prefix}.json") print(f"[+] Saved {len(rows)} products -> {out_prefix}.json") if fmt in ("csv", "both"): - write_csv(rows, f"{out_prefix}.csv", row_cls=row_cls) + escaped = write_csv(rows, f"{out_prefix}.csv", row_cls=row_cls) print(f"[+] Saved {len(rows)} products -> {out_prefix}.csv") + # Through `stats`, because this function's return value is an exit + # code. The caller puts it in the sidecar, which is what declares + # that the CSV and the JSON deliberately differ. + if stats is not None: + stats["csv_cells_escaped"] = escaped + if escaped: + print(f"[!] {escaped} CSV cell(s) began with a formula character " + f"and were prefixed with {CSV_FORMULA_ESCAPE!r} so a " + f"spreadsheet reads them as text. The JSON output is " + f"unchanged — see csv_cells_escaped in the sidecar.") return 0 if rows else EXIT_NO_PRODUCTS @@ -630,8 +730,8 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, # feed with an opaque cursor and hands back a null one when there is nothing # after the page just read. That is the site saying "this is all of it", and # it is the cleanest completion signal any repo in this family has — no -# selector, no convention, no guess (§7's layer 3, with the site doing the -# arithmetic). +# selector, no convention, no guess, with the site doing the +# arithmetic. # # "listed_items_exhausted" is the one that is specific to OpenSea and is # COMPLETE rather than partial, which is the most consequential line in this @@ -691,7 +791,9 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, # names has a hole for every name nobody added to it. complete = stop_reason in COMPLETE_STOP_REASONS and not pages_failed row_cls = ROW_CLASS_BY_MODE.get(mode, Product) - rc = save(rows, out_prefix, fmt, allow_empty=allow_empty, row_cls=row_cls) + write_stats: dict = {} + rc = save(rows, out_prefix, fmt, allow_empty=allow_empty, row_cls=row_cls, + stats=write_stats) wrote_output = bool(rows) or allow_empty if wrote_output: @@ -702,12 +804,15 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, pages_requested=pages_requested, pages_completed=pages_completed, pages_failed=pages_failed, mode=mode, source=source, start_url=start_url, final_url=final_url, products=len(rows), - extra=extra)) + # UNDER the caller's `extra`, never over it: a collision here + # would be housekeeping silently dropping something the site said. + extra=({**write_stats, **(extra or {})} if write_stats + else extra))) if not rows: # Nothing gathered at all, and WHY decides the code. The three # outcomes are genuinely different things and a pipeline branches on - # them (§8: blocked is not empty is not partial): + # them (blocked is not empty is not partial): # # blocked something stood between the run and the content # did not complete we never reached the site — a dead proxy, a diff --git a/page_flow.py b/page_flow.py index 9b5a07f..1ffbfe4 100644 --- a/page_flow.py +++ b/page_flow.py @@ -2,7 +2,7 @@ OpenSea answers a request five ways and four of them want a different response, which is why this module exists rather than the same triage being -written three times inside three engines and drifting apart (§1): +written three times inside three engines and drifting apart: content the page's inlined state names rows, or its tiles rendered empty a collection slug that does not exist (a real HTTP 404), or a @@ -42,7 +42,7 @@ page_height() -> Optional[int] document.body.scrollHeight sleep(ms) -> None wait -No JavaScript crosses that boundary in either direction (§1): Selenium's +No JavaScript crosses that boundary in either direction: Selenium's `execute_script` takes a function BODY with an explicit `return` where Playwright and pyppeteer take `() => expr`, so this module names the OPERATION and each engine spells it in its own driver's dialect. @@ -78,7 +78,7 @@ "collections": SELECTORS["collection_link"], } -# Above 1, per §5: waiting for a single match resolves on the site's own +# Above 1: waiting for a single match resolves on the site's own # navigation — the header links to /collections and to several collections # by name — long before a grid paints. Measured first paints on 2026-09-17: # 32 distinct item links on a collection page, 25 collection links on the @@ -135,11 +135,11 @@ def wait_for_count(count: Callable[[str], int], sleep: Callable[[int], None], page, 2026-09-17). Playwright's `wait_for_function` hands the browser a string to evaluate, which that policy refuses outright: on a sibling site with the same gap it died with `EvalError` and took the run down with - exit 1 (§18). Counting elements over the protocol is a CDP call, works + exit 1. Counting elements over the protocol is a CDP call, works under any CSP, and spells the same in all three drivers. Returns the count it ended on, whether or not it reached the floor: - reporting a timeout is not the same as exiting on one (§8). + reporting a timeout is not the same as exiting on one. """ waited = 0 found = count(selector) @@ -156,7 +156,7 @@ def wait_for_count(count: Callable[[str], int], sleep: Callable[[int], None], # --------------------------------------------------------------------------- # Lazy loading # --------------------------------------------------------------------------- -# §8's second case. OpenSea's grids ARE infinite-scroll — the site's own UI +# Lazy loading. OpenSea's grids ARE infinite-scroll — the site's own UI # fetches the next fifty over GraphQL as you reach the bottom — but this # scraper never needs that, because it asks for the next page over the same # endpoint the grid uses and gets a hundred rows instead of fifty. @@ -170,7 +170,7 @@ def wait_for_count(count: Callable[[str], int], sleep: Callable[[int], None], # distance — a fixed wheel stops short on a long grid and the trigger is # never reached — and require the count AND the height to hold still for # THREE rounds, because the next batch takes longer to arrive than a single -# pause (§8). +# pause. SCROLL_STABLE_ROUNDS = 3 SCROLL_MAX_ROUNDS = 6 SCROLL_PAUSE_MS = 2_000 @@ -219,7 +219,7 @@ def classify(html: Optional[str], status: Optional[int] = None, Getting that wrong is not a style question: a sibling repo shipped two of three engines calling this as `classify(html, url=…)`, both crashed on their first fetch, and nothing short of a live run or a signature-binding - check saw it (§17). This repo's smoke suite binds every shared-module + check saw it. This repo's smoke suite binds every shared-module call in every engine for that reason. `mode` matters here and does not in the siblings: which inlined operation @@ -232,7 +232,7 @@ def classify(html: Optional[str], status: Optional[int] = None, # The retry/solve/blocked decision as DATA rather than as three copies of an -# if-chain in three engines (§1). +# if-chain in three engines. # # parse is there anything on this page worth writing down? # retry would fetching it again, in a fresh context, plausibly help? @@ -243,7 +243,7 @@ def classify(html: Optional[str], status: Optional[int] = None, # A collection slug that does not exist, under a real HTTP 404, or a feed # the site itself says is empty. Asked and answered; a second fetch # returns the same thing. Not blocked: exit 4 is "ran fine, found - # nothing", which is exactly what happened (§8). + # nothing", which is exactly what happened. "empty": {"parse": False, "retry": False, "solve": False, "blocked": False}, # Served and still painting. Parsed rather than discarded, because by the # time an engine asks, the readiness wait has already run. @@ -258,11 +258,11 @@ def classify(html: Optional[str], status: Optional[int] = None, # address and the date), so `solve: True` here is a CAPABILITY and not a # measurement of this site. The one thing that must never be written is # that it cannot be solved, because that would be a sentence about this - # repo's code dressed up as a fact about a paid product (§19). + # repo's code dressed up as a fact about a paid product. # # Nothing is charged for a page that carries no widget: the solver raises # rather than building a task from a sitekey-less detection, which is - # §8's "detected != paying" with a price tag on it. + # "detected is not paying", with a price tag on it. "challenge": {"parse": False, "retry": True, "solve": True, "blocked": True}, "blocked": {"parse": False, "retry": True, "solve": False, "blocked": True}, } @@ -271,7 +271,7 @@ def classify(html: Optional[str], status: Optional[int] = None, # Every state the parser can return has a row here, and every row here is a # state the parser can return. Asserted at import rather than in a test, # because a state with no policy falls back to `blocked` — which is the safe -# direction and also the silent one (§17: a policy that looks enforced and is +# direction and also the silent one (a policy that looks enforced and is # not). assert set(STATE_POLICY) == set(PAGE_STATES), ( "STATE_POLICY and product_parser.PAGE_STATES disagree: " @@ -328,11 +328,38 @@ def is_unpainted(state: str, html: Optional[str]) -> bool: BLOCK_RETRIES_WITHOUT_POOL = 3 SOLVES_PER_PAGE = 1 + +class SolveBudget: + """One page's allowance of paid captcha solves. + + Every engine creates ONE of these per page and passes it to EVERY call + that may buy a solve: the check before the page is classified and the + one after. It is spent immediately BEFORE the solver is called, so a + detection that pays for nothing (no key, rows already on the page, no + sitekey) costs nothing, and it is never reset by a retry, a fresh + context or a rotation to another exit: a page that keeps coming back as + a challenge is a reason to re-fetch, not a new allowance. + """ + + def __init__(self, limit: int = None): + self.limit = SOLVES_PER_PAGE if limit is None else int(limit) + self.spent = 0 + + def remaining(self) -> int: + return max(0, self.limit - self.spent) + + def spend(self) -> bool: + """Take one solve from the allowance. False when none is left.""" + if self.spent >= self.limit: + return False + self.spent += 1 + return True + # There is deliberately NO `BLOCK_RETRIES_WITH_POOL` beside the constant # above, and its absence is the point. It used to be here, set to 4, and # nothing read it: with a pool the budget is `--proxy-block-retries`, which # defaults to 2, so the constant was a second number claiming a different -# policy while the flag quietly decided the real one. §17 calls that a +# policy while the flag quietly decided the real one. That is a # policy constant nothing consults — the same defect as dead code, and # harder to see, because the prose reads like enforcement. The with-pool # budget is the FLAG; the constant above is the fallback for when there is @@ -340,9 +367,9 @@ def is_unpainted(state: str, html: Optional[str]) -> bool: # Whether a retry has to discard the browser context rather than reload the # page. True, on the family's rule that a challenge issued against one -# session is not cleared by asking that same session again — and on §8's +# session is not cleared by asking that same session again — and on the rule # "a rotation is a fresh browser", which is the same rule with a proxy in it. -# Consulted by all three engines; a constant no engine read would be the §17 +# Consulted by all three engines; a constant no engine read would be the # defect of a policy that looks enforced and is not. RETRY_NEEDS_FRESH_CONTEXT = True @@ -350,7 +377,7 @@ def is_unpainted(state: str, html: Optional[str]) -> bool: # --------------------------------------------------------------------------- # A throttle is not a block # --------------------------------------------------------------------------- -# §24's lesson, carried here from a sibling site that could measure it: a +# A lesson carried here from a sibling site that could measure it: a # refusal and a throttle want OPPOSITE responses. A block says "this exit is # unwelcome", so the answer is a different exit. A throttle says "too fast", # so the answer is the SAME exit and a longer wait — rotating away from it @@ -482,14 +509,14 @@ def block_advice(html: Optional[str], headless: bool, has_pool: bool, Says what is measured and says what is not. The temptation is to repeat a sibling repo's ordering — "use a real window first" — but that ordering came from ITS measurements. Here headless and headful were measured - identical, so repeating it would be §13's inherited number wearing a + identical, so repeating it would be an inherited number wearing a fact's clothes. """ marker = detect_block_marker(html or "") or "no vendor marker" lead = f"blocked ({marker})" hints: List[str] = [] - # SAY WHAT WAS OBSERVED BEFORE SAYING WHY (§28). When the engine itself + # SAY WHAT WAS OBSERVED BEFORE SAYING WHY. When the engine itself # stripped the proxy credentials it knows the likeliest cause, and the # general advice below — raise --delay, try --headful — points a reader # away from it. Measured 2026-10-13: Selenium with an authenticated @@ -540,6 +567,28 @@ def block_advice(html: Optional[str], headless: bool, has_pool: bool, # --------------------------------------------------------------------------- # Pagination — cursors, and the one that ends a walk early on purpose # --------------------------------------------------------------------------- +def number_at_least(flag: str, minimum, kind=int): + """An argparse `type=` that refuses a value below `minimum`, naming `flag`. + + Counts and delays are refused at parse time rather than clamped: a + negative `--delay` or a `--pages 0` would otherwise be quietly reinterpreted + and look honoured while not being. + """ + def parse(value): + try: + number = kind(value) + except (TypeError, ValueError): + raise argparse.ArgumentTypeError( + "%s must be a %s, got %r" + % (flag, "whole number" if kind is int else "number", value)) + if number < minimum: + raise argparse.ArgumentTypeError( + "%s must be at least %s, got %r" % (flag, minimum, value)) + return number + parse.__name__ = "number_at_least_%s" % flag.strip("-").replace("-", "_") + return parse + + def attempts(value) -> int: """`--retries` as a number of ATTEMPTS, never zero. @@ -547,15 +596,14 @@ def attempts(value) -> int: usage message rather than four seconds into a run. The bug this closes was found by a third-party audit on 2026-10-13, and - the audit was right about the symptom and wrong about the cause (§28). + the audit was right about the symptom and wrong about the cause. Its live run passed `--retries 0`; the load loop is `for attempt in range(1, args.retries + 1)`, so with 0 it never executes, `goto` is never called, and the engine then classifies `about:blank` — **39 bytes** — as a refusal. The audit reported "OpenSea returned 39 bytes with no vendor markers" and concluded availability was unproven. - Reproduced and settled here with one variable changed, which is §28's - rule: + Reproduced and settled here with one variable changed: --retries 0 exit 3, 39 bytes, four attempts, 37 s --retries 1 exit 0, 1,395,922 bytes, 50 rows, 100% coverage, 11 s @@ -619,7 +667,7 @@ def price_boundary_reached(cursor: Optional[str]) -> bool: remainder of the collection in no particular order. THE REASON FOR THIS STOP CHANGED, and the first version of it has - rotted, which is worth leaving written down (§13). + rotted, which is worth leaving written down. Measured 2026-09-17: passing such a cursor back answered `{"errors":[{"message":"Something went wrong"}]}`, so this was written as @@ -633,7 +681,7 @@ def price_boundary_reached(cursor: Optional[str]) -> bool: So the stop is unchanged and its justification is now the DATA rather than an error message: see `prices_ran_out`, which the engines check - beside this. §7 says a terminating condition must be a property of the + beside this. A terminating condition must be a property of the catalogue rather than of the markup, and "the endpoint rejects this cursor" was the markup. """ @@ -646,7 +694,7 @@ def prices_ran_out(rows, sort: str) -> bool: The data-side half of the stop above, and the half that survives the site changing its mind about the cursor. Under a price ordering a page with not one priced row means the boundary was crossed on the page - before it, and §7 wants the terminating condition to be exactly this + before it, and the terminating condition should be exactly this kind of fact. Only meaningful under a price ordering: under `created` an unpriced page @@ -663,7 +711,7 @@ def sort_note(sort: str, rows: int, totals: Optional[dict] = None) -> str: Printed rather than swallowed, because "I asked for 20 pages and got 3" with no explanation is indistinguishable from the family's most expensive bug — a dead next-page selector quietly returning a third of the data - with exit 0 (§7). Here the run really did get every item that has a + with exit 0. Here the run really did get every item that has a price, and the difference has to be visible in the log and in the sidecar. """ @@ -689,7 +737,7 @@ def sort_note(sort: str, rows: int, totals: Optional[dict] = None) -> str: def totals_note(totals: Optional[dict], rows: int) -> Optional[str]: """What share of the collection this run actually holds. - §21's "complete and exhaustive are different words", with the site doing + "Complete" and "exhaustive" are different words, with the site doing the arithmetic: a three-page run of a 9,998-item collection is genuinely complete under its ordering AND is 3% of the collection. A sidecar that records only the first is lying by omission. @@ -714,14 +762,14 @@ def collection_totals_on_page(html: Optional[str]) -> dict: def page_gap(html: Optional[str], parsed: int) -> Optional[int]: """None, always, on this site — and that is the honest answer. - §8's rank arithmetic works where a site NUMBERS its items: 30 rows + Rank arithmetic works where a site NUMBERS its items: 30 rows spanning ranks 1-50 proves 20 cards never loaded. OpenSea numbers nothing. A token's rarity RANK exists but is a property of the token rather than of this page's ordering, and a ranking's position is one this scraper assigns from the row order rather than one the site publishes. None rather than 0, because an unknown gap is not a gap of zero and the - two must not read the same in a sidecar (§8). + two must not read the same in a sidecar. """ return None @@ -733,7 +781,7 @@ def degraded_to_dom(rows) -> bool: and it emits an address and a name and NO prices (see `product_parser.rows_from_dom`). So a file built entirely out of it has the right columns, the right row count and no money in it — which is the - §8 failure this family keeps meeting: a tool doing less than it says + failure this family keeps meeting: a tool doing less than it says while reporting success. Measured 2026-10-13, which is why this exists: opensea.io stopped @@ -756,7 +804,7 @@ def parsed_nothing_from_a_served_page(html: Optional[str], rows: int) -> bool: """A page that was SERVED, links to items, and produced no rows. That is this parser's bug and not an empty collection, and the two must - not report the same (§20). The engines give it its own stop reason so a + not report the same. The engines give it its own stop reason so a reader opens `product_parser.py` instead of checking their URL for a typo. @@ -782,7 +830,7 @@ def concurrency_refusal(url: str = "", mode: str = "") -> Optional[str]: """Why concurrency above 1 is refused for this run. Refused WITH the reason rather than silently running one worker, which - would look like the flag did something (§18). + would look like the flag did something. This is the one place OpenSea is strictly worse than a site with page numbers: the family's `--concurrency` exists because `?page=5` can be diff --git a/playwright_scraper.py b/playwright_scraper.py index 0df970e..a2c49df 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -165,7 +165,7 @@ class PageOutcome: Collected per page and merged afterwards rather than folded into shared state as the loop goes: dedupe that mutates a running set inside the loop - makes the OUTPUT depend on the order pages happened to arrive in (§8). + makes the OUTPUT depend on the order pages happened to arrive in. Pages are strictly sequential here, which is exactly why keeping the merge order-independent costs nothing and keeps the family's contract. """ @@ -183,7 +183,7 @@ class PageOutcome: # count, floor. Page 1 only; the cursor pages do not carry it. totals: Optional[dict] = None # None, always, on this site: OpenSea numbers nothing, and an unknown gap - # must not read the same as a gap of zero (§8). + # must not read the same as a gap of zero. gap: Optional[int] = None # How many rows page 1 rendered that this run could not use, because it # asked a different question than the page answers, or because the page @@ -209,7 +209,7 @@ def ok(self) -> bool: # three engines make it identically. What lives here is only HOW to ask this # particular driver. # -# The primitives are NAMED OPERATIONS rather than JavaScript (§1). Selenium's +# The primitives are NAMED OPERATIONS rather than JavaScript. Selenium's # execute_script takes a function BODY with an explicit `return` while # Playwright and pyppeteer take `() => expr`, so a shared module handing JS # across this boundary would quietly acquire one driver's dialect. @@ -238,7 +238,7 @@ def _scroll_to_bottom(page) -> None: To `document.body.scrollHeight` rather than by a fixed wheel distance: a fixed wheel stopped three rounds short of the bottom on a sibling site's 7,600px grid, so the lazy-load trigger was never reached and a run took - 30 of 50 cards while looking settled (§8). + 30 of 50 cards while looking settled. A real function object, not a string — see the module docstring on this site's CSP. @@ -261,7 +261,7 @@ def _page_height(page) -> Optional[int]: # SAME threshold in all three engines. A threshold that differed between them # would mean one engine reporting `complete` where its twins report `partial` # on the identical run, which is precisely the drift the shared modules exist -# to prevent (§6). +# to prevent. _SITE_HOST_FRAGMENTS = ("opensea.io",) @@ -320,7 +320,7 @@ def _proxy_failure(exc) -> str: Distinguishing this from an ordinary timeout matters because the two want opposite responses: a timeout deserves a retry from the same exit, while an unusable exit deserves a different one — retrying it unchanged just - spends the budget on a proxy that is not going to answer (§8). + spends the budget on a proxy that is not going to answer. """ text = str(exc) for marker in _PROXY_ERROR_MARKERS: @@ -335,7 +335,7 @@ def _launch_local(pw, args, pool): Factored out so a proxy rotation can tear the whole browser down and call it again. Swapping the proxy under a live session would be cheaper and wrong: cookies a bot manager issued against one exit, replayed from - another, are a stronger signal than either address alone (§8). + another, are a stronger signal than either address alone. """ launch_kwargs = {"headless": args.headless} proxy = to_playwright(pool.current) if pool else None @@ -387,7 +387,7 @@ def _launch_local(pw, args, pool): # inside that call, and `TurnstileTaskProxyless` needs all four. No static # read of the HTML, however careful, can produce a solvable task — so the # hook goes on the CONTEXT, before any page script runs, and covers every - # document including the one a redirect lands on (§19). + # document including the one a redirect lands on. # # Harmless where there is no Turnstile, which on this site is every page # measured so far. Installed anyway because the alternative is @@ -455,7 +455,7 @@ def _connect_remote(pw, args): # terminal, in CI output and in any log the run is piped to, which is # the one thing this project promises does not happen. The host and # port are KEPT: which endpoint failed is the useful half and is not - # the secret (§8). + # the secret. raise PWError( f"could not connect to --cdp-endpoint " f"{_mask_credentials(args.cdp_endpoint)}: " @@ -500,7 +500,7 @@ def _connect_remote(pw, args): # Matching GLOBALLY rather than once is the point: a Playwright connection # error repeats the endpoint five times, so a masker that handled only the # first occurrence would print the password four times and look like it was -# working (§8). +# working. _CREDENTIALS_IN_URL_RE = re.compile(r"([a-z][a-z0-9+.\-]*://)[^\s/@]+:[^\s/@]+@", re.IGNORECASE) @@ -541,7 +541,7 @@ def _content_when_settled(page, attempts: int = 4, pause_ms: int = 700): # A REAL FUNCTION OBJECT, not a string. opensea.io's CSP has no # `unsafe-eval`, so `page.evaluate("")` and # `wait_for_function` are both refused by the browser; a function passed like -# this goes through `Runtime.callFunctionOn` and is unaffected (§18). +# this goes through `Runtime.callFunctionOn` and is unaffected. # # Issued from INSIDE the page rather than from Python for three reasons, and # each of them is something a separate HTTP client would get wrong: @@ -556,7 +556,7 @@ def _content_when_settled(page, attempts: int = 4, pause_ms: int = 700): # with no second proxy configuration to keep in step. # # The body is built by `product_parser.graphql_body` and not here, so all -# three engines send byte-identical requests (§6). +# three engines send byte-identical requests. GRAPHQL_FETCH_JS = """ async ({endpoint, body}) => { try { @@ -584,7 +584,7 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): That last one matters more here than in most of the family — this endpoint answers HTTP 200 with an `errors` array rather than a 4xx, so a caller that only checks the status code reads a failure as an empty page - (§8, fail loudly). + (fail loudly). """ outcome = PageOutcome(page_num=page_num, url=session.page.url) body = graphql_body( @@ -593,7 +593,7 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): activity_filter=args.activity) # TWO INDEPENDENT BUDGETS, and keeping them independent is the whole - # point (§24). `attempt` counts TRANSPORT faults against `--retries`, + # point. `attempt` counts TRANSPORT faults against `--retries`, # which is the caller's allowance for things going wrong. A throttle is # not one of those: it is the site asking us to slow down, the exit is # fine, and waiting is the answer — so a throttle wait does not consume a @@ -657,8 +657,8 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): return outcome if page_flow.is_throttle(status): - # A THROTTLE, not a refusal, and the two want opposite responses - # (§24). Waiting at the SAME exit is the answer; rotating away + # A THROTTLE, not a refusal, and the two want opposite responses. + # Waiting at the SAME exit is the answer; rotating away # would spend the block budget and an exit on a page that was # about to come back. It has its own budget so that `--retries`, # which is the caller's allowance for transient faults, is not @@ -687,7 +687,7 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): if page_flow.is_refusal(status): # The endpoint refusing is a REFUSAL, not an empty page. Reported - # as blocked so the run's exit code says so (§8). + # as blocked so the run's exit code says so. logger.error("Page %d: the endpoint answered HTTP %d.", page_num, status) outcome.blocked_by = "endpoint-http-%d" % status @@ -708,13 +708,13 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): return outcome -def handle_captcha_if_present(page, args) -> bool: +def handle_captcha_if_present(page, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Runs after EVERY navigation, for ANY page. The static-HTML and runtime reCAPTCHA detectors are run and RECONCILED against each other rather than short-circuited, because they can disagree about the variant and the - parameters for one are rejected for the other (§8). + parameters for one are rejected for the other. WHAT IS AND IS NOT KNOWN HERE. No challenge was met in the captures taken for this repo, and the pages carry no captcha configuration at all: 0 @@ -724,7 +724,7 @@ def handle_captcha_if_present(page, args) -> bool: UNEXERCISED on opensea.io, and saying that plainly is the point — a sibling repo shipped a README sentence claiming a captcha here "cannot be solved", which was a statement about its own missing code dressed up as a - fact about a paid product, and it cost a release (§19). + fact about a paid product, and it cost a release. What IS true: the site sits behind Cloudflare, a managed challenge renders a Turnstile, and 2Captcha solves that with @@ -741,7 +741,7 @@ def handle_captcha_if_present(page, args) -> bool: # instant — which is why this check sits here rather than after the # readiness wait. The other way round would cost 25 wasted seconds on a # page the challenge genuinely gates, where solving FIRST is what makes - # the content appear (§8). + # the content appear. already_rendered = _count(page, _ready_selector(args)) when_blocked = getattr(args, "solve_captcha", "when-blocked") == "when-blocked" @@ -784,6 +784,12 @@ def handle_captcha_if_present(page, args) -> bool: logger.warning("No 2captcha API key, so this challenge cannot be " "solved — continuing with whatever the page holds.") return False + # The page's allowance is spent HERE, immediately before the purchase, + # so both call sites in _fetch_first_page and every block-retry share it. + if not budget.spend(): + logger.warning("This page has already used its %d paid solve(s) — " + "not buying another.", budget.limit) + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -861,14 +867,14 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: outcome = PageOutcome(page_num=1, url=url) has_pool = bool(pool and len(pool) > 1) # `RETRY_ON_BLOCKED` is CONSULTED, not merely documented — a policy - # constant nothing reads is the same defect as dead code (§17). + # constant nothing reads is the same defect as dead code. block_retries = 0 if not page_flow.RETRY_ON_BLOCKED else ( args.proxy_block_retries if has_pool else page_flow.BLOCK_RETRIES_WITHOUT_POOL) # Counted across the whole block-retry loop, not per attempt: a page that # keeps coming back as a challenge would otherwise buy one solve per # rotation, which is how a run quietly turns into a bill. - solves_bought = 0 + budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False status = None @@ -887,7 +893,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: # (net::ERR_PROXY_CONNECTION_FAILED), not PWTimeout — # catching only the latter lets it escape as a traceback, # which is the likeliest failure the first time anyone points - # --proxy-file at a real list (§8). + # --proxy-file at a real list. reason = _proxy_failure(e) if reason: exit_failed, load_failed = reason, True @@ -910,14 +916,14 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: if load_failed: break - if handle_captcha_if_present(session.page, args): + if handle_captcha_if_present(session.page, args, budget): session.page.wait_for_timeout(1000) html = _content_when_settled(session.page) or "" state = _classify(session.page, html, status, args.mode) # "Not painted yet" is not a fault, and telling it apart from one is - # the distinction §8 is about. Wait for the anchor and re-classify + # the distinction that matters here. Wait for the anchor and re-classify # BEFORE the retry decision — retrying a shell buys another shell. if page_flow.is_unpainted(state, html): wait_timeout = page_flow.content_timeout_ms(args.mode) @@ -936,14 +942,13 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: state = _classify(session.page, html, status, args.mode) if (page_flow.should_solve(state) - and solves_bought < page_flow.SOLVES_PER_PAGE): - solves_bought += 1 - if handle_captcha_if_present(session.page, args): + and budget.remaining()): + if handle_captcha_if_present(session.page, args, budget): session.page.wait_for_timeout(1000) html = _content_when_settled(session.page) or html state = _classify(session.page, html, status, args.mode) # The VERIFIED outcome, and the only one worth reporting: a - # "ready" task result is not evidence the token works (§19). + # "ready" task result is not evidence the token works. if state == "content": logger.info("The solve was accepted — page 1 is content " "now.") @@ -973,7 +978,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: # clears an edge challenge where a reload does not, which is # what `page_flow.RETRY_NEEDS_FRESH_CONTEXT` says — and a # constant no engine consulted would be dead policy dressed - # up as enforcement (§17). + # up as enforcement. # # NOT relaunched over --cdp-endpoint: a Scraping Browser # profile allows one live connection, so reconnecting risks @@ -1034,7 +1039,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: selector, threshold = _ready_selector(args), _min_matches(args, html) content_timeout = page_flow.content_timeout_ms(args.mode) # A POLL, not wait_for_function: this site's CSP has no - # `unsafe-eval` and would refuse an evaluated string outright (§18). + # `unsafe-eval` and would refuse an evaluated string outright. found = page_flow.wait_for_count( lambda sel: _count(session.page, sel), session.page.wait_for_timeout, selector, threshold, content_timeout) @@ -1049,7 +1054,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: # Dumping on success, not only on failure: a run can return the right # NUMBER of rows with a field silently unpopulated, and then the only way - # to tell a parsing bug from a too-early snapshot is the exact bytes (§9). + # to tell a parsing bug from a too-early snapshot is the exact bytes. if args.dump_html: dump_path = args.dump_html if args.pages == 1 else f"{args.dump_html}.page1" with open(dump_path, "w", encoding="utf-8") as f: @@ -1066,7 +1071,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: # TWO conditions, and the second is the one a site change breaks. # # `args.ssr_usable` is about the REQUEST: whether this run asks the same - # question the page renders (§21's ordering lesson). `state_answered` is + # question the page renders (the ordering decides which rows exist). `state_answered` is # about the RESPONSE: whether the page inlined its rows at all. # # Measured 2026-10-13: opensea.io stopped shipping `urql_transport` on @@ -1101,7 +1106,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: # is asking, and its cursor belongs to that other ordering. Dropped # rather than mixed in: two orderings in one file are two samples, # and `sort` in the sidecar could only be right about one of them - # (§21 — the ordering is a column, not a presentation detail). + # (the ordering is a column, not a presentation detail). outcome.ssr_dropped = len(rows) logger.info("%s", ssr_mismatch_note( args.mode, sort=args.sort, ranking=args.ranking, @@ -1119,7 +1124,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: except Exception as e: # noqa: BLE001 logger.warning("Could not capture screenshot: %s", e) if page_flow.parsed_nothing_from_a_served_page(html, len(rows)): - # §20: a page that was SERVED, links to N items and parses to + # A page that was SERVED, links to N items and parses to # zero rows is THIS PARSER'S bug, not an empty collection. # Reported as what it is, so a reader opens product_parser.py # instead of checking their URL for a typo. @@ -1142,7 +1147,7 @@ def _report_coverage(rows: List, page_num: int, mode: str) -> None: """Log what share of this page carries the columns the mode is for. Reported every time, not only when it looks wrong, so a consumer gets the - number rather than a threshold someone guessed (§13). + number rather than a threshold someone guessed. """ if not rows: return @@ -1185,7 +1190,7 @@ def scrape(args) -> int: concurrency = max(1, args.concurrency) if concurrency > 1: # Refused WITH the reason rather than quietly running one worker, - # which would look like the flag did something (§18). + # which would look like the flag did something. logger.warning("--concurrency %d is refused: %s.", concurrency, page_flow.concurrency_refusal(args.url, args.mode)) concurrency = 1 @@ -1274,7 +1279,7 @@ def scrape(args) -> int: blocked = outcome.blocked_by is not None break if not outcome.products: - # The data-side termination condition §7 asks for: a + # The data-side termination condition: a # page with no rows on it is the end of the feed, # whatever the cursor says. stop_reason = "no_new_products" @@ -1285,7 +1290,7 @@ def scrape(args) -> int: rank_seen += len(outcome.products) cursor = outcome.next_cursor - # The DATA-side half of the price boundary (§7). The + # The DATA-side half of the price boundary. The # cursor's null key is the cheap signal and it belongs to # the site's internals — it was an error in September and # pages fine in October. A page with not one priced row @@ -1314,7 +1319,7 @@ def scrape(args) -> int: # A run built ENTIRELY out of the DOM fallback is a degraded read, and # saying so is the whole point of carrying `data_source` as a column - # (§8: never present a guess as a fact). The rows are real and the + # (never present a guess as a fact). The rows are real and the # prices are not there at all, so this is neither a block nor an empty # result — it is a complete-looking file holding a worse answer, and it # gets its own stop reason so a reader lands on the parser rather than @@ -1356,7 +1361,7 @@ def scrape(args) -> int: "activity_filter": args.activity if args.mode == "activity" else None, "collection": args.collection_slug or None, "locale": locale_of(final_url), - # §21's arithmetic: what the site says the collection holds, beside + # What the site says the collection holds, beside # what this run read. `complete` means the walk finished, not that it # read the whole collection. "collection_totals": totals, @@ -1446,45 +1451,60 @@ def parse_args(): f"{DEFAULT_ACTIVITY_FILTER}). `sales` is the one most " f"callers want; `all` includes listings, offers, " f"transfers and mints.") - p.add_argument("--pages", type=int, default=1, - help=f"Pages to fetch (default 1, cap {PAGE_CAP}). Page 1 " - f"is the page itself and carries 50 rows; every page " - f"after it is a cursor request to the site's own " - f"endpoint and carries up to --limit. A run stops " - f"early, and reports COMPLETE, when the site hands " - f"back no cursor or when a price-ordered walk reaches " - f"the end of the listed items.") - p.add_argument("--limit", type=int, default=MAX_LIMIT, metavar="N", - help=f"Rows per cursor page (default and maximum " - f"{MAX_LIMIT} — the endpoint states that limit " - f"itself). Page 1 always carries the 50 the site " - f"server-renders, whatever this says.") + p.add_argument( + "--pages", + type=page_flow.number_at_least("--pages", 1), + default=1, + help=f"Pages to fetch (default 1, cap {PAGE_CAP}). Page 1 " + f"is the page itself and carries 50 rows; every page " + f"after it is a cursor request to the site's own " + f"endpoint and carries up to --limit. A run stops " + f"early, and reports COMPLETE, when the site hands " + f"back no cursor or when a price-ordered walk reaches " + f"the end of the listed items.") + p.add_argument( + "--limit", + type=page_flow.number_at_least("--limit", 1), + default=MAX_LIMIT, metavar="N", + help=f"Rows per cursor page (default and maximum " + f"{MAX_LIMIT} — the endpoint states that limit " + f"itself). Page 1 always carries the 50 the site " + f"server-renders, whatever this says.") p.add_argument("--category", default=None, help="Accepted for the family's shape and mapped onto " "--collection, because on OpenSea the thing a run is " "pointed at IS a collection.") - p.add_argument("--delay", type=float, default=1.0, - help="Delay between pages, seconds (default 1.0). The " - "endpoint publishes its own budget in an " - "`x-ratelimit-remaining` header (400 when this was " - "written), so this is politeness with a number behind " - "it rather than a guess.") - p.add_argument("--concurrency", type=int, default=1, metavar="N", - help="Accepted for the family's shape and REFUSED above 1, " - "with the reason: OpenSea paginates with an opaque " - "cursor, so page 5's request does not exist until " - "page 4 has been read and there is nothing to hand a " - "second worker. Run several collections at once, one " - "process each.") + p.add_argument( + "--delay", + type=page_flow.number_at_least("--delay", 0, float), + default=1.0, + help="Delay between pages, seconds (default 1.0). The " + "endpoint publishes its own budget in an " + "`x-ratelimit-remaining` header (400 when this was " + "written), so this is politeness with a number behind " + "it rather than a guess.") + p.add_argument( + "--concurrency", + type=page_flow.number_at_least("--concurrency", 1), + default=1, metavar="N", + help="Accepted for the family's shape and REFUSED above 1, " + "with the reason: OpenSea paginates with an opaque " + "cursor, so page 5's request does not exist until " + "page 4 has been read and there is nothing to hand a " + "second worker. Run several collections at once, one " + "process each.") p.add_argument("--retries", type=page_flow.attempts, default=3, help="Attempts per page load before giving up (default 3). " "The pause between attempts doubles each time. A page " "that comes back EMPTY is not retried — see " "page_flow.STATE_POLICY — because an empty feed is a " "correct answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, - help="Seconds before the first retry, doubling thereafter " - "(default 2.0)") + p.add_argument( + "--retry-delay", + type=page_flow.number_at_least("--retry-delay", 0, float), + default=2.0, + help="Seconds before the first retry, doubling thereafter " + "(default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="opensea_items", help="Output file prefix") @@ -1519,10 +1539,13 @@ def parse_args(): p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup, so concurrent runs do " "not all begin on the first exit in the file.") - p.add_argument("--proxy-block-retries", type=int, default=2, - help="When a page comes back refused, retry it from this " - "many OTHER exits before giving up (default 2). Needs " - "a pool of more than one; ignored otherwise.") + p.add_argument( + "--proxy-block-retries", + type=page_flow.number_at_least("--proxy-block-retries", 0), + default=2, + help="When a page comes back refused, retry it from this " + "many OTHER exits before giving up (default 2). Needs " + "a pool of more than one; ignored otherwise.") p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 rows were found. Off " @@ -1537,7 +1560,7 @@ def parse_args(): # ONE OS-family tag, not a list — and the default is what makes # --fingerprint work at all. It shipped as "Windows,Chrome,Desktop" across # this family, which the API rejects with HTTP 400, so --fingerprint - # failed on every invocation in four repos at once (§17). + # failed on every invocation in four repos at once. p.add_argument("--fp-tags", default="Windows", help="ONE OS-family tag for the fingerprint filter: " "Windows, Microsoft Windows or Android. NOT a list — " @@ -1564,7 +1587,7 @@ def parse_args(): "sitekey. No challenge was met on opensea.io while " "this was written, so the path is unexercised HERE — " "which is a fact about this repo's testing and not a " - "claim about what a solver can do (§19). A page " + "claim about what a solver can do. A page " "carrying no widget is reported unsolved rather than " "charged for.") p.add_argument("--min-score", type=float, default=0.7, @@ -1601,7 +1624,7 @@ def parse_args(): "`.pageN.json`, because what the parser saw " "there was a JSON body and not a document.") # HEADLESS by default, and that is this site's own measurement rather - # than the family's habit (§19 says to measure it once per site). + # than the family's habit (measure it once per site). p.add_argument("--headless", dest="headless", action="store_true", default=True, help="Run headless. THE DEFAULT, and measured identical to " @@ -1614,10 +1637,24 @@ def parse_args(): args = p.parse_args() # Fill --twocaptcha-key / --cdp-endpoint / --proxy / --url from the # environment or .env when the flag was not given. An explicit flag wins. + # What the user TYPED, captured before the environment fills anything in. + # An OPENSEA_URL from the environment or .env is a default for a bare + # run; it must never outrank --collection, --category or --mode + # collections, or the run silently reads the environment's collection. + typed_url = args.url env_config.apply(args) if args.category and not args.collection: args.collection = args.category + if not typed_url and args.url and (args.collection + or args.mode == "collections"): + logger.info("Ignoring OPENSEA_URL (%s): --collection/--category or " + "--mode collections says what to read.", args.url) + args.url = None + if (typed_url and args.collection + and collection_slug_from_url(typed_url) != args.collection): + p.error(f"--url {typed_url!r} and --collection {args.collection!r} " + f"name different things. Give one of the two.") if not args.url: if args.collection: mode = args.mode or "items" @@ -1639,7 +1676,7 @@ def parse_args(): # Refused rather than attempted. The parser's state reader, its path # patterns and its whole GraphQL vocabulary are this site's, so # pointing it at another marketplace would not fail loudly — it would - # return zero rows and look like an empty collection (§5). + # return zero rows and look like an empty collection. p.error(why) normalized = normalize_url(args.url) diff --git a/product_parser.py b/product_parser.py index 39bc574..89f44c7 100644 --- a/product_parser.py +++ b/product_parser.py @@ -12,7 +12,7 @@ header. The page you are looking at and the endpoint it calls hand back the SAME objects — the server-rendered state inlined in the HTML is literally an urql cache of those GraphQL responses — so ONE normaliser reads both routes -and they cannot drift (§21, which is where this pattern was learned). +and they cannot drift (which is where this pattern was learned). That is why this file has two entry points and one set of row builders: @@ -28,7 +28,7 @@ * **No product JSON-LD.** Measured 2026-09-17: a collection page carries three `application/ld+json` blocks and they are `BreadcrumbList`, `Brand` and `WebSite` — site navigation and a collection description, not items. - §4 says count the blocks before claiming a JSON-LD path, and counting them + The rule is to count the blocks before claiming a JSON-LD path, and counting them is what says there isn't one. The site's own inlined state is the primary path instead, and the URL pattern is the fallback. @@ -36,7 +36,7 @@ cursor-paginated: a response carries `nextPageCursor`, an opaque base64 string, and the next request passes it back as `after`. There is no address for page 2 of a collection, which is why `page_url()` returns None - and `--concurrency` above 1 is refused with the reason (§7, §18). + and `--concurrency` above 1 is refused with the reason. WHAT IT DOES HAVE THAT IS EASY TO GET WRONG ------------------------------------------- @@ -49,7 +49,7 @@ * **A price is a TOKEN amount, and the token is named.** `ETH`, `WETH`, `SOL`, `USDC`. It is never assumed: `_price()` reads the symbol the site - published and returns None when there is not one, which is §4's rule with + published and returns None when there is not one, which is the never-assume-a-currency rule with crypto in place of fiat. * **An EMPTY symbol is a real state, and it is common.** On the Bored Ape @@ -63,7 +63,7 @@ collection with nothing listed, but a ranking's rolling stats publish `floorPriceChange: 0` and `sales: 0` for a window in which nothing happened — those zeros are real measurements and stay. The nulls stay - null. (§21's "zero is not a rating", in the shape this site takes it.) + null. ("Zero is not a rating", in the shape this site takes it.) """ from __future__ import annotations @@ -114,7 +114,7 @@ # fallback. Anchored on the URL PATTERN and never on a class: OpenSea's # classes are Tailwind utilities and build hashes that change on every # deploy, while `/item/{chain}/…` is the address it publishes to search -# engines (§4). +# engines. SELECTORS = { "item_link": 'a[href*="/item/"]', "item_card": 'a[href*="/item/"]', @@ -245,7 +245,7 @@ def mode_for_url(url: str) -> Optional[str]: def unsupported_reason(url: str) -> Optional[str]: """Why this scraper refuses `url`, or None if it will take it. - Refused rather than attempted, and WITH the reason (§5). Every path + Refused rather than attempted, and WITH the reason. Every path pattern, every payload field name and the whole GraphQL vocabulary in this file are OpenSea's, so pointing this at another marketplace would not fail loudly — it would return zero rows and read as an empty @@ -352,7 +352,7 @@ def page_url(url: str, page_num: int) -> None: family has it and the smoke suite checks that the engines agree about what it returns. Returning None is the honest answer and the engines read it as "ask the site for the next cursor instead"; returning a made- - up `?page=N` would be the §7 failure in its purest form, since OpenSea + up `?page=N` would be the selector-only pagination failure in its purest form, since OpenSea answers an unknown query parameter with page 1 and a run built on it would report COMPLETE holding fifty rows over and over. """ @@ -423,7 +423,7 @@ def ssr_operations(html: Optional[str]) -> List[dict]: Returns [] rather than raising on anything malformed: a page that ships a payload this cannot read is a page the DOM fallback should get a turn - at, not a crash (§8 — but a failure that returns [] must be VISIBLE, + at, not a crash (but a failure that returns [] must be VISIBLE, which is what `parsed_nothing_from_a_served_page` in page_flow is for). """ if not html: @@ -483,7 +483,7 @@ def collection_totals(html: Optional[str]) -> Dict[str, Any]: A collection page ships its own `collectionBySlug.stats` beside the fifty items it renders — `totalSupply`, `listedItemCount`, its floor and its - volume. That is the arithmetic §21 calls for: a run of three pages is + volume. That is the arithmetic that matters: a run of three pages is genuinely `complete` under the ordering it used AND is 300 rows of a 9,998-item collection, and a sidecar that says only "complete" is lying by omission. The engines put this in `extra` and log the percentage. @@ -521,7 +521,7 @@ def count_cards(html: Optional[str]) -> int: Counts DISTINCT addresses, not anchors: a rendered tile links to its item twice (the image and the name), so 192 `/item/` anchors on a collection page are 32 items. The family learned that counting links instead of ids - is what lets a tile-scoping walk steal its neighbour's data (§4), and the + is what lets a tile-scoping walk steal its neighbour's data, and the same arithmetic error would make a readiness threshold meaningless here. """ if not html: @@ -535,7 +535,7 @@ def count_cards(html: Optional[str]) -> int: # --------------------------------------------------------------------------- # Was this page served by OpenSea at all? # --------------------------------------------------------------------------- -# The structural test §8 and §18 both arrive at: a page the site really +# The structural test: a page the site really # served is BUILT OUT OF ITS OWN ASSETS, and an interstitial — or Chromium's # own network-error page, which carries `opensea.io` and would # fool any title check — is not. @@ -547,8 +547,8 @@ def count_cards(html: Optional[str]) -> int: # seadn.io 309 to 634 # # Note the 404 is in that list and passes, which is correct: OpenSea's "not -# found" IS an OpenSea page, and it is `empty` rather than `blocked` (§8 — -# blocked is not empty). +# found" IS an OpenSea page, and it is `empty` rather than `blocked` (blocked +# is not empty). _SITE_ASSET_MARKERS = ("/_next/static/", "seadn.io", "i2c.seadn.io") _SITE_ASSET_FLOOR = 2 @@ -562,7 +562,7 @@ def served_by_opensea(html: Optional[str]) -> bool: # Vendor markers for a challenge. EVERY ONE OF THESE WAS COUNTED ON PAGES -# KNOWN TO BE GOOD FIRST (§18), on 2026-09-17, across the homepage, a +# KNOWN TO BE GOOD FIRST, on 2026-09-17, across the homepage, a # collection page, an item page, a Solana collection page, a Japanese-locale # collection page, the ranking and the site's own 404: # @@ -588,11 +588,11 @@ def served_by_opensea(html: Optional[str]) -> bool: # So the obvious marker fires on a perfectly good 1.26 MB page holding the # full catalogue, and the one that works is absent from it. Carrying # `cf-turnstile` would report exit 3 on every page fetched over the paid -# path — which is exactly what happened to a sibling repo's first live run -# (§8, §19). `smoke_test.py` pins this against a real Scraping Browser +# path — which is exactly what happened to a sibling repo's first live run. +# `smoke_test.py` pins this against a real Scraping Browser # capture, because a sibling shipped the same check and it passed for the # wrong reason: it ran only against pages fetched with a plain client, which -# carry no injection at all (§21). +# carry no injection at all. # # With it gone, nothing in this set matches anything that extension injects, # so this repo needs no extension-tag strip either; adding one would be dead @@ -615,7 +615,7 @@ def served_by_opensea(html: Optional[str]) -> bool: # entity-escape the punctuation in its own markers — `errors.edgesuite.net` # reaching a raw HTTP client and `errors.edgesuite.net` reaching a browser # DOM, which is one marker with two spellings and a check that silently -# covers only one of them (§20). Unescaping a bounded PREFIX handles both +# covers only one of them. Unescaping a bounded PREFIX handles both # without spending the cost on two megabytes of grid, and without a product # name deep in a collection description reading as a marker. _MARKER_SCAN_BYTES = 200_000 @@ -648,7 +648,7 @@ def is_challenge_page(html: Optional[str]) -> bool: # Chromium's own network-error page. Not an interstitial and not the site: it # carries `opensea.io`, so a title check calls it a real page, # and the only thing that reads it correctly is asking whether the document -# was built out of the site's assets (§18, proven on a third site here). +# was built out of the site's assets (proven on a third site here). _BROWSER_ERROR_MARKERS = ("ERR_PROXY_CONNECTION_FAILED", "ERR_TUNNEL_CONNECTION_FAILED", "ERR_NAME_NOT_RESOLVED", @@ -665,12 +665,12 @@ def is_browser_error_page(html: Optional[str]) -> bool: # The site's own way of saying a feed has nothing in it. An UNAMBIGUOUS # positive signal — no interstitial carries it — so it is checked before any -# threshold (§17's classification-order trap). +# threshold (order signals by how much they prove). # -# WHAT IS DELIBERATELY NOT IN THIS SET, and it is the §18 trap caught in the +# WHAT IS DELIBERATELY NOT IN THIS SET, and it is the marker trap caught in the # act. OpenSea's "not found" page reads `404: This page could not be found.`, # which looks like the obvious marker for an unknown collection slug. Counted -# first on pages known to be good, as §18 requires: that sentence appears +# first on pages known to be good, as every marker must be: that sentence appears # TWICE on every served page measured — the collection page, the ranking, an # item page, a Solana collection and a Japanese-locale page alike — because # Next.js ships its bundled not-found component inside every RSC payload. @@ -698,7 +698,7 @@ def detect_page_state(html: Optional[str], status: Optional[int] = None, """Which of the five states this response is. Ordered by how much each signal PROVES rather than by how cheap it is to - check (§17): an inlined payload naming rows is proof the site served + check: an inlined payload naming rows is proof the site served this page, and it is checked before any status code or threshold. The heuristic — "was this built out of OpenSea's assets?" — is last among the positives, because a minimal real page can carry fewer asset references @@ -871,7 +871,7 @@ def detect_page_state(html: Optional[str], status: Optional[int] = None, # handed over is not even valid for the other ordering: a `--sort created` # run passing the page's price cursor was answered `Invalid cursor` and # reported partial (measured on mad-lads, 2026-09-17, which is exactly the -# second-collection run §15 asks for). +# second-collection run a new site needs). # # So the engines compare, and where the request does not match the page they # take page 1 as proof-of-service and totals only, then start the feed from @@ -930,7 +930,7 @@ def graphql_body(mode: str, *, slug: str = "", limit: int = MAX_LIMIT, Built here rather than in the engines so all three send byte-identical requests — a query that differed between engines would mean one of them - reporting columns its twins cannot (§6). + reporting columns its twins cannot. """ if mode not in QUERIES: raise ValueError(f"no GraphQL query for mode {mode!r}") @@ -958,7 +958,7 @@ def graphql_errors(payload: Any) -> List[str]: The endpoint answers HTTP 200 with an `errors` array rather than a 4xx, so a caller that only checks the status code reads a failure as an empty - page. That is exactly the "fail loudly" case (§8). + page. That is exactly the "fail loudly" case. """ if not isinstance(payload, dict): return [] @@ -993,7 +993,7 @@ def graphql_data(payload: Any, mode: str = "items") -> Optional[dict]: # first element becomes `null`. Recognising it here lets a floor scrape stop # where the prices stop, which is what a price-ordered walk is for. # -# WHAT THIS USED TO SAY, AND WHY THE CORRECTION IS LEFT HERE (§13). Measured +# WHAT THIS USED TO SAY, AND WHY THE CORRECTION IS LEFT HERE. Measured # on boredapeyachtclub 2026-09-17: three pages of 100, 282 of the 300 # listed, then a null key — and passing that cursor back answered # `{"errors":[{"message":"Something went wrong"}]}`. The stop was written as @@ -1004,7 +1004,7 @@ def graphql_data(payload: Any, mode: str = "items") -> Optional[dict]: # rows deep with no error, and pages 4 to 10 — seven hundred rows — carried # zero priced items. OpenSea fixed their end. The stop is unchanged because # the fact it stood for is unchanged, but it no longer rests on an error -# message: `page_flow.prices_ran_out` reads the rows instead, which is §7's +# message: `page_flow.prices_ran_out` reads the rows instead, on the # rule that a terminating condition belongs to the catalogue rather than to # the transport. A captured response for both behaviours is pinned in the # fixtures under `_endpoint_edges`. @@ -1029,7 +1029,7 @@ def _clean(value: Any) -> Any: OpenSea publishes `"symbol": ""` for a token it has not named — 11 of 50 last sales on one capture — and an empty string written into a currency - column reads as a currency (§8: never present a guess as a fact, and an + column reads as a currency (never present a guess as a fact, and an empty string is a guess that something was there). """ if isinstance(value, str) and not value.strip(): @@ -1111,7 +1111,7 @@ def item_row(node: Any, *, url_locale: Optional[str] = None, The same object shape comes from `collectionItems.items[]`, from `itemByIdentifier` on a single item's page, and from the inlined state of - either — which is why there is one builder rather than three (§21). + either — which is why there is one builder rather than three. """ if not isinstance(node, dict): return None @@ -1281,7 +1281,7 @@ def rows_from_nodes(nodes: Sequence[Any], mode: str, *, url: str = "", `page` and `first_position` are threaded rather than defaulted, because `position` restarts at 1 on every page: without the page number beside it a row from page 2 claims the same position as one from page 1 and the two - are indistinguishable in the output (§18). + are indistinguishable in the output. """ builder = _ROW_BUILDER.get(mode) if builder is None or not isinstance(nodes, (list, tuple)): @@ -1311,7 +1311,7 @@ def rows_from_graphql(payload: Any, url: str = "", *, mode: str = "items", Raises nothing on an error response — the caller reads `graphql_errors` for that and decides. Returning [] here and letting the caller log the messages keeps "the endpoint said no" distinguishable from "the feed is - empty", which is the whole of §8's blocked/empty/partial distinction in + empty", which is the whole of the blocked/empty/partial distinction in one function. """ data = graphql_data(payload, mode) @@ -1335,7 +1335,7 @@ def rows_from_dom(html: str, url: str, *, mode: str = "items", page: int = 1, Runs ONLY when the inlined state did not parse, and it is deliberately thin: a rendered OpenSea tile carries an address and a name and no machine-readable price — the prices in the markup are formatted display - text inside build-hashed elements, which is exactly what §4 says never to + text inside build-hashed elements, which is exactly what a parser should never anchor on. So these rows carry what the URL proves and leave every price null, with `data_source="dom"` saying so. @@ -1398,7 +1398,7 @@ def parse_rows(html: str, url: str, *, page: int = 1, mode: str = "items", first_rank: Optional[int] = None) -> List[Any]: """Every row on this page, inlined state first and the DOM as fallback. - The order is the one §4 prescribes with the site's own data in place of + The order is structured data first, with the site's own data in place of JSON-LD: the structured source carries every row regardless of what has painted, and the rendered markup is what is left when it does not parse. """ diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index f24316b..3b5675c 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -19,7 +19,7 @@ coroutines on a private event loop through `_AsyncBridge` and hands `page_flow` the same plain synchronous callables its twins do. The bridge also gives every call an explicit, enforced timeout, which pyppeteer's own -API does not offer (§8: every remote call is bounded). +API does not offer (every remote call is bounded). Example ------- @@ -215,7 +215,7 @@ def _install_turnstile_intercept(session) -> None: and chlPageData that live only inside that call. pyppeteer spells it `evaluateOnNewDocument`; the other two engines spell the same thing `context.add_init_script` and `Page.addScriptToEvaluateOnNewDocument`, - which is why this cannot live in the shared module (§1). + which is why this cannot live in the shared module. """ try: session.bridge.run( @@ -264,7 +264,7 @@ def open(self): exit_url = self.pool.current # Credentials go through page.authenticate(), never onto the # command line: --proxy-server= becomes part of the browser's - # argv, readable by anything that can run `ps` (§8). + # argv, readable by anything that can run `ps`. scrubbed, credentials = split_credentials(exit_url) launch_args.append(f"--proxy-server={scrubbed}") logger.info("Using proxy exit %s", mask(exit_url)) @@ -307,7 +307,7 @@ def _apply_fingerprint(self): format and at `data.ua` in the other, and a key that exists in neither makes `--fingerprint` silently set no user agent at all — which defeats the flag rather than breaking it, and was live in four - sibling repos at once (§16). + sibling repos at once. The init script is the same one the Playwright engine installs on its context. Shared deliberately: two engines applying different halves @@ -354,7 +354,7 @@ def close(self): # --------------------------------------------------------------------------- # Only "how to ask this driver" lives here. pyppeteer takes `() => expr` like # Playwright and unlike Selenium, which is exactly why page_flow names -# operations rather than passing JavaScript across the boundary (§1). +# operations rather than passing JavaScript across the boundary. def _count(session, selector: str) -> int: try: return len(session.bridge.run(session.page.querySelectorAll(selector))) @@ -372,7 +372,7 @@ def _sleep(ms: int) -> None: # DOCUMENT's status, which is the only unambiguous signal that an unknown # collection slug is `empty` rather than a `shell` that has not painted — # without it this engine would record a different state than its twins on the -# identical URL (§6). +# identical URL. _SITE_HOST_FRAGMENTS = ("opensea.io",) @@ -452,7 +452,7 @@ def _min_matches(args, html: str = "") -> int: def _classify(session, html: str, status=None, mode: str = "items") -> str: # `status` POSITIONAL and second, matching the other two engines and the - # callee's real signature (§17). + # callee's real signature. return page_flow.classify(html, status, _current_url(session), mode) @@ -478,7 +478,7 @@ def _proxy_failure(exc) -> str: # The cursor pages. pyppeteer takes an async arrow function like Playwright # does; the request, the endpoint and the body are identical across all three # engines, and the body is built by `product_parser.graphql_body` so the -# requests are byte-identical (§6). +# requests are byte-identical. # # Issued from INSIDE the page: same-origin cookies (including Cloudflare's # `__cf_bm`), the connection the page already has, and the same exit as the @@ -507,7 +507,7 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): Never raises for an EXPECTED failure. This endpoint answers HTTP 200 with an `errors` array rather than a 4xx, so a caller that only checks the - status code reads a failure as an empty page (§8, fail loudly). + status code reads a failure as an empty page (fail loudly). """ outcome = PageOutcome(page_num=page_num, url=_current_url(session)) body = graphql_body( @@ -516,7 +516,7 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): activity_filter=args.activity) # TWO INDEPENDENT BUDGETS, and keeping them independent is the whole - # point (§24). `attempt` counts TRANSPORT faults against `--retries`, + # point. `attempt` counts TRANSPORT faults against `--retries`, # which is the caller's allowance for things going wrong. A throttle is # not one of those: it is the site asking us to slow down, the exit is # fine, and waiting is the answer — so a throttle wait does not consume a @@ -576,8 +576,8 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): return outcome if page_flow.is_throttle(status): - # A THROTTLE, not a refusal, and the two want opposite responses - # (§24). Waiting at the SAME exit is the answer; rotating away + # A THROTTLE, not a refusal, and the two want opposite responses. + # Waiting at the SAME exit is the answer; rotating away # would spend the block budget and an exit on a page that was # about to come back. Its own budget, so that `--retries` — the # caller's allowance for transient faults — is not quietly @@ -625,7 +625,7 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): return outcome -def handle_captcha_if_present(session, args) -> bool: +def handle_captcha_if_present(session, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Identical policy to the other two engines — see @@ -672,6 +672,12 @@ def _evaluate(js): logger.warning("No 2captcha API key, so this challenge cannot be " "solved — continuing with whatever the page holds.") return False + # The page's allowance is spent HERE, immediately before the purchase, + # so both call sites in _fetch_first_page and every block-retry share it. + if not budget.spend(): + logger.warning("This page has already used its %d paid solve(s) — " + "not buying another.", budget.limit) + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -737,7 +743,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: block_retries = 0 if not page_flow.RETRY_ON_BLOCKED else ( args.proxy_block_retries if has_pool else page_flow.BLOCK_RETRIES_WITHOUT_POOL) - solves_bought = 0 + budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False status = None @@ -774,7 +780,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: if load_failed: break - if handle_captcha_if_present(session, args): + if handle_captcha_if_present(session, args, budget): time.sleep(1.0) html = _content(session) or "" @@ -797,9 +803,8 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: state = _classify(session, html, status, args.mode) if (page_flow.should_solve(state) - and solves_bought < page_flow.SOLVES_PER_PAGE): - solves_bought += 1 - if handle_captcha_if_present(session, args): + and budget.remaining()): + if handle_captcha_if_present(session, args, budget): time.sleep(1.0) html = _content(session) or html state = _classify(session, html, status, args.mode) @@ -903,7 +908,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: outcome.totals = page_flow.collection_totals_on_page(html or "") or None # TWO conditions, and the second is the one a site change breaks. # `args.ssr_usable` is about the REQUEST (does this run ask the question - # the page renders, §21); `state_answered` is about the RESPONSE (did the + # the page renders); `state_answered` is about the RESPONSE (did the # page inline its rows at all). Measured 2026-10-13: opensea.io stopped # shipping `urql_transport` on the ranking and the activity feed, and # without the second condition the run kept page 1's DOM-fallback rows — @@ -922,7 +927,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: "a fault here.", args.mode, len(rows)) else: # The rows page 1 rendered answer a DIFFERENT question than this run - # is asking, and its cursor belongs to that other ordering (§21). + # is asking, and its cursor belongs to that other ordering. outcome.ssr_dropped = len(rows) logger.info("%s", ssr_mismatch_note( args.mode, sort=args.sort, ranking=args.ranking, @@ -1071,7 +1076,7 @@ def scrape(args) -> int: rank_seen += len(outcome.products) cursor = outcome.next_cursor - # The DATA-side half of the price boundary (§7). The + # The DATA-side half of the price boundary. The # cursor's null key is the cheap signal and it belongs to # the site's internals — an error in September, paging fine # in October. A page with not one priced row is a fact about @@ -1098,7 +1103,7 @@ def scrape(args) -> int: # A run built ENTIRELY out of the DOM fallback is a degraded read, and # saying so is the whole point of carrying `data_source` as a column - # (§8: never present a guess as a fact). The rows are real and the + # (never present a guess as a fact). The rows are real and the # prices are not there at all, so this is neither a block nor an empty # result — it is a complete-looking file holding a worse answer, and it # gets its own stop reason so a reader lands on the parser rather than @@ -1201,26 +1206,41 @@ def parse_args(): default=DEFAULT_ACTIVITY_FILTER, help=f"Which events --mode activity keeps (default " f"{DEFAULT_ACTIVITY_FILTER}).") - p.add_argument("--pages", type=int, default=1, - help=f"Pages to fetch (default 1, cap {PAGE_CAP}).") - p.add_argument("--limit", type=int, default=MAX_LIMIT, metavar="N", - help=f"Rows per cursor page (default and maximum " - f"{MAX_LIMIT} — the endpoint states that limit " - f"itself).") + p.add_argument( + "--pages", + type=page_flow.number_at_least("--pages", 1), + default=1, + help=f"Pages to fetch (default 1, cap {PAGE_CAP}).") + p.add_argument( + "--limit", + type=page_flow.number_at_least("--limit", 1), + default=MAX_LIMIT, metavar="N", + help=f"Rows per cursor page (default and maximum " + f"{MAX_LIMIT} — the endpoint states that limit " + f"itself).") p.add_argument("--category", default=None, help="Accepted for the family's shape and mapped onto " "--collection.") - p.add_argument("--delay", type=float, default=1.0, - help="Delay between pages, seconds (default 1.0).") - p.add_argument("--concurrency", type=int, default=1, metavar="N", - help="Accepted for the family's shape and REFUSED above 1, " - "with the reason: page 5's request does not exist " - "until page 4 has been read.") + p.add_argument( + "--delay", + type=page_flow.number_at_least("--delay", 0, float), + default=1.0, + help="Delay between pages, seconds (default 1.0).") + p.add_argument( + "--concurrency", + type=page_flow.number_at_least("--concurrency", 1), + default=1, metavar="N", + help="Accepted for the family's shape and REFUSED above 1, " + "with the reason: page 5's request does not exist " + "until page 4 has been read.") p.add_argument("--retries", type=page_flow.attempts, default=3, help="Attempts per page load before giving up (default 3).") - p.add_argument("--retry-delay", type=float, default=2.0, - help="Seconds before the first retry, doubling thereafter " - "(default 2.0)") + p.add_argument( + "--retry-delay", + type=page_flow.number_at_least("--retry-delay", 0, float), + default=2.0, + help="Seconds before the first retry, doubling thereafter " + "(default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="opensea_items", help="Output file prefix") @@ -1248,9 +1268,12 @@ def parse_args(): "re-navigation.") p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup.") - p.add_argument("--proxy-block-retries", type=int, default=2, - help="When a page comes back refused, retry it from this " - "many OTHER exits before giving up (default 2).") + p.add_argument( + "--proxy-block-retries", + type=page_flow.number_at_least("--proxy-block-retries", 0), + default=2, + help="When a page comes back refused, retry it from this " + "many OTHER exits before giving up (default 2).") p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 rows were found.") @@ -1274,7 +1297,7 @@ def parse_args(): "challenge was met on opensea.io while this was " "written, so the path is unexercised HERE — a fact " "about this repo's testing and not a claim about what " - "a solver can do (§19).") + "a solver can do.") p.add_argument("--min-score", type=float, default=0.7, help="reCAPTCHA v3 minimum score to request.") p.add_argument("--cdp-endpoint", default=None, @@ -1304,10 +1327,24 @@ def parse_args(): p.add_argument("--headful", dest="headless", action="store_false", help="Run with a real browser window. Needs a display.") args = p.parse_args() + # What the user TYPED, captured before the environment fills anything in. + # An OPENSEA_URL from the environment or .env is a default for a bare + # run; it must never outrank --collection, --category or --mode + # collections, or the run silently reads the environment's collection. + typed_url = args.url env_config.apply(args) if args.category and not args.collection: args.collection = args.category + if not typed_url and args.url and (args.collection + or args.mode == "collections"): + logger.info("Ignoring OPENSEA_URL (%s): --collection/--category or " + "--mode collections says what to read.", args.url) + args.url = None + if (typed_url and args.collection + and collection_slug_from_url(typed_url) != args.collection): + p.error(f"--url {typed_url!r} and --collection {args.collection!r} " + f"name different things. Give one of the two.") if not args.url: if args.collection: mode = args.mode or "items" diff --git a/requirements-playwright.txt b/requirements-playwright.txt index 2d19dbb..decccef 100644 --- a/requirements-playwright.txt +++ b/requirements-playwright.txt @@ -11,7 +11,7 @@ # # A sibling repo in this family needs `playwright install chrome` instead, # because ITS site answered the bundled Chromium with HTTP 429 and real -# Chrome with 200. That is a fact about that site, not about this one (§13), +# Chrome with 200. That is a fact about that site, not about this one, # which is why it was re-measured here rather than copied. `--browser-channel # chrome` remains available if you want a real Chrome anyway. playwright>=1.40.0 diff --git a/scraper_api_client.py b/scraper_api_client.py index bd98229..a818914 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -65,6 +65,7 @@ BOT_CHALLENGE_MARKERS) from output_writer import save, ROW_CLASS_BY_MODE import env_config +import page_flow logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("scraper_api_client") @@ -336,8 +337,11 @@ def parse_args(): p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="opensea_items_scraperapi", help="Output file prefix") - p.add_argument("--timeout", type=int, default=60, - help=f"API-side task timeout in seconds (1-{MAX_API_TIMEOUT}, default 60)") + p.add_argument( + "--timeout", + type=page_flow.number_at_least("--timeout", 1), + default=60, + help=f"API-side task timeout in seconds (1-{MAX_API_TIMEOUT}, default 60)") p.add_argument("--cdp-url", default=None, help="Route the fetch through an existing browser session over CDP " "(sent as the API's `cdpurl` param), e.g. ws://user:pass@host:port") @@ -353,12 +357,18 @@ def parse_args(): p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 products were parsed. Off by " "default so a failed fetch can't overwrite a good result.") - p.add_argument("--retries", type=int, default=1, - help="Extra attempts if a bot-challenge page comes back. One retry is " - "usually worth it. Each attempt is a separate billable task, so " - "this defaults to 1.") - p.add_argument("--retry-delay", type=int, default=10, - help="Seconds between retries (default 10)") + p.add_argument( + "--retries", + type=page_flow.number_at_least("--retries", 0), + default=1, + help="Extra attempts if a bot-challenge page comes back. One retry is " + "usually worth it. Each attempt is a separate billable task, so " + "this defaults to 1.") + p.add_argument( + "--retry-delay", + type=page_flow.number_at_least("--retry-delay", 0), + default=10, + help="Seconds between retries (default 10)") p.add_argument("--dump-html", default=None, help="Also write the raw returned HTML to this path (always, even on success)") args = p.parse_args() diff --git a/selenium_scraper.py b/selenium_scraper.py index 1395677..6967c5f 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -100,7 +100,7 @@ def _mask_credentials(text: str) -> str: Global, not first-match: an error can repeat an endpoint several times, and a masker that handles one occurrence prints the password for the rest - while looking like it works (§8). + while looking like it works. """ return _CREDENTIALS_IN_URL_RE.sub(r"\1***:***@", text or "") @@ -163,7 +163,7 @@ class _Session: Same contract as the Playwright engine's _BrowserSession, including the rule that a rotation means a genuinely FRESH browser: cookies a bot manager issued against one exit, replayed from another, are a stronger - signal than either address alone (§8). + signal than either address alone. """ def __init__(self, args, pool): @@ -179,7 +179,7 @@ def open(self): options.debugger_address) # No UA, no proxy, no fingerprint on this path: the remote browser # brings its own, and stacking a second creates a contradiction - # rather than better cover (§8). + # rather than better cover. self.driver = webdriver.Chrome(options=options) self._apply_timeouts() self._install_turnstile_intercept() @@ -217,7 +217,7 @@ def open(self): # It has to be asked for at driver creation; there is no way to turn # it on later. Without it this engine could not tell a real 404 from # a shell, and would report `shell` where its twins report `empty` on - # the identical URL (§6). + # the identical URL. options.set_capability("goog:loggingPrefs", {"performance": "ALL"}) self.driver = webdriver.Chrome(options=options) @@ -247,7 +247,7 @@ def _apply_fingerprint(self): # shape here: the UA lives at `userAgent.userAgent` in one format and # at `data.ua` in the other, and a key that exists in neither makes # `--fingerprint` silently set no user agent at all — which defeats - # the flag rather than breaking it (§16). + # the flag rather than breaking it. from fingerprint_client import (get_fingerprint, fingerprint_user_agent, playwright_init_script) fp = get_fingerprint(self.args.twocaptcha_key, tags=self.args.fp_tags, @@ -276,7 +276,7 @@ def _install_turnstile_intercept(self): Selenium spells it `Page.addScriptToEvaluateOnNewDocument` over CDP; the other two engines spell the same thing `add_init_script` and `evaluateOnNewDocument`, which is why this cannot live in the shared - module (§1). + module. """ try: self.driver.execute_cdp_cmd( @@ -308,7 +308,7 @@ def close(self): # Only "how to ask this driver" lives here. Note the JS dialect: Selenium's # execute_script runs a function BODY and needs an explicit `return`, unlike # the `() => expr` both other engines take — which is exactly why page_flow -# names operations instead of passing JavaScript across the boundary (§1). +# names operations instead of passing JavaScript across the boundary. def _count(session, selector: str) -> int: try: return len(session.driver.find_elements(By.CSS_SELECTOR, selector)) @@ -354,7 +354,7 @@ def _page_height(session) -> Optional[int]: # 404, and that status is the only unambiguous signal that a page is `empty` # rather than a `shell` that has not painted — without it this engine would # spend a 25-second readiness wait on every missing collection and record a -# different state than its twins on the identical URL (§6). +# different state than its twins on the identical URL. # # The log DRAINS ON READ, so both facts are collected in one pass and cached # on the session; two independent readers would each see half the entries. @@ -424,7 +424,7 @@ def _current_url(session) -> str: def _classify(session, html: str, status=None, mode: str = "items") -> str: # `status` is POSITIONAL and second. Two engines in a sibling repo passed - # it as a keyword and both crashed on their first fetch (§17); this + # it as a keyword and both crashed on their first fetch; this # repo's smoke suite binds every shared-module call in every engine # against the callee's real signature for that reason. return page_flow.classify(html, status, _current_url(session), mode) @@ -454,7 +454,7 @@ def _proxy_failure(exc) -> str: # argument is the completion callback. The other two engines pass an async # arrow function; the request, the endpoint and the body are identical, and # the body itself is built by `product_parser.graphql_body` so all three send -# byte-identical requests (§6). +# byte-identical requests. # # Issued from INSIDE the page for the same three reasons as in the primary # engine: same-origin cookies (including Cloudflare's `__cf_bm`), the @@ -484,7 +484,7 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): Never raises for an EXPECTED failure. This endpoint answers HTTP 200 with an `errors` array rather than a 4xx, so a caller that only checks the - status code reads a failure as an empty page (§8, fail loudly). + status code reads a failure as an empty page (fail loudly). """ outcome = PageOutcome(page_num=page_num, url=_current_url(session)) body = graphql_body( @@ -493,7 +493,7 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): activity_filter=args.activity) # TWO INDEPENDENT BUDGETS, and keeping them independent is the whole - # point (§24). `attempt` counts TRANSPORT faults against `--retries`, + # point. `attempt` counts TRANSPORT faults against `--retries`, # which is the caller's allowance for things going wrong. A throttle is # not one of those: it is the site asking us to slow down, the exit is # fine, and waiting is the answer — so a throttle wait does not consume a @@ -552,8 +552,8 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): return outcome if page_flow.is_throttle(status): - # A THROTTLE, not a refusal, and the two want opposite responses - # (§24). Waiting at the SAME exit is the answer; rotating away + # A THROTTLE, not a refusal, and the two want opposite responses. + # Waiting at the SAME exit is the answer; rotating away # would spend the block budget and an exit on a page that was # about to come back. Its own budget, so that `--retries` — the # caller's allowance for transient faults — is not quietly @@ -601,7 +601,7 @@ def _graphql_page(session, args, cursor: Optional[str], page_num: int): return outcome -def handle_captcha_if_present(session, args) -> bool: +def handle_captcha_if_present(session, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Identical policy to the other two engines — see @@ -651,6 +651,12 @@ def handle_captcha_if_present(session, args) -> bool: logger.warning("No 2captcha API key, so this challenge cannot be " "solved — continuing with whatever the page holds.") return False + # The page's allowance is spent HERE, immediately before the purchase, + # so both call sites in _fetch_first_page and every block-retry share it. + if not budget.spend(): + logger.warning("This page has already used its %d paid solve(s) — " + "not buying another.", budget.limit) + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -713,7 +719,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: block_retries = 0 if not page_flow.RETRY_ON_BLOCKED else ( args.proxy_block_retries if has_pool else page_flow.BLOCK_RETRIES_WITHOUT_POOL) - solves_bought = 0 + budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False status = None @@ -748,7 +754,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: if load_failed: break - if handle_captcha_if_present(session, args): + if handle_captcha_if_present(session, args, budget): time.sleep(1.0) html = _content(session) or "" @@ -771,9 +777,8 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: state = _classify(session, html, status, args.mode) if (page_flow.should_solve(state) - and solves_bought < page_flow.SOLVES_PER_PAGE): - solves_bought += 1 - if handle_captcha_if_present(session, args): + and budget.remaining()): + if handle_captcha_if_present(session, args, budget): time.sleep(1.0) html = _content(session) or html state = _classify(session, html, status, args.mode) @@ -878,7 +883,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: outcome.totals = page_flow.collection_totals_on_page(html or "") or None # TWO conditions, and the second is the one a site change breaks. # `args.ssr_usable` is about the REQUEST (does this run ask the question - # the page renders, §21); `state_answered` is about the RESPONSE (did the + # the page renders); `state_answered` is about the RESPONSE (did the # page inline its rows at all). Measured 2026-10-13: opensea.io stopped # shipping `urql_transport` on the ranking and the activity feed, and # without the second condition the run kept page 1's DOM-fallback rows — @@ -897,7 +902,7 @@ def _fetch_first_page(session, args, pool) -> PageOutcome: "a fault here.", args.mode, len(rows)) else: # The rows page 1 rendered answer a DIFFERENT question than this run - # is asking, and its cursor belongs to that other ordering (§21). + # is asking, and its cursor belongs to that other ordering. outcome.ssr_dropped = len(rows) logger.info("%s", ssr_mismatch_note( args.mode, sort=args.sort, ranking=args.ranking, @@ -1042,7 +1047,7 @@ def scrape(args) -> int: rank_seen += len(outcome.products) cursor = outcome.next_cursor - # The DATA-side half of the price boundary (§7). The + # The DATA-side half of the price boundary. The # cursor's null key is the cheap signal and it belongs to # the site's internals — an error in September, paging fine # in October. A page with not one priced row is a fact about @@ -1067,7 +1072,7 @@ def scrape(args) -> int: # A run built ENTIRELY out of the DOM fallback is a degraded read, and # saying so is the whole point of carrying `data_source` as a column - # (§8: never present a guess as a fact). The rows are real and the + # (never present a guess as a fact). The rows are real and the # prices are not there at all, so this is neither a block nor an empty # result — it is a complete-looking file holding a worse answer, and it # gets its own stop reason so a reader lands on the parser rather than @@ -1170,26 +1175,41 @@ def parse_args(): default=DEFAULT_ACTIVITY_FILTER, help=f"Which events --mode activity keeps (default " f"{DEFAULT_ACTIVITY_FILTER}).") - p.add_argument("--pages", type=int, default=1, - help=f"Pages to fetch (default 1, cap {PAGE_CAP}).") - p.add_argument("--limit", type=int, default=MAX_LIMIT, metavar="N", - help=f"Rows per cursor page (default and maximum " - f"{MAX_LIMIT} — the endpoint states that limit " - f"itself).") + p.add_argument( + "--pages", + type=page_flow.number_at_least("--pages", 1), + default=1, + help=f"Pages to fetch (default 1, cap {PAGE_CAP}).") + p.add_argument( + "--limit", + type=page_flow.number_at_least("--limit", 1), + default=MAX_LIMIT, metavar="N", + help=f"Rows per cursor page (default and maximum " + f"{MAX_LIMIT} — the endpoint states that limit " + f"itself).") p.add_argument("--category", default=None, help="Accepted for the family's shape and mapped onto " "--collection.") - p.add_argument("--delay", type=float, default=1.0, - help="Delay between pages, seconds (default 1.0).") - p.add_argument("--concurrency", type=int, default=1, metavar="N", - help="Accepted for the family's shape and REFUSED above 1, " - "with the reason: page 5's request does not exist " - "until page 4 has been read.") + p.add_argument( + "--delay", + type=page_flow.number_at_least("--delay", 0, float), + default=1.0, + help="Delay between pages, seconds (default 1.0).") + p.add_argument( + "--concurrency", + type=page_flow.number_at_least("--concurrency", 1), + default=1, metavar="N", + help="Accepted for the family's shape and REFUSED above 1, " + "with the reason: page 5's request does not exist " + "until page 4 has been read.") p.add_argument("--retries", type=page_flow.attempts, default=3, help="Attempts per page load before giving up (default 3).") - p.add_argument("--retry-delay", type=float, default=2.0, - help="Seconds before the first retry, doubling thereafter " - "(default 2.0)") + p.add_argument( + "--retry-delay", + type=page_flow.number_at_least("--retry-delay", 0, float), + default=2.0, + help="Seconds before the first retry, doubling thereafter " + "(default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="opensea_items", help="Output file prefix") @@ -1214,9 +1234,12 @@ def parse_args(): "re-navigation.") p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup.") - p.add_argument("--proxy-block-retries", type=int, default=2, - help="When a page comes back refused, retry it from this " - "many OTHER exits before giving up (default 2).") + p.add_argument( + "--proxy-block-retries", + type=page_flow.number_at_least("--proxy-block-retries", 0), + default=2, + help="When a page comes back refused, retry it from this " + "many OTHER exits before giving up (default 2).") p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 rows were found.") @@ -1240,7 +1263,7 @@ def parse_args(): "challenge was met on opensea.io while this was " "written, so the path is unexercised HERE — a fact " "about this repo's testing and not a claim about what " - "a solver can do (§19).") + "a solver can do.") p.add_argument("--min-score", type=float, default=0.7, help="reCAPTCHA v3 minimum score to request.") p.add_argument("--cdp-endpoint", default=None, @@ -1269,10 +1292,24 @@ def parse_args(): p.add_argument("--headful", dest="headless", action="store_false", help="Run with a real browser window. Needs a display.") args = p.parse_args() + # What the user TYPED, captured before the environment fills anything in. + # An OPENSEA_URL from the environment or .env is a default for a bare + # run; it must never outrank --collection, --category or --mode + # collections, or the run silently reads the environment's collection. + typed_url = args.url env_config.apply(args) if args.category and not args.collection: args.collection = args.category + if not typed_url and args.url and (args.collection + or args.mode == "collections"): + logger.info("Ignoring OPENSEA_URL (%s): --collection/--category or " + "--mode collections says what to read.", args.url) + args.url = None + if (typed_url and args.collection + and collection_slug_from_url(typed_url) != args.collection): + p.error(f"--url {typed_url!r} and --collection {args.collection!r} " + f"name different things. Give one of the two.") if not args.url: if args.collection: mode = args.mode or "items" diff --git a/smoke_test.py b/smoke_test.py index 76b8baa..a5387e3 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -32,7 +32,7 @@ `lastSale.token.symbol` is the empty string on 11 of 50 rows of the Bored Ape capture, which written through reads as a currency. - * `test_markers_do_not_match_a_good_page` is the §18 rule as a test. Every + * `test_markers_do_not_match_a_good_page` is the count-the-marker-on-a-good-page rule as a test. Every marker in every set is asserted ABSENT from six pages known to be good. It is also what pins the decision NOT to treat OpenSea's own "404: This page could not be found." as an empty-state marker: Next.js ships that @@ -159,7 +159,7 @@ def test_fixtures_are_present(): def test_values_on_real_fixtures(): - """Assert VALUES on real fixtures, not coverage (§10). + """Assert VALUES on real fixtures, not coverage. Every expectation here was COMPUTED from the untrimmed capture by make_fixtures.py rather than typed by hand: a hand-typed expectation is a @@ -293,7 +293,7 @@ def test_empty_symbol_is_null(): def test_enforcement_flags_are_pinned_not_trusted(): """Pin the CURRENT behaviour of a column never seen taking its other value. - §10 says to pin a known limitation rather than half-guard it, so a future + The rule is to pin a known limitation rather than half-guard it, so a future change becomes a decision instead of a surprise. Neither enforcement flag has ever been observed True — 3,400 items across two orderings and ten collections, measured 2026-09-17 — so what is asserted here is the SHAPE @@ -336,7 +336,7 @@ def test_prices_are_tokens_not_dollars(): "%s: price_usd (%r) should be the dollar figure and " "price (%r) the token amount — they look swapped" % (row.sku, row.price_usd, row.price)) - # No defaulted currency anywhere (§4's last rung: absent is null). + # No defaulted currency anywhere (absent is null). for row in rows: if row.price is None: ok &= check(row.currency is None, @@ -417,7 +417,7 @@ def test_collection_rows(): def test_collection_totals(): - """§21's arithmetic: complete and exhaustive are different words.""" + """Complete and exhaustive are different words.""" group("collection totals") ok = True totals = P.collection_totals(html_of("collection")) @@ -441,7 +441,7 @@ def test_collection_totals(): # Markers, and the ones deliberately absent # =========================================================================== def test_markers_do_not_match_a_good_page(): - """§18: count every marker on a page you know is good, FIRST. + """Count every marker on a page you know is good, FIRST. A marker that matches every page is worse than no marker: a sibling repo reported "Blocked by akamai" and exit 3 on a 191 KB page the site had @@ -498,7 +498,7 @@ def test_cf_turnstile_is_not_a_marker(): def test_extension_injection_does_not_read_as_a_challenge(): - """The §19 trap, pinned against the fixture that can actually spring it. + """The injected-marker trap, pinned against the fixture that can actually spring it. 2Captcha's Scraping Browser ships an auto-solve extension that injects its own Turnstile hunter into every page it loads. MEASURED on this site @@ -520,7 +520,7 @@ def test_extension_injection_does_not_read_as_a_challenge(): THE FIXTURE IS THE POINT. A sibling repo shipped this check and it PASSED FOR THE WRONG REASON: it ran only against pages fetched with a - plain client, which carry no injection at all (§21). This one runs + plain client, which carry no injection at all. This one runs against a real Scraping Browser capture that does. """ group("scraping browser extension injection") @@ -551,7 +551,7 @@ def test_extension_injection_does_not_read_as_a_challenge(): def test_the_not_found_sentence_is_not_a_marker(): - """Pin the §18 trap this repo caught in the act. + """Pin the marker trap this repo caught in the act. OpenSea's "not found" page reads `404: This page could not be found.`, which looks like the obvious marker for an unknown collection slug. @@ -600,7 +600,7 @@ def test_classification(): ok &= check(P.served_by_opensea(html_of(name)), "%s should read as served by OpenSea" % name) # And the order of the signals: an unambiguous positive before a - # threshold (§17's classification-order trap). + # threshold (order signals by how much they prove). challenge = ("Just a moment..." "") @@ -623,10 +623,10 @@ def test_state_policy_is_complete(): ok &= check(not page_flow.should_retry("empty"), "an empty feed is a correct answer and must not be retried") ok &= check(not page_flow.counts_as_blocked("empty"), - "empty is exit 4, not exit 3 (§8)") + "empty is exit 4, not exit 3") ok &= check(page_flow.should_solve("challenge"), "a challenge must reach the solver — and note that saying it " - "cannot be solved is the one sentence §19 forbids") + "cannot be solved is the one sentence this repo must never write") ok &= check(page_flow.counts_as_blocked("blocked"), "blocked must count as blocked") # A state with no policy falls back to blocked, which is the safe @@ -693,7 +693,7 @@ def cursor(value): def test_page_url_is_none(): - """§7's layer 2 does not exist here, and saying so is the point.""" + """URL-built pagination does not exist here, and saying so is the point.""" group("page addresses") ok = True ok &= check(P.page_url("https://opensea.io/collection/x", 2) is None, @@ -704,7 +704,7 @@ def test_page_url_is_none(): refusal = page_flow.concurrency_refusal( "https://opensea.io/collection/x", "items") ok &= check(refusal and "cursor" in refusal, - "concurrency must be refused WITH the reason (§18)") + "concurrency must be refused WITH the reason") ok &= check(page_flow.concurrency_limit("", "items") == 1, "every mode here is single-worker") return ok @@ -874,7 +874,7 @@ def test_graphql_bodies(): def test_graphql_errors_are_loud(): - """The endpoint answers 200 with an `errors` array. Fail loudly (§8).""" + """The endpoint answers 200 with an `errors` array. Fail loudly.""" group("graphql errors") ok = True payload = {"errors": [{"message": "Something went wrong"}], "data": None} @@ -891,7 +891,7 @@ def test_endpoint_error_shapes(): """Real captured responses for the shapes a page capture cannot show. A third-party audit on 2026-10-13 asked for exactly these, and the point - is that they are CAPTURED rather than written: §26 records a fixture + is that they are CAPTURED rather than written: a sibling recorded a fixture assembled from two captures that claimed something no response says. Each of these came back from the live endpoint that day. @@ -913,7 +913,7 @@ def test_endpoint_error_shapes(): # EVERY one of these came back HTTP 200. A caller that branches on the # status alone reads all four as success, which is why graphql_errors - # exists at all (§8, fail loudly). + # exists at all (fail loudly). for name, captured in edges.items(): ok &= check(captured.get("status") == 200, "%s came back HTTP %r — if the endpoint has started " @@ -970,7 +970,7 @@ def body(name): def test_graphql_and_state_agree(): - """One normaliser, two routes — the claim this repo is built on (§21). + """One normaliser, two routes — the claim this repo is built on. Feeds the SAME objects through both entry points and asserts the rows are identical. The live version of this check measured 49 of 50 rows @@ -1031,7 +1031,7 @@ def test_dom_fallback(): ok &= check(row.price is None, "a fallback row must not invent a price — the rendered " "markup carries only formatted display text") - # And the §20 signal: a served page that links to items and parses to + # And the parser-failure signal: a served page that links to items and parses to # nothing is THIS PARSER'S bug, not an empty collection. ok &= check(page_flow.parsed_nothing_from_a_served_page(html, 0), "a served page with links and no rows must be reported as a " @@ -1091,7 +1091,7 @@ def test_a_dom_only_run_is_not_complete(): "an empty run is not a degraded run; it has its own states") # And every engine has to consult it, or they disagree about a run's - # status on the identical response (§6). + # status on the identical response. for name in ENGINES: source = _engine_source(name) ok &= check("page_flow.degraded_to_dom(" in source, @@ -1100,7 +1100,7 @@ def test_a_dom_only_run_is_not_complete(): # this check looked for the `state_answered = …` line, and a control # that removed the condition from the `ssr_used` expression left # that line untouched — so the suite stayed GREEN against the - # reverted fix. §26: a control that stays green is a hole in the + # reverted fix. A control that stays green is a hole in the # suite, not a pass. ok &= check("args.ssr_usable and state_answered" in source, "%s computes ssr_used without `and state_answered`, so " @@ -1124,7 +1124,7 @@ def test_output_contract(): with tempfile.TemporaryDirectory() as tmp: prefix = os.path.join(tmp, "out") # A run that finds nothing writes NOTHING, so last night's good - # output survives (§8). + # output survives. with open(prefix + ".json", "w") as f: f.write('[{"sku": "yesterday"}]') rc = output_writer.save([], prefix, "both") @@ -1182,7 +1182,7 @@ def test_output_contract(): def test_page_and_position_are_unique(): - """§18's arithmetic bug, pinned: `position` restarts on every page.""" + """An arithmetic bug, pinned: `position` restarts on every page.""" group("page and position") ok = True groups = [] @@ -1216,7 +1216,7 @@ def test_merge_is_page_ordered(): forward, _, _ = output_writer.merge_pages([(1, rows_a), (2, rows_b)]) backward, _, _ = output_writer.merge_pages([(2, rows_b), (1, rows_a)]) ok &= check([r.sku for r in forward] == [r.sku for r in backward], - "the merged order depends on which page arrived first (§8)") + "the merged order depends on which page arrived first") ok &= check([r.sku for r in forward][:3] == [r.sku for r in rows_a], "page 1's rows must come first") # is_deeper is inert here, and that is a measured property of the modes @@ -1252,7 +1252,7 @@ def test_dedupe(): ENGINES = ("playwright_scraper", "puppeteer_scraper", "selenium_scraper") # Flags one engine has and its twins do not, with the reason. The list IS the -# documentation, so closing a difference fails this check too — §20's rule +# documentation, so closing a difference fails this check too — the rule # that a parity check must assert in BOTH directions. FLAG_EXCEPTIONS = { "--chromium-path": ("puppeteer_scraper",), # pyppeteer downloads its own @@ -1299,7 +1299,7 @@ def _engine_flags(name): def test_a_throttle_is_not_a_block(): - """§24's distinction, driven end to end against a FAKE driver. + """The throttle/block distinction, driven end to end against a FAKE driver. A refusal and a throttle want opposite responses: a block means a different exit, a throttle means the SAME exit and a longer wait. @@ -1438,7 +1438,7 @@ class _Args: engine.time.sleep = real_sleep # The three engines must all carry the branch, or they disagree about a - # run's exit code (§6). + # run's exit code. for name in ENGINES: source = _engine_source(name) ok &= check("page_flow.is_throttle(" in source, @@ -1472,7 +1472,7 @@ def test_engine_flag_parity(): "FLAG_EXCEPTIONS says %s is only on %s, but it is on %s" % (flag, tuple(sorted(expected)), carriers)) - # The family contract (§9), asserted against the primary engine. + # The family contract, asserted against the primary engine. contract = ("--url --pages --category --format --out --delay --retries " "--retry-delay --concurrency --proxy --proxy-file " "--proxy-rotate --proxy-shuffle --proxy-block-retries " @@ -1487,7 +1487,7 @@ def test_engine_flag_parity(): # A flag this repo BANS, scoped to the engines: a --country on a scraper # could disagree with the URL it was given. It is legitimate on - # fingerprint_client.py, which is why the check is scoped (§10). + # fingerprint_client.py, which is why the check is scoped. for name in ENGINES: ok &= check("--country" not in flags[name], "%s has a --country flag; on this site the locale is a " @@ -1503,12 +1503,12 @@ def test_retries_is_a_count_of_attempts(): navigation at all and the engine then judges `about:blank` — 39 bytes — and reports a block. A third-party audit on 2026-10-13 passed `--retries 0`, got exit 3, and concluded this site was refusing us while - it served 250 rows to the same machine a minute later (§28). + it served 250 rows to the same machine a minute later. Read off DISK rather than through an import, deliberately: a check gated behind importing the engine skips for exactly the engines whose driver is absent from the venv, so a fix that reached one engine of three would - pass locally and fail in engine-smoke (§27.4). + pass locally and fail in engine-smoke. """ group("--retries counts attempts") ok = True @@ -1573,7 +1573,7 @@ def test_engine_constants_agree(): "the engines disagree about %s: %s — one engine " "reporting a different threshold than its twins on the " "identical run is exactly the drift the shared modules " - "exist to prevent (§6)" % (constant, seen)) + "exist to prevent" % (constant, seen)) return ok @@ -1583,7 +1583,7 @@ def test_engines_import_their_driver_at_module_level(): A sibling repo imported `launch`/`connect` inside the launch path, so the module imported cleanly with no pyppeteer installed: the group never skipped, and the CI job that exists to fail on unexpected skips could not - have caught a broken import (§10). + have caught a broken import. """ group("module-level driver imports") ok = True @@ -1615,7 +1615,7 @@ def test_engine_parity(skips): parameter is positional. Both crashed on their FIRST fetch, and it was invisible to import, `--help`, `compileall`, the undefined-name walk and 400+ green assertions — because none of those calls a function the way a - live run does (§17). + live run does. """ group("shared-module call signatures") ok = True @@ -1684,7 +1684,7 @@ def test_no_undefined_names(): A sibling repo's engine died with `NameError` on a line reached only while fetching, after an import had been removed. The module imported - cleanly, `--help` worked, `compileall` passed and CI was green (§10). + cleanly, `--help` worked, `compileall` passed and CI was green. Deliberately COARSE — one pool of bindings per module, no scope tracking — so it under-reports rather than inventing problems. @@ -1746,11 +1746,11 @@ def test_no_undefined_names(): def test_no_unreachable_statements(): """A statement after return/raise/break/continue in the SAME block. - §22's check, which found the same fifteen lines in six sibling repos — + A check which found the same fifteen lines in six sibling repos — byte for byte, present since each one's first commit: a function whose `def` line had been lost, leaving its docstring and body absorbed into the end of the function above it. It parses, it imports, `--help` works, - `compileall` passes, and the §10 undefined-name walk cannot see it and + `compileall` passes, and the undefined-name walk cannot see it and SHOULD not — that walk pools every binding in a file rather than tracking scopes, so a name in the dead block resolves against a real parameter elsewhere. This is a second check, not a tightening of the @@ -1779,11 +1779,11 @@ def test_no_unreachable_statements(): "%s line %d: %s is unreachable — the " "statement above it on line %d always " "leaves the block. A lost `def` line " - "looks exactly like this (§22)." + "looks exactly like this." % (filename, dead.lineno, type(dead).__name__, statement.lineno)) break - # A binding check that binds nothing passes for the wrong reason (§22). + # A binding check that binds nothing passes for the wrong reason. ok &= check(scanned >= 10, "the unreachable-statement walk only looked at %d file(s)" % scanned) @@ -1796,7 +1796,7 @@ def test_dockerfile_matches_its_entrypoint(): All three repos in this family shipped an image that died with ModuleNotFoundError on every invocation, `--help` included, because `proxy_pool.py` was missing from the COPY list. CI never builds the - image, so nothing noticed (§10). + image, so nothing noticed. """ group("dockerfile") ok = True @@ -1855,7 +1855,7 @@ def test_dockerfile_matches_its_entrypoint(): # A check that spells its own banned phrases has to exempt the file it lives # in — and three repos in this family did exactly that, leaving the one file # most likely to acquire a stray phrase or a pasted credential as the one -# file nobody scanned (§22). Built this way, the scan covers itself. +# file nobody scanned. Built this way, the scan covers itself. _A = "anti" + "detect" BANNED_PHRASES = ( "cloud" + " browser", @@ -1868,10 +1868,10 @@ def test_dockerfile_matches_its_entrypoint(): def test_wording(): - """§12's table, enforced. It reached two public repo DESCRIPTIONS while + """The banned-wording table, enforced. It reached two public repo DESCRIPTIONS while the suite scanned only files, so `.github/repo-metadata.yml` is scanned here too — a banned-wording check should cover the surfaces the repo - publishes, not only the files it contains (§21).""" + publishes, not only the files it contains.""" group("wording") ok = True scanned = 0 @@ -1891,10 +1891,10 @@ def test_wording(): # No escapes. An escape would have to name the phrase it # excuses, and naming it is using it — which is how this # check's own escape clause became the only hit in the repo - # the moment the check started scanning its own file (§22). + # the moment the check started scanning its own file. ok &= check(phrase.lower() not in text.lower(), "%s contains the banned phrase %r — write " - "'Scraping Browser API' instead (§12)" + "'Scraping Browser API' instead" % (os.path.relpath(path, REPO_ROOT), phrase)) ok &= check(scanned > 10, "the wording scan looked at almost nothing") return ok @@ -1905,7 +1905,10 @@ def test_env_example_matches_the_code(): ok = True with open(os.path.join(REPO_ROOT, ".env.example"), encoding="utf-8") as f: example = f.read() - documented = set(re.findall(r"^([A-Z][A-Z0-9_]+)=", example, re.M)) + # A variable documented as a commented-out example still counts as + # documented: OPENSEA_URL ships that way so a copy cannot take a run over. + documented = set(re.findall(r"^(?:#\s*)?([A-Z][A-Z0-9_]+)=", example, + re.M)) read = set(env_config.ENV_KEYS) ok &= check(documented == read, "the .env.example and ENV_KEYS disagree: only in the example " @@ -1917,7 +1920,7 @@ def test_env_example_matches_the_code(): "or OPENSEA_*)" % name) # A copied example must read as UNSET for every credential, and stay - # usable for the one value that is not one (§17). + # usable for the one value that is not one. import argparse as argparse_module with tempfile.TemporaryDirectory() as tmp: path = os.path.join(tmp, ".env") @@ -1937,12 +1940,13 @@ def test_env_example_matches_the_code(): "a copied .env.example's CDP endpoint was read as " "configured — it would connect with the literal " "'{login}-zone-…' as its username and get a 401 a " - "long way from its cause (§17)") + "long way from its cause") ok &= check(namespace.proxy is None, "a copied .env.example's proxy was read as configured") - ok &= check(namespace.url and namespace.url.startswith("https://"), - "the example's non-credential default should stay " - "usable") + ok &= check(namespace.url is None, + "a copied .env.example set a default --url; it must " + "ship commented out, because an environment URL used " + "to override --collection and --mode collections") finally: for k in read: os.environ.pop(k, None) @@ -1957,14 +1961,14 @@ def test_credentials_never_reach_logs(): # Built by concatenation on purpose: no LINE of this file then holds a # string shaped like a credentialled URL, so the committed-credential # check needs no allowlist entry for it — and an allowlist entry is the - # thing that would also let a real one through (§17). + # thing that would also let a real one through. secret = "hunter" + "2" url = "http://user:" + secret + "@exit.example.com:2334" masked = proxy_pool.mask(url) ok &= check("hunter2" not in masked, "mask() leaked a password") ok &= check("exit.example.com" in masked and "2334" in masked, "mask() should KEEP host and port — which exit a run used is " - "the point of the log and is not the secret (§8)") + "the point of the log and is not the secret") # Globally, not once: a Playwright connection error repeats an endpoint # five times, and a masker that handles the first prints the password the # other four times while looking like it works. @@ -1990,7 +1994,7 @@ def test_credentials_never_reach_logs(): def test_ci_checks_is_wired_up(): - """ONE implementation, invoked from both CI and this suite (§17). + """ONE implementation, invoked from both CI and this suite. Three repos in this family shipped `.github/ci_checks.py` that NOTHING ran, while the workflow carried an inline grep doing a narrower version @@ -2007,7 +2011,7 @@ def test_ci_checks_is_wired_up(): # absent — never on a file inside it being missing. Two sibling repos run # this suite INSIDE their Docker image, which deliberately carries no # `.github`, so a check that reads a workflow is correct in the repo and - # red in the image (§22). The narrowness is the point: a check that + # red in the image. The narrowness is the point: a check that # quietly starts passing once its input disappears is the same failure # wearing a new face, so `.github` present with a file missing is still a # failure. Before this existed the whole suite died here with @@ -2036,6 +2040,36 @@ def test_ci_checks_is_wired_up(): ok &= check(result.returncode == 0, "the committed-credential check FAILS on this repository: " "%s" % (result.stdout + result.stderr)[-400:]) + + # Its exclusions are judged RELATIVE to the repository. Judged on the + # absolute path, a clone made under a directory called `live`, + # `captures` or `venv-work` skipped every file and printed "nothing + # credential-shaped" having read nothing. A virtualenv inside the repo + # (recognised by pyvenv.cfg, whatever it is called) is still skipped. + import shutil + key = "ab" * 16 + with tempfile.TemporaryDirectory() as tmp: + clone = os.path.join(tmp, "venv-work", "live", "captures", "repo") + os.makedirs(os.path.join(clone, ".github")) + shutil.copy(script, os.path.join(clone, ".github", "ci_checks.py")) + with open(os.path.join(clone, "planted_cfg.py"), "w") as f: + f.write("API = %r\n" % key) + env_dir = os.path.join(clone, "myenv", "lib") + os.makedirs(env_dir) + with open(os.path.join(clone, "myenv", "pyvenv.cfg"), "w") as f: + f.write("home = /usr\n") + with open(os.path.join(env_dir, "vendored.py"), "w") as f: + f.write("X = %r\n" % key) + planted = subprocess.run( + [sys.executable, os.path.join(clone, ".github", "ci_checks.py"), + "--secret-check"], cwd=clone, capture_output=True, text=True) + out = planted.stdout + planted.stderr + ok &= check(planted.returncode == 1 and "planted_cfg.py" in out, + "a key planted in a clone under live/captures/venv-* parents " + "was not found: %s" % out[-300:]) + ok &= check("vendored.py" not in out, + "the scan walked into a virtualenv (pyvenv.cfg) inside the " + "repository") return ok @@ -2070,7 +2104,7 @@ def test_sample_output(): def test_no_dead_policy_constants(): - """A policy constant nothing reads is the same defect as dead code (§17).""" + """A policy constant nothing reads is the same defect as dead code.""" group("policy constants") ok = True sources = {name: _engine_source(name) for name in ENGINES} @@ -2080,7 +2114,7 @@ def test_no_dead_policy_constants(): # set to 4, and read by NOTHING — the engines take their with-pool # budget from `--proxy-block-retries` instead. A check with a hand-kept # list only ever covers what someone remembered to add, which is the - # same failure as the policy it is checking for (§17). + # same failure as the policy it is checking for. policy = [n for n in dir(page_flow) if n.isupper() and not n.startswith("_") and isinstance(getattr(page_flow, n), (bool, int)) @@ -2088,18 +2122,467 @@ def test_no_dead_policy_constants(): "SCROLL_MAX_ROUNDS", "SCROLL_STABLE_ROUNDS")] ok &= check(len(policy) >= 3, "the policy-constant scan found almost nothing to check") + # A constant page_flow reads ITSELF, inside an object every engine must + # construct, is consumed through that object. SOLVES_PER_PAGE is the + # default limit of SolveBudget; the engines read it by building one. + consumed_through = {"SOLVES_PER_PAGE": "page_flow.SolveBudget("} for constant in sorted(policy): + token = consumed_through.get(constant, constant) consumers = [name for name, text in sources.items() - if constant in text] + if token in text] ok &= check(len(consumers) == len(ENGINES), "page_flow.%s is read by %s and not by all three engines " "— a policy that looks enforced and is not is worse than " - "no policy (§17). Either every engine consults it, or it " + "no policy. Either every engine consults it, or it " "should not exist." % (constant, consumers or "nothing")) print(" policy constants checked: %s" % sorted(policy)) return ok +# --------------------------------------------------------------------------- +# Engines driven with their DRIVER STUBBED, in a subprocess +# --------------------------------------------------------------------------- +# These run in the core-only venv too: a check that needs the driver +# installed skips exactly where nobody looks, and the three things below +# (the solve budget, OPENSEA_URL versus the builder flags, numeric flag +# validation) are logic, not browser behaviour. A subprocess, so the stub +# modules never leak into the rest of this suite's imports. +_STUB_HARNESS = r""" +import json, os, sys, tempfile, time, types +from unittest import mock + +def _module(name, **attrs): + m = types.ModuleType(name) + m.__dict__.update(attrs) + sys.modules[name] = m + return m + +class _E(Exception): + pass + +engine, scenario = sys.argv[1], sys.argv[2] +if engine == "playwright_scraper": + _module("playwright") + _module("playwright.sync_api", sync_playwright=mock.MagicMock(), + Error=_E, TimeoutError=type("PWTimeout", (_E,), {})) +elif engine == "puppeteer_scraper": + _module("pyppeteer", launch=mock.MagicMock(), connect=mock.MagicMock()) +else: + _module("selenium", webdriver=mock.MagicMock()) + _module("selenium.webdriver", Chrome=mock.MagicMock()) + _module("selenium.common") + _module("selenium.common.exceptions", WebDriverException=_E) + _module("selenium.webdriver.chrome") + _module("selenium.webdriver.chrome.options", Options=mock.MagicMock()) + _module("selenium.webdriver.common") + _module("selenium.webdriver.common.by", By=mock.MagicMock()) + +sys.path.insert(0, os.getcwd()) +mod = __import__(engine) + +if scenario == "budget": + time.sleep = lambda *a, **k: None + out = os.path.join(tempfile.mkdtemp(), "out") + sys.argv = [engine, "--url", "https://opensea.io/collection/x", + "--out", out] + args = mod.parse_args() + args.twocaptcha_key = "stub-never-sent" + calls = [] + challenge = types.SimpleNamespace( + kind="reCAPTCHA v2", source="stub", sitekey="stub", action=None, + is_turnstile=False, solved_user_agent=None) + def solver(*a, **k): + calls.append(1) + return "stub-token" + html = "challenge" + for name, value in (("solve_recaptcha", solver), + ("reconcile_detections", lambda *a, **k: challenge), + ("detect_recaptcha_v3", lambda *a, **k: None), + ("detect_recaptcha_in_page", lambda *a, **k: None), + ("_classify", lambda *a, **k: "challenge"), + ("_count", lambda *a, **k: 0), + ("_content_when_settled", lambda *a, **k: html), + ("_content", lambda *a, **k: html), + ("_current_url", + lambda *a, **k: "https://opensea.io/collection/x"), + ("_document_status", lambda *a, **k: 403)): + if hasattr(mod, name): + setattr(mod, name, value) + mod._fetch_first_page(mock.MagicMock(), args, None) + print(json.dumps({"solves": len(calls)})) +elif scenario == "envurl": + os.environ["OPENSEA_URL"] = "https://opensea.io/collection/boredapeyachtclub" + result = {} + for label, argv in (("collection", ["--collection", "pudgypenguins"]), + ("category", ["--category", "pudgypenguins"]), + ("ranking", ["--mode", "collections"]), + ("bare", []), + ("typed", ["--url", + "https://opensea.io/collection/azuki"])): + sys.argv = [engine] + argv + try: + result[label] = mod.parse_args().url + except SystemExit as e: + result[label] = "exit %s" % e.code + print(json.dumps(result)) +elif scenario == "flags": + result = {} + for flag, value in (("--pages", "0"), ("--limit", "0"), ("--delay", "-1"), + ("--retry-delay", "-1"), ("--concurrency", "0"), + ("--proxy-block-retries", "-1"), ("--retries", "0")): + sys.argv = [engine, "--url", "https://opensea.io/collection/x", + flag, value] + err = __import__("io").StringIO() + try: + with mock.patch("sys.stderr", err): + mod.parse_args() + result[flag] = "accepted" + except SystemExit as e: + text = err.getvalue() + result[flag] = ("refused" if e.code == 2 and "error:" in text + and flag in text.split("error:", 1)[1] + else "exit %s: %s" % (e.code, text[-200:])) + print(json.dumps(result)) +""" + + +def _drive_engine(engine, scenario): + """Run one scenario of _STUB_HARNESS; returns the JSON it printed.""" + env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") + for key in env_config.ENV_KEYS: + env.pop(key, None) + r = subprocess.run([sys.executable, "-c", _STUB_HARNESS, engine, scenario], + cwd=REPO_ROOT, capture_output=True, text=True, + env=env, timeout=120) + lines = [l for l in r.stdout.splitlines() if l.startswith("{")] + if r.returncode != 0 or not lines: + return {"error": (r.stderr or r.stdout)[-600:]} + return json.loads(lines[-1]) + + +def test_solve_budget_covers_every_call_site(): + """One paid solve per page, however many times the page comes back. + + Each engine checks for a challenge twice per fetch attempt — before the + page is classified and after — and re-fetches up to + BLOCK_RETRIES_WITHOUT_POOL times. Only the second check used to be + counted, so a page that kept coming back as a challenge bought a solve + on every attempt: 5 solver calls against a limit of 1, in all three + engines. Driven with the solver stubbed, so nothing is ever sent. + """ + group("captcha solve budget") + ok = True + for engine in ENGINES: + result = _drive_engine(engine, "budget") + ok &= check("error" not in result, + "%s: the budget harness did not run: %s" + % (engine, result.get("error"))) + ok &= check(result.get("solves") == page_flow.SOLVES_PER_PAGE, + "%s bought %s solve(s) for one page that kept coming back " + "as a challenge; the limit is %d per page" + % (engine, result.get("solves"), page_flow.SOLVES_PER_PAGE)) + budget = page_flow.SolveBudget(1) + ok &= check(budget.spend() and not budget.spend() and budget.remaining() == 0, + "SolveBudget(1) must allow exactly one purchase") + return ok + + +def test_env_url_never_outranks_builder_flags(): + """OPENSEA_URL is a default for a bare run, not a rival to --collection. + + With the URL .env.example used to ship, `--collection pudgypenguins` + silently scraped Bored Apes and `--mode collections` was refused, + because the environment filled --url before the builder flags ran. + """ + group("OPENSEA_URL versus the builder flags") + ok = True + for engine in ENGINES: + r = _drive_engine(engine, "envurl") + ok &= check("error" not in r, "%s: the env-URL harness did not run: %s" + % (engine, r.get("error"))) + ok &= check(str(r.get("collection", "")).endswith( + "/collection/pudgypenguins"), + "%s: --collection pudgypenguins with OPENSEA_URL set read " + "%r" % (engine, r.get("collection"))) + ok &= check(str(r.get("category", "")).endswith( + "/collection/pudgypenguins"), + "%s: --category pudgypenguins with OPENSEA_URL set read " + "%r" % (engine, r.get("category"))) + ok &= check(str(r.get("ranking", "")).endswith("/collections"), + "%s: --mode collections with OPENSEA_URL set gave %r" + % (engine, r.get("ranking"))) + ok &= check(str(r.get("bare", "")).endswith("/boredapeyachtclub"), + "%s: a bare run should still use OPENSEA_URL, got %r" + % (engine, r.get("bare"))) + ok &= check(str(r.get("typed", "")).endswith("/collection/azuki"), + "%s: a typed --url must win over OPENSEA_URL, got %r" + % (engine, r.get("typed"))) + return ok + + +def test_numeric_flags_refuse_nonsense(): + """Counts and delays below their floor are a usage error naming the flag.""" + group("numeric flag validation") + ok = True + for engine in ENGINES: + r = _drive_engine(engine, "flags") + ok &= check("error" not in r, "%s: the flag harness did not run: %s" + % (engine, r.get("error"))) + for flag in ("--pages", "--limit", "--delay", "--retry-delay", + "--concurrency", "--proxy-block-retries", "--retries"): + ok &= check(r.get(flag) == "refused", + "%s %s with a value below its floor: %r (expected an " + "argparse error naming the flag)" + % (engine, flag, r.get(flag))) + # The Scraper API client counts EXTRA attempts, so 0 is legal there and + # only a negative is refused. + for value, expect in (("-1", 2), ("0", None)): + r = subprocess.run( + [sys.executable, "-c", + "import sys; sys.argv=['x','--url','https://opensea.io/collection/x'," + "'--retries','%s']; import scraper_api_client as s; s.parse_args();" + "print('parsed')" % value], + cwd=REPO_ROOT, capture_output=True, text=True, + env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + if expect == 2: + ok &= check(r.returncode == 2 and "--retries" in r.stderr, + "scraper_api_client --retries -1 should be a usage " + "error, got rc %s" % r.returncode) + else: + ok &= check(r.returncode == 0 and "parsed" in r.stdout, + "scraper_api_client --retries 0 (no extra attempts) " + "should parse, got rc %s %s" % (r.returncode, + r.stderr[-200:])) + return ok + + +def test_writes_are_atomic(): + """A failure while writing must leave the previous good file intact. + + `open(path, "w")` truncated before writing, so an error inside json.dump + left `out.json` 12 bytes of invalid JSON and `out.csv` 0 bytes. Modes are + pinned under three umasks, because under the default 022 every candidate + rule agrees and only a non-default one tells them apart. + """ + group("atomic writes") + ok = True + from unittest import mock + rows = [output_writer.Item(sku="a", title="good", price=1.0)] + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "out") + with contextlib.redirect_stdout(io.StringIO()): + output_writer.finish_run( + rows, out, "both", False, blocked=False, + stop_reason="completed", pages_requested=1, + pages_completed=1, start_url="u", final_url="u", + mode="items") + before = {ext: open(out + ext, "rb").read() + for ext in (".json", ".csv", ".meta.json")} + + def boom(*a, **k): + raise OSError("disk full (injected)") + for target, ext, call in ( + ("json.dump", ".json", + lambda: output_writer.write_json(rows, out + ".json")), + ("csv.DictWriter.writerow", ".csv", + lambda: output_writer.write_csv(rows, out + ".csv", + output_writer.Item)), + ("json.dump", ".meta.json", + lambda: output_writer.write_run_meta(out, {"status": "x"}))): + raised = False + with mock.patch(target, boom), \ + contextlib.redirect_stdout(io.StringIO()): + try: + call() + except OSError: + raised = True + ok &= check(raised, "the injected failure in %s did not surface" + % target) + ok &= check(open(out + ext, "rb").read() == before[ext], + "a failure inside %s destroyed the previous good %s" + % (target, ext)) + leftovers = [n for n in os.listdir(tmp) if n.endswith(".tmp")] + ok &= check(not leftovers, "a failed write left temp files behind: %s" + % leftovers) + + for umask in (0o022, 0o002, 0o077): + new_path = os.path.join(tmp, "fresh_%o.json" % umask) + old = os.umask(umask) + try: + output_writer.write_json(rows, new_path) + finally: + os.umask(old) + mode = os.stat(new_path).st_mode & 0o777 + ok &= check(mode == 0o666 & ~umask, + "a new file under umask %03o got mode %03o, expected " + "%03o (what open() would give)" + % (umask, mode, 0o666 & ~umask)) + kept = os.path.join(tmp, "kept.json") + output_writer.write_json(rows, kept) + os.chmod(kept, 0o640) + output_writer.write_json(rows, kept) + ok &= check(os.stat(kept).st_mode & 0o777 == 0o640, + "rewriting an existing file did not keep its mode") + return ok + + +def test_csv_formula_cells_are_escaped(): + """A formula-shaped string is text in the CSV, untouched in the JSON.""" + group("CSV formula neutralisation") + ok = True + evil = '=HYPERLINK("http://example.invalid","x")' + rows = [output_writer.Collection(sku="c", title=evil, floor_change=-5.0, + category="@art")] + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "out") + with contextlib.redirect_stdout(io.StringIO()): + output_writer.finish_run( + rows, out, "both", False, blocked=False, + stop_reason="completed", pages_requested=1, + pages_completed=1, start_url="u", final_url="u", + mode="collections", + extra={"csv_cells_escaped": "caller wins", "sort": None}) + with open(out + ".csv", newline="", encoding="utf-8") as f: + row = next(csv_module.DictReader(f)) + with open(out + ".json", encoding="utf-8") as f: + data = json.load(f)[0] + with open(out + ".meta.json", encoding="utf-8") as f: + meta = json.load(f) + ok &= check(row["title"] == "'" + evil, + "a formula-shaped title reached the CSV unescaped: %r" + % row["title"]) + ok &= check(row["category"] == "'@art", + "an @-led cell reached the CSV unescaped") + ok &= check(row["floor_change"] == "-5.0", + "a negative NUMBER was escaped; only strings may be") + ok &= check(data["title"] == evil, + "the JSON output must keep the site's bytes") + ok &= check(meta.get("csv_cells_escaped") == "caller wins", + "csv_cells_escaped must merge UNDER the caller's extra") + with contextlib.redirect_stdout(io.StringIO()): + output_writer.finish_run( + rows, out, "csv", False, blocked=False, + stop_reason="completed", pages_requested=1, + pages_completed=1, start_url="u", final_url="u", + mode="collections") + with open(out + ".meta.json", encoding="utf-8") as f: + ok &= check(json.load(f).get("csv_cells_escaped") == 2, + "the sidecar must count the escaped cells") + listed = output_writer.Item(sku="i", traits=["-x", "y"]) + with contextlib.redirect_stdout(io.StringIO()): + n = output_writer.write_csv([listed], out + "2.csv", + output_writer.Item) + ok &= check(n == 1, "a joined list cell starting with '-' must be " + "escaped too (escape AFTER the list serialiser)") + return ok + + +def test_diff_runs_tracks_every_mode(): + """diff_runs compares columns each mode's row class really has.""" + import diff_runs + group("diff_runs per mode") + ok = True + for mode, cls in output_writer.ROW_CLASS_BY_MODE.items(): + names = {f.name for f in dataclass_fields(cls)} + tracked = diff_runs.TRACKED_FIELDS_BY_MODE.get(mode) + ok &= check(bool(tracked), "diff_runs tracks nothing for --mode %s" + % mode) + missing = [f for f in (tracked or ()) if f not in names] + ok &= check(not missing, "diff_runs tracks %s for --mode %s, which %s " + "does not have" % (missing, mode, cls.__name__)) + counts = diff_runs.COUNT_FIELDS_BY_MODE.get(mode, ()) + ok &= check(set(counts) <= set(tracked or ()), + "a %s count field is not tracked" % mode) + planted = { + "items": ({"sku": "a", "price": 1.0, "best_offer": None}, + {"sku": "a", "price": 2.0, "best_offer": None}), + "collections": ({"sku": "c", "volume": 100.0, "top_offer": 1.0}, + {"sku": "c", "volume": 5000.0, "top_offer": 1.0}), + "activity": ({"sku": "e", "event_type": "SALE", "to_address": "0x1"}, + {"sku": "e", "event_type": "TRANSFER", + "to_address": "0x1"}), + } + for mode, (before, after) in planted.items(): + result = diff_runs.diff_products([before], [after], mode=mode) + ok &= check(len(result["changed"]) == 1, + "a planted change in a %s run surfaced as %d changed" + % (mode, len(result["changed"]))) + inferred = diff_runs.diff_products([before], [after]) + ok &= check(inferred["mode"] == mode, + "diff_runs inferred %r from %s rows" + % (inferred["mode"], mode)) + return ok + + +def test_gitignore_covers_dumps_and_keeps_shipped_files(): + """Dumps and scratch output are ignored; shipped files are not.""" + group(".gitignore") + ok = True + if not os.path.isdir(os.path.join(REPO_ROOT, ".git")): + print(" SKIP not a git checkout") + return ok + + def ignored(path): + return subprocess.run(["git", "check-ignore", "-q", path], + cwd=REPO_ROOT).returncode == 0 + for path in ("snap.page2", "x.json.page2", "out/a", "output/a", "runs/a", + "results/a", "tmp/a", "dumps/a", "page.html", ".env", + ".env.bak", ".env.local", "live/a.html", "captures/a.html"): + ok &= check(ignored(path), "%s is not ignored" % path) + for path in ("sample_output.json", "sample_output.csv", + "fixtures_generated.json", ".env.example", + "tests/test_smoke.py", ".github/ci_checks.py", + "smoke_test.py"): + ok &= check(not ignored(path), "%s is ignored but ships" % path) + return ok + + +def test_no_private_citations(): + """Shipped files state their rules in their own words. + + They used to cite section numbers of an internal document nobody outside + can open. The banned tokens are assembled from pieces so this file does + not match itself. + """ + group("no private citations") + ok = True + section = chr(0xA7) + banned = [re.compile(re.escape(section) + r"\s*\d"), + re.compile(re.escape("CLAUDE" + ".md")), + re.compile("claude" + "_2scraper"), + re.compile("2scraper" + "_repo_rules"), + re.compile(r"(?i)\b(?:play" + r"book|tem" + r"plate)\s+" + + re.escape(section))] + try: + tracked = subprocess.run(["git", "ls-files"], cwd=REPO_ROOT, + capture_output=True, text=True, + check=True).stdout.split() + except (subprocess.CalledProcessError, FileNotFoundError): + tracked = [os.path.relpath(os.path.join(d, n), REPO_ROOT) + for d, _, ns in os.walk(REPO_ROOT) + if ".git" not in d.split(os.sep) for n in ns] + hits, scanned = [], 0 + for rel in tracked: + path = os.path.join(REPO_ROOT, rel) + if not os.path.isfile(path): + continue + try: + with open(path, encoding="utf-8") as f: + text = f.read() + except (UnicodeDecodeError, OSError): + continue + scanned += 1 + for pattern in banned: + m = pattern.search(text) + if m: + hits.append("%s: %r" % (rel, m.group(0))) + break + ok &= check(scanned > 10, "the citation scan read only %d files" % scanned) + ok &= check(not hits, "private citations in shipped files: %s" + % hits[:10]) + return ok + + def test_engine_modules_import(skips): """Every engine must import, or say which library it is missing.""" group("engine imports") @@ -2259,6 +2742,14 @@ def main(): ok &= test_x_debug_header_is_redacted() ok &= test_scraper_api_payload_and_status() ok &= test_engine_modules_import(skips) + ok &= test_solve_budget_covers_every_call_site() + ok &= test_env_url_never_outranks_builder_flags() + ok &= test_numeric_flags_refuse_nonsense() + ok &= test_writes_are_atomic() + ok &= test_csv_formula_cells_are_escaped() + ok &= test_diff_runs_tracks_every_mode() + ok &= test_no_private_citations() + ok &= test_gitignore_covers_dumps_and_keeps_shipped_files() passed = _total_checks - len(_failures) if passed < CLAIMED_CHECK_FLOOR: