diff --git a/.env.example b/.env.example index 4a9158e..dda3dd5 100644 --- a/.env.example +++ b/.env.example @@ -54,8 +54,8 @@ TWOCAPTCHA_KEY=your_2captcha_api_key_here # Measured over this endpoint on 2026-09-16: 254 rows across two archive # days at 100% payload coverage, 60 from a tag feed, 55 from an author page # (against a local browser's 10), 1 from a story page. A profile's -# credentials live about a day, so the shape below is documented and no -# working endpoint is ever pasted into this repo. +# credentials expire, so the shape below is documented and no working +# endpoint is ever pasted into this repo. # pid-{id} a profile with persistent cookies, ONE live connection # each — reuse pids rather than minting one per run # diff --git a/.github/ci_checks.py b/.github/ci_checks.py index 60df7a8..e9f7ef9 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -34,7 +34,8 @@ ENGINE_LIBS = ("playwright", "pyppeteer", "selenium", "webdriver_manager") CLIS = ["playwright_scraper.py", "puppeteer_scraper.py", "selenium_scraper.py", - "scraper_api_client.py", "fingerprint_client.py", "env_config.py"] + "scraper_api_client.py", "fingerprint_client.py", "env_config.py", + "diff_runs.py"] SAMPLE_FILES = ("sample_output.json", "sample_output.csv") @@ -77,7 +78,7 @@ # pasted while debugging. An allowlist entry would have switched the # check off exactly there. A sibling repo's copy of this script FAILED ON # ITS OWN MAIN for the opposite reason: its fixtures were neither - # allowlisted nor written this way (§17). + # allowlisted nor written this way. "{login}", "{user}", "password}@", "***", "u:p@h", "login:password@host:port", # the shape a refusal message prints "user:secret@", # the proxy-pool masking fixtures @@ -115,7 +116,7 @@ # into pip's vendored code and flags a 32-hex string in `_elffile.py` as # key-shaped. Correct about the string, wrong about the file — and it is the # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people -# have to argue with is one they learn to suppress (CLAUDE.md §22). +# have to argue with is one they learn to suppress. # # Deliberately NOT a narrowing of what gets scanned: an untracked file is # still read, because a key pasted into a scratch file beside the scripts is @@ -142,18 +143,55 @@ def _is_in_virtualenv(path): return False +# Read too, though nothing here is source: data and dumps are where a key or +# a session arrives by accident (a fixture captured with a key in a request, +# a `--dump-html` of a page that echoed one), and `.env.bak`/`.env.local` +# are what a working `.env` gets renamed to. These shapes are git-ignored by +# design, so a hit in an IGNORED one is reported as a warning — the ignore +# rule is what keeps it out of a commit, and a local key in a local file is +# not a defect. A hit in one git would NOT ignore fails like any other. +DATA_SUFFIXES = (".json", ".html", ".csv", ".toml", ".cfg", ".ini") +DATA_NAMES = ("Dockerfile",) + +# Skipped by name, and only RELATIVE to the repository: tested against the +# path's absolute parts, a clone that merely lives under a directory called +# `venv` excluded every file, and the scan reported "0 files scanned, +# nothing credential-shaped" — a clean pass over nothing. +SKIPPED_DIRS = {".git", "__pycache__", ".venv", "venv", ".pytest_cache"} + + +def _is_data_file(path): + name = path.name + return (path.suffix in DATA_SUFFIXES or name in DATA_NAMES + or (name.startswith(".env.") and name != ".env.example")) + + def scanned_files(): for path in sorted(REPO.rglob("*")): - if not path.is_file() or path.suffix not in SCANNED_SUFFIXES: + if not path.is_file(): + continue + if path.suffix not in SCANNED_SUFFIXES and not _is_data_file(path): continue - if any(part in {".git", "__pycache__", ".venv", "venv"} - for part in path.parts): + if any(part in SKIPPED_DIRS for part in path.relative_to(REPO).parts): continue if _is_in_virtualenv(path): continue yield path +def _git_ignored(paths): + """The subset of `paths` git would ignore; empty outside a checkout.""" + if not paths: + return set() + try: + done = subprocess.run( + ["git", "check-ignore", "--stdin"], cwd=REPO, capture_output=True, + text=True, input="\n".join(str(p.relative_to(REPO)) for p in paths)) + except FileNotFoundError: + return set() + return {REPO / line for line in done.stdout.splitlines() if line} + + def help_check(): failed = [] for name in CLIS: @@ -213,32 +251,41 @@ def sample_check(): failed.append("sample_output.csv header differs from output_writer.Product") if not failed: - discounted = sum(1 for r in rows if r.get("original_price")) + paywalled = sum(1 for r in rows if r.get("is_paywalled")) print(f"ok {len(rows)} rows, {len(expected)} columns, " - f"{discounted} discounted, schema matches") + f"{paywalled} paywalled, schema matches") return failed def secret_check(): - failed = [] + failed, warned = [], [] scanned = 0 - for path in scanned_files(): + files = list(scanned_files()) + ignored_data = _git_ignored([p for p in files if _is_data_file(p)]) + for path in files: scanned += 1 + sink = warned if path in ignored_data else failed for lineno, line in enumerate( path.read_text(encoding="utf-8", errors="replace").splitlines(), 1): rel = path.relative_to(REPO) if CREDENTIALLED_URL.search(line) and not any( token in line for token in CREDENTIAL_ALLOWED): - failed.append(f"{rel}:{lineno} looks like a URL with real " - f"credentials in it") + sink.append(f"{rel}:{lineno} looks like a URL with real " + f"credentials in it") for match in HEX32.findall(line): if any(token in line.lower() for token in HEX32_ALLOWED): continue - failed.append(f"{rel}:{lineno} contains {match[:6]}… — a " - f"32-char hex string, the shape of a 2captcha key") - + sink.append(f"{rel}:{lineno} contains {match[:6]}… — a " + f"32-char hex string, the shape of a 2captcha key") + + for line in warned: + print(f"WARNING {line} (git-ignored, so not a commit risk while the " + f"ignore rule holds; delete it if it is not yours)") + if scanned == 0: + failed.append("0 files scanned — a scan that read nothing cannot say " + "the repository is clean") if not failed: print(f"ok {scanned} files scanned, nothing credential-shaped") return failed diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 4fe7f01..45cff61 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -20,7 +20,7 @@ name: canary # # Three days rather than one on purpose: with a single page the pagination is # never exercised, which is exactly how a silently-dead page walk survives in -# this family (§11). +# this family. # # HEADLESS, on a bare runner, AND THAT IS A REAL TEST # --------------------------------------------------- @@ -37,23 +37,28 @@ name: canary # # It goes through the ENVIRONMENT rather than a commented-out --proxy line, # because env_config.py already reads exactly that variable, a credential -# must never reach a command line (§3), and an unset secret arrives as an +# must never reach a command line, and an unset secret arrives as an # empty string which the loader treats as "not configured". One line is # correct with and without it. # -# WHAT FAILS THE BADGE AND WHAT ONLY WARNS -# ---------------------------------------- -# A block (exit 3) WARNS: Cloudflare's managed challenge tracks the address's -# recent request rate, and a shared runner's rate is not something this -# workflow controls, so a red badge would say something about the runner -# rather than about this code. A check that is always red teaches everyone to -# ignore checks. But a warning must not read as a pass either, so every -# warning path also writes to the step summary saying in words that the -# canary did not test anything today. +# WHAT FAILS THE BADGE +# -------------------- +# Only a run that actually read Medium is green. A block (exit 3), a run that +# never got its pages (exit 5), zero rows (exit 4), or a coverage floor being +# missed all FAIL the job. A green canary that tested nothing is worse than a +# red one: it says the site was checked when it was not. # -# Parsing zero rows from a page the site plainly served (exit 4), or a -# coverage floor being missed, DOES fail — that is the regression this exists -# to catch. +# A block is not a defect in this code, but it is not a pass either, so each +# scrape is given a second chance before it counts: on exit 3 the step waits +# two minutes and runs once more in a fresh browser. Cloudflare's challenge +# here tracks the address's recent request RATE, and a rest is what measurably +# clears it. If it is still refused after that, the job goes red and the +# annotation says it was a refusal, not a parser regression. Setting a +# MEDIUM_PROXY secret gives the canary an exit whose rate it controls. +# +# A partial run (exit 6) passes only through the data checks below: it read +# real rows from the site, and the floors judge whether the parser still +# works on them. on: schedule: @@ -101,32 +106,21 @@ jobs: # the job, not the shell's errexit. run: | set +e - python3 playwright_scraper.py \ - --url "https://medium.com/tag/python/archive/${{ steps.day.outputs.date }}" \ - --pages 3 --delay 20 --retries 3 --retry-delay 30 --headless \ - --out canary_archive --format json \ - --dump-html canary_archive.html - echo "exit_code=$?" >> "$GITHUB_OUTPUT" - - - name: Interpret the archive exit code - run: | - case "${{ steps.archive.outputs.exit_code }}" in - 0) echo "::notice::Archive walk complete." ;; - 3) - echo "::warning::Blocked before parsing (exit 3) from a bare GitHub runner. Cloudflare's challenge here tracks the address's recent request RATE, which a shared runner does not control — so this says nothing about this code and the run is not failed. It also means the site-side check did NOT happen today." - echo "A runner address met Cloudflare's refusal. The parser was not exercised. Set a \`MEDIUM_PROXY\` secret to give this canary an exit whose request rate it controls; a block would then fail the job, which is the point." >> "$GITHUB_STEP_SUMMARY" ;; - 4) - echo "::error::Ran fine and parsed 0 rows (exit 4) from a tag archive that normally holds ~100 stories a day. The legacy payload (window[\"obvInit\"]) or its references shape has most likely moved — this is exactly the regression this canary exists to catch. The HTML dump in the artefacts tells you which." - exit 1 ;; - 6) - echo "::warning::Partial run (exit 6). The commonest cause here is the address being refused partway through the walk: a day archive is 2.3 MB and one address went from serving a whole day to refused inside three of them. canary_archive.meta.json carries the stop_reason and which pages failed." ;; - *) - echo "::error::Unexpected exit ${{ steps.archive.outputs.exit_code }}." - exit 1 ;; - esac + for attempt in 1 2; do + python3 playwright_scraper.py \ + --url "https://medium.com/tag/python/archive/${{ steps.day.outputs.date }}" \ + --pages 3 --delay 20 --retries 3 --retry-delay 30 --headless \ + --out canary_archive --format json \ + --dump-html canary_archive.html + code=$? + [ "$code" = 3 ] && [ "$attempt" = 1 ] || break + echo "::notice::Archive walk refused (exit 3); resting 120s and trying once more." + sleep 120 + done + echo "exit_code=$code" >> "$GITHUB_OUTPUT" - name: Sanity-check the archive run - if: steps.archive.outputs.exit_code == '0' || steps.archive.outputs.exit_code == '6' + if: always() && (steps.archive.outputs.exit_code == '0' || steps.archive.outputs.exit_code == '6') shell: python run: | import json, sys @@ -135,6 +129,8 @@ jobs: rows = json.loads(Path("canary_archive.json").read_text()) meta = json.loads(Path("canary_archive.meta.json").read_text()) fail = [] + if not rows: + fail.append("the archive run wrote no rows") # Floors, not exact figures. Three measured days were 128, 97 and # 98 stories; a floor of 40 for a whole day leaves room for a quiet @@ -168,7 +164,7 @@ jobs: # `position` restarts at 1 on every page, so the column is # worthless without `page` beside it — and a page number that was - # never threaded through is invisible to every other check (§18). + # never threaded through is invisible to every other check. pairs = {(r.get("page"), r.get("position")) for r in rows} if len(pairs) != len(rows): fail.append("page+position is not unique across the run") @@ -185,17 +181,26 @@ jobs: id: feed env: MEDIUM_PROXY: ${{ secrets.MEDIUM_PROXY }} + # Runs even when the archive half failed, so one red run still says + # which of the two paths broke. + if: always() run: | set +e sleep 30 - python3 playwright_scraper.py \ - --url "https://medium.com/tag/python" \ - --retries 3 --retry-delay 30 --headless \ - --out canary_feed --format json --dump-html canary_feed.html - echo "exit_code=$?" >> "$GITHUB_OUTPUT" + for attempt in 1 2; do + python3 playwright_scraper.py \ + --url "https://medium.com/tag/python" \ + --retries 3 --retry-delay 30 --headless \ + --out canary_feed --format json --dump-html canary_feed.html + code=$? + [ "$code" = 3 ] && [ "$attempt" = 1 ] || break + echo "::notice::Tag feed refused (exit 3); resting 120s and trying once more." + sleep 120 + done + echo "exit_code=$code" >> "$GITHUB_OUTPUT" - name: Sanity-check the tag-feed run - if: steps.feed.outputs.exit_code == '0' + if: always() && (steps.feed.outputs.exit_code == '0' || steps.feed.outputs.exit_code == '6') shell: python run: | import json, sys @@ -245,16 +250,26 @@ jobs: print("::error::%s" % f) sys.exit(1) - - name: Interpret the tag-feed exit code + - name: Verdict — green only if both halves read Medium + # Exit 0 and exit 6 have already been judged by the data checks + # above, which fail the job on their own. Everything else fails here, + # with what it means in words. + if: always() run: | - case "${{ steps.feed.outputs.exit_code }}" in - 0) echo "::notice::Tag feed complete." ;; - 3) echo "::warning::Tag feed blocked (exit 3) — see the note on the archive step." ;; - 4) - echo "::error::Parsed 0 rows from a tag feed the site served. The Apollo state or its Post shape has moved." - exit 1 ;; - *) echo "::warning::Tag feed exited ${{ steps.feed.outputs.exit_code }}." ;; - esac + fail=0 + for half in "archive:${{ steps.archive.outputs.exit_code }}" "feed:${{ steps.feed.outputs.exit_code }}"; do + name=${half%%:*}; code=${half#*:} + case "$code" in + 0) echo "::notice::$name: complete." ;; + 6) echo "::warning::$name: partial run (exit 6); its rows passed the data checks, see the meta.json artefact for which pages failed." ;; + 3) echo "::error::$name: blocked (exit 3) on both attempts. Medium's Cloudflare refused this runner's address, so the parser was NOT tested today. This is an access failure, not a parser regression; a MEDIUM_PROXY secret gives the canary an exit whose request rate it controls."; fail=1 ;; + 4) echo "::error::$name: ran and parsed 0 rows (exit 4) from a listing that normally holds dozens of stories. The payload shape has most likely moved; the HTML dump in the artefacts tells you which."; fail=1 ;; + 5) echo "::error::$name: the pages were never obtained (exit 5): a navigation timeout or a refused scroll batch with nothing gathered."; fail=1 ;; + "") echo "::error::$name: the scrape step did not run."; fail=1 ;; + *) echo "::error::$name: unexpected exit $code."; fail=1 ;; + esac + done + exit $fail - name: Upload what the run actually saw # On EVERY run, passing or not: a canary that only keeps its diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7a62099..35329e6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,83 +49,6 @@ jobs: pip install pytest pytest -q - - name: Every CLI answers --help - # A crash in argparse setup or a bad local import is invisible to the - # test suite but breaks the first thing any user types. The engine - # libraries are deliberately NOT installed here, so an ImportError - # naming one of them is expected; anything else is a failure. - run: | - python - <<'EOF' - import subprocess, sys - - ENGINE_LIBS = ("playwright", "pyppeteer", "selenium", "webdriver_manager") - failed = [] - - for script in ["playwright_scraper.py", "puppeteer_scraper.py", - "selenium_scraper.py", "scraper_api_client.py", - "fingerprint_client.py", "env_config.py", - "diff_runs.py"]: - r = subprocess.run([sys.executable, script, "--help"], - capture_output=True, text=True) - if r.returncode == 0: - print(f"ok {script}") - continue - blob = r.stdout + r.stderr - if "ModuleNotFoundError" in blob and any(l in blob for l in ENGINE_LIBS): - print(f"skipped {script} (engine library not installed here)") - continue - print(f"FAILED {script}\n{blob}") - failed.append(script) - - sys.exit(1 if failed else 0) - EOF - - - name: Committed sample output is real - # The README links to these two files, so a push without them ships - # broken links. And a sample invented by hand drifts from the schema and - # teaches readers the wrong field names, so these are cut from a real - # run rather than written by hand. - run: | - python - <<'EOF' - import csv, json, sys - from pathlib import Path - - MARKERS = ("sample-product-", "example brand", "sample product", - "product description text", "lorem ipsum") - - fail = [] - for name in ("sample_output.json", "sample_output.csv"): - if not Path(name).is_file(): - fail.append(f"{name} is missing — regenerate it from a real run") - - if not fail: - rows = json.loads(Path("sample_output.json").read_text()) - if not rows: - fail.append("sample_output.json is empty") - blob = json.dumps(rows).lower() - hit = [m for m in MARKERS if m in blob] - if hit: - fail.append(f"sample_output.json looks fabricated: {hit}") - - # Column set must match what the writers actually produce. - from output_writer import Product - from dataclasses import asdict - expected = list(asdict(Product()).keys()) - for i, row in enumerate(rows): - if list(row.keys()) != expected: - fail.append(f"row {i} columns drifted from output_writer.Product") - break - - with open("sample_output.csv", newline="", encoding="utf-8") as f: - header = next(csv.reader(f)) - if header != expected: - fail.append("sample_output.csv header drifted from output_writer.Product") - - for line in fail: - print("FAILED:", line) - sys.exit(1 if fail else 0) - EOF - - name: Config reporter runs with no .env present run: python env_config.py @@ -151,10 +74,10 @@ jobs: # endpoint, a planted 32-hex key and a planted http proxy URL. run: python3 .github/ci_checks.py --secret-check - - name: The repo's own --help and sample checks agree with this workflow - # ci_checks.py implements all three checks. The two above are also - # done as workflow steps for the clearer failure output; running them - # here as well is what keeps the two from drifting apart again. + - name: Every CLI answers --help, and the committed sample is real + # One implementation, in ci_checks.py, rather than inline copies here + # that drift from it (an inline copy is the one place an import of + # the wrong row class can hide until the first push). run: python3 .github/ci_checks.py --help-check --sample-check engine-smoke: diff --git a/.gitignore b/.gitignore index ebb01ac..41a3d18 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,20 @@ _*.py *_debug.html *_debug.png *.html.page* +# `--dump-html X --pages N` writes X.page1..X.pageN whatever X is called, and +# `--dump-html X.html --pages 1` writes X.html wherever it was asked to. A +# top-level .html is never part of this repo, so all of them are ignored, and +# so are the kinds of directory a live session drops captures into. +*.page[0-9]* +/*.html +live/ +out/ +output/ +runs/ +results/ +tmp/ +scratch/ +dumps/ # --- captures kept for parser work --------------------------------------- # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py); @@ -82,7 +96,7 @@ secrets* # recoverable only because that repo was private at the time. On 2026-09-17 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC # repository, one `git add -A` from publication. Craigslist was fixed then -# and the fix never reached anyone else, which is §16's rule unapplied: when -# you fix one, check the siblings. +# and the fix never reached anyone else: when you fix one, check the +# siblings. .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 804628b..1991b83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,20 @@ empty output file. ## [Unreleased] +> **Behaviour changes in this batch:** +> - `--pages 0`, `--concurrency 0`, and a negative `--delay`, `--retry-delay` +> or `--proxy-block-retries` are now usage errors (exit 2) naming the flag, +> as is a negative `--retries` on `scraper_api_client.py`. +> - Output files are written to a temporary file and renamed into place. A +> new file gets the mode `open()` would have given it (`0o666 & ~umask`); an +> existing one keeps its mode. +> - CSV cells beginning with `=`, `+`, `-`, `@`, a tab, CR or LF get a leading +> apostrophe (CSV only; JSON is unchanged, numbers are never touched) and +> the sidecar records `csv_cells_escaped`. +> - A page classified `empty` is no longer parsed, so it contributes 0 rows. +> - `--category` is documented as accepted and unused, and warns when passed. +> - The canary is red when it did not read Medium (exit 3, 4 or 5). + > **Behaviour change:** `--retries 0` is now a usage error (exit 2). It used > to be accepted and made the page-load loop unreachable, so nothing was > fetched and the engine reported the browser's own blank page as a result: @@ -20,6 +34,42 @@ empty output file. ### Fixed +- **A paid solve could be bought once per retry.** The check right after + navigation called the solver with no budget, so one page with no proxy pool + could buy four solves against a limit of one (measured with detection forced + and the solver stubbed). One `page_flow.SolveBudget` per page now covers both + call sites in all three engines, is checked before each purchase and is not + reset by a retry or relaunch. The README no longer says nothing can ever be + charged: a rendered reCAPTCHA, not seen on Medium in testing, is the one paid + path, at most once per page. +- **An interrupted write destroyed the previous output.** `out.json`, + `out.csv` and `out.meta.json` were truncated in place; a crash mid-write + left a 13-byte invalid JSON or an empty CSV. Writes are now atomic. +- **Spreadsheet formulas from story text reached the CSV unchanged.** +- **The parse ignored the state policy.** `should_parse` gated the wait and the + scroll but not the parse, so a page classified `empty` emitted rows. +- **The HTTP status never reached the classifier** in Playwright and pyppeteer, + so a bare 403/429/5xx with no recognisable body was not read as a refusal. + Selenium exposes no status and still classifies on the body. +- **The credential scan read nothing from a clone under a `venv` directory** + and reported it clean. Exclusions are now judged relative to the repo, a + scan of zero files fails, and data files (`.json`, `.html`, `.csv`, ...) and + renamed `.env.*` files are read too; a hit in a git-ignored file is a warning. +- **Multi-page and top-level HTML dumps were not git-ignored** (`x.page3`, + `root.html`, `live/`, `out/` and similar). +- **Comments described another site**, two cited a file that does not exist, + and the sample check counted a column `Post` does not have. +- **A 401 from `--cdp-endpoint` claimed the login lasts "about a day".** That + figure was never measured; the message now says the credentials were + refused. +- **The README's exit 5 row** now says what 5 means here: nothing was + obtained (remote API error, navigation timeout, dead proxy, refused batch). +- **Public files cited private notes by section number.** Removed, with a + suite check that fails if one comes back. +- `tests.yml` no longer carries inline copies of the `--help` and sample + checks; it runs `.github/ci_checks.py`, and the suite fails if a workflow + imports a local module inline. + - **`--retries 0` fetched nothing and called it a result.** The load loop is `range(1, retries + 1)`, so 0 never runs it and the engine judged `about:blank`. Three third-party audits of this family ran exactly that and @@ -201,7 +251,7 @@ contract and no tests. Nothing of that survives except the licence. archive days, and **55 rows from an author page against a local browser's 10**: that feed extends by scrolling, the scroll is refused from an ordinary address and is not refused from the Scraping Browser's exit. -- **`--fingerprint` works, and the five defects §16 names are all absent** — +- **`--fingerprint` works, and the five known fingerprint defects are all absent** — the user agent reaches the browser, the locale is derived by LANGUAGE (`jp` → `ja-JP`, not `jp-JP`), the timezone is applied, `--tags Windows` is accepted, and a 401 carries no key. One NEW defect of the same family diff --git a/README.md b/README.md index 2c71e8f..7a8bed1 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ A Medium story scraper. Reads a tag feed, a tag's day archive, an author's profile or a single story, and writes JSON and CSV with one row per story. -Four engines — Playwright (primary), Selenium, pyppeteer, and 2Captcha's -Scraping Browser over CDP. All four produce the same rows, the same exit +Four engines — Playwright (primary), Selenium, pyppeteer, and the 2Captcha +Scraping Browser API over CDP. All four produce the same rows, the same exit codes and the same run metadata. ```bash @@ -206,8 +206,7 @@ script runs, and 2Captcha solves what comes out as `TurnstileTaskProxyless`. measurement above rather than any limit of the solver: the challenge here is transient — 9 of 27 first attempts met it, and all 9 were served in full on the next attempt in a fresh context. A retry costs nothing and a solve costs -money, so this scraper never spends one and nothing is ever charged for a -block here. If you need it on a site where the challenge is not transient, +money, so this scraper never sends that challenge to a solver. If you need it on a site where the challenge is not transient, `foodpanda-scraper` in this family implements the interception and solves it. --- @@ -224,7 +223,7 @@ measured on 2026-09-16, and it is not "getting in": | path | what it got | what it costs | |---|---|---| | **Scraper API** (`scraper_api_client.py`) | all four modes, HTTP 200, rows **identical** to a local browser's — including the whole 2.3 MB archive day in one request | $0.0005 a request, no browser at all | -| **Scraping Browser** (`--cdp-endpoint`) | 254 rows over two archive days; **55 rows from an author page against a local browser's 10** | one live connection per profile | +| **2Captcha Scraping Browser API** (`--cdp-endpoint`) | 254 rows over two archive days; **55 rows from an author page against a local browser's 10** | one live connection per profile | The author-page figure is the interesting one. An author's feed extends by scrolling, and from an ordinary address that scroll adds nothing — the POST @@ -237,7 +236,10 @@ straight into `--mode post`. **Captcha solving specifically buys nothing here.** Medium's refusal is Cloudflare's managed challenge, which publishes no sitekey, so this scraper -never spends a solve on a block and nothing is ever charged for one. +never sends it to a solver. The only paid path is a reCAPTCHA widget +rendered on the page, which was not seen on Medium in testing: with a key +set it buys **at most one solve per page**, however many times the page is +retried. And the thing that actually limits a long run is **rate**: an archive walk pulls 2.3 MB per day URL, and one address went from serving a whole day to @@ -275,7 +277,7 @@ identical output and exit codes. | 2 | bad usage | | 3 | blocked | | 4 | zero stories | -| 5 | remote API error | +| 5 | nothing was obtained: a remote API error, a navigation timeout, a dead or unauthenticated proxy, a refused scroll batch with no rows gathered | | 6 | partial run | A run that finds nothing **writes nothing** — last night's good output is not diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 67e965e..dba40cc 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -54,8 +54,9 @@ engines relaunch the context between attempts for exactly this reason. **Do not buy a captcha solve for either.** A managed challenge carries no sitekey at all — measured 0 `data-sitekey` attributes and 0 iframes, with `cType: 'managed'` in Cloudflare's own config. There is nothing to hand -2Captcha, so this scraper never spends a solve on a block and nothing is ever -charged for one. +2Captcha, so this scraper never sends a block to a solver. (A rendered +reCAPTCHA widget, not seen on Medium in testing, is the only paid path, and +it is limited to one solve per page.) **What does exhaust an address is volume.** A day archive is 2.3 MB, and one address went from serving a whole day to the hard WAF refusal inside three of diff --git a/captcha_solver.py b/captcha_solver.py index d6251ff..4324f68 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -13,15 +13,16 @@ occurrences and 15 to 17 of `recaptcha` — wired, mounted, never challenged. That is precisely why neither string is in `product_parser.BOT_CHALLENGE_MARKERS`: a marker present on every good page -is worse than no marker (§18), and including it made a perfectly good 364 KB +is worse than no marker, and including it made a perfectly good 364 KB tag feed report "challenge: recaptcha" before it was measured. What Medium serves when it refuses is Cloudflare's MANAGED challenge, and a managed challenge has no widget and no sitekey at all — `cType: 'managed'`, 0 `data-sitekey` attributes, 0 iframes on the one measured. So there is nothing here for a solver to answer, and `page_flow.STATE_POLICY` marks that -state retry-but-do-not-solve: nothing is ever charged for a block on this -site. This module is a contingency, not part of the happy path — a bot +state retry-but-do-not-solve: a block on this site is never sent to a +solver, and a rendered reCAPTCHA (the one paid path) gets at most one solve +per page. This module is a contingency, not part of the happy path — a bot manager's configuration can change between deploys, and a scraper that cannot name what stopped it is much harder to fix. diff --git a/env_config.py b/env_config.py index 981def2..9a62a9d 100644 --- a/env_config.py +++ b/env_config.py @@ -67,7 +67,7 @@ # A literal-only check reported both of those as CONFIGURED, so `cp # .env.example .env` followed by a run connected to cb.2captcha.com with the # string `{login}-zone-…` as its username and got a 401 — a confusing failure -# a long way from its cause, which is precisely what §3's rule exists to +# a long way from its cause, which is precisely what the placeholder rule exists to # prevent. Any `{…}` left in a value means the line was never filled in. _BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}") @@ -204,7 +204,7 @@ def _dests_given_on_argv(dests, argv=None): headful path and its Scraping Browser path are both first-class, and switching between them is routine rather than exceptional. - Scanning argv is the smallest fix that keeps §3's precedence intact -- + Scanning argv is the smallest fix that keeps the precedence intact -- explicit flag, then exported variable, then `.env`, then default -- with "explicit" now meaning what the caller typed rather than what happens to be truthy. diff --git a/fingerprint_client.py b/fingerprint_client.py index 7037176..736801d 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -113,7 +113,7 @@ def _cache_path(cache_dir: str, params: dict, generate: bool, fingerprint 5393493 from disk and raised nothing, because a real key had fetched the same parameters earlier. A user whose fingerprint subscription lapsed would see `--fingerprint` keep working on their own - machine and 401 on a fresh one (§16: a path that looks like it works). + machine and 401 on a fresh one (a path that looks like it works). Keying on the key costs one extra fetch per key and makes a 401 a 401. """ @@ -264,7 +264,7 @@ def playwright_context_kwargs(fp: dict) -> dict: # 2026-09-11: a fingerprint stating `deviceScaleFactor: 1.25` produced a # browser reporting `window.devicePixelRatio === 1`, so the identity # contradicted itself on an axis any fingerprinter reads for free. Same - # shape as the three defects §16 lists -- a key the API returns that + # shape as three earlier fingerprint defects -- a key the API returns that # nothing applies -- and found the same way, by comparing the live # browser against the fingerprint rather than by reading the code. scale = (fp.get("screen") or {}).get("deviceScaleFactor") @@ -345,7 +345,7 @@ def main() -> int: # loader. It used to read `os.environ` alone, which meant a key put in # `.env` exactly as README and .env.example instruct worked for every # engine and failed here with "No API key" — a documented mechanism not - # applied on one path, which is the shape of half the defects §16 lists. + # applied on one path, which is the shape of half the fingerprint defects found so far. # `.env` has to be LOADED before it can be read: `env_value` looks at # os.environ, which `load_env` is what fills from the file. Calling it # here rather than relying on an engine having called it is the whole diff --git a/make_fixtures.py b/make_fixtures.py index 1ed78e7..a001660 100644 --- a/make_fixtures.py +++ b/make_fixtures.py @@ -45,7 +45,7 @@ Medium stories are bylined public writing, but republishing a named person's prose in a scraper's test corpus is a separate act from the site showing it -on its own page (§10, where a sibling repo committed a real customer's +on its own page (where a sibling repo committed a real customer's review, name and photo ids). The STRUCTURE is what the checks need — the payload's keys, the URL shapes, the ids, the clap and response counts, the reading times, the tags, the timestamps, the paywall flags — and all of that @@ -375,7 +375,7 @@ def _build(name, html, url, keep, names): # every page Medium serves — refused and served alike — which is exactly # why `challenge-platform` is not in any marker set, and the suite # asserts that fact against these fixtures. A fixture without it would - # let the marker back in unnoticed (§18). + # let the marker back in unnoticed. cf = _CF_SCRIPT_RE.search(html) if cf: parts.append(cf.group(0)) @@ -449,7 +449,7 @@ def depersonalise(text, names): # credentials and they expired the moment the page was fetched, but a # 32-hex run in a public repo reads as a live key to every scanner that # looks — including this repo's own — and the checks need the SHAPE of - # the script, not the value (§10). + # the script, not the value. # Replaced with a NON-HEX run, not with zeros: a 32-character string of # zeros is still 32 hex characters and still trips the repo's own # credential grep, which is how the first version of this scrub passed diff --git a/output_writer.py b/output_writer.py index 4edac36..236f596 100644 --- a/output_writer.py +++ b/output_writer.py @@ -25,12 +25,12 @@ currency, no discount, no stock and no brand, and there is no measurement to write down beside those names because there is nothing on the page to measure. Six columns null on every row of every run of every mode is exactly -what §9 says must not exist, so they are not here. +a column that should not exist, so they are not here. What IS kept, byte-identical and in order, is the family prefix — `source`, `scraped_at`, `url`, `sku`, `title` — so one column name works across the family and a consumer reading six of these repos reads the same first five -columns in the same order (§9). +columns in the same order. Two family columns are absent for measured reasons rather than definitional ones, and those measurements belong here: @@ -53,11 +53,15 @@ Everything below is row-class-agnostic: pass `row_cls` so an empty CSV still gets the right header for the mode that produced it. """ +import contextlib import csv import json +import os +import tempfile +import threading from dataclasses import dataclass, asdict, field, fields from datetime import datetime, timezone -from typing import Optional, List, Set, Sequence, Any, Type +from typing import Optional, List, Set, Sequence, Any, Type, Tuple # The host a row came from. It genuinely varies on Medium: a story lives on @@ -81,7 +85,7 @@ class Post: # imported or cross-posted story points at ANOTHER SITE — 4 of the 20 # entries in one tag feed's JSON-LD pointed at habr.com and dev.to. A # parser that trusts the canonical writes habr.com into a Medium row - # while every other column still looks right (FINDINGS.md §5). + # while every other column still looks right. # `canonical_url` below carries it where it differs, which is a genuinely # useful column and not a substitute for this one. url: str = "" @@ -124,8 +128,8 @@ class Post: # NOT `virtuals.recommends`, which sits beside it, is populated on 128 of # 128 archive posts, and is the retired pre-2017 recommend count: 25 # against a `totalClapCount` of 248 on the same story. Reading it would - # have filled this column completely and wrongly, which is the failure - # §10 exists to catch. + # have filled this column completely and wrongly, which is why the + # suite asserts values and not coverage. claps: Optional[int] = None responses: Optional[int] = None # Minutes, as Medium computes them — a float, not a rounded "15 min read". @@ -172,7 +176,7 @@ class Post: # The fetch this row came from, and its position within it. Unique as a # pair across a run; `smoke_test.py` asserts it, because `position` # restarts at 1 on every page and the column is worthless without `page` - # beside it (§18). + # beside it. page: Optional[int] = None position: Optional[int] = None @@ -203,7 +207,7 @@ def dedupe_by_key(rows: Sequence[Any], seen: Set[str], key: str = "sku") -> List than its siblings do: a scroll batch re-parses the WHOLE feed, cards already read included, so every batch after the first arrives mostly duplicate by design. A batch that drops all of its rows is the signal - that the feed is exhausted, which is §7's data-based terminating + that the feed is exhausted, which is the data-based terminating condition and the only one available here. A row with no key is always kept: there is nothing to check a duplicate @@ -244,12 +248,95 @@ def _csv_value(v: Any) -> Any: return v +# A cell starting with one of these is executed by Excel, Google Sheets and +# LibreOffice. Every string in a row was written by whoever wrote the story: +# titles, subtitles, author names and tags are all third-party text. +CSV_FORMULA_LEADS = ("=", "+", "-", "@", "\t", "\r", "\n") + +# Spreadsheets treat a leading apostrophe as "the rest is text" and do not +# show it; a consumer parsing the CSV with `csv` sees it, which is why the +# count goes in the sidecar instead of staying silent. +CSV_FORMULA_ESCAPE = "'" + + +def _csv_escape(v: Any) -> Tuple[Any, bool]: + """Neutralise a formula-shaped cell. Returns (value, was_escaped). + + Only STRINGS are touched: turning the number -5 into the text '-5 would + break every sum a consumer writes over the column. CSV only — the JSON + output keeps the site's bytes, and `csv_cells_escaped` in the sidecar + declares that the two files differ. + """ + if isinstance(v, str) and v.startswith(CSV_FORMULA_LEADS): + return CSV_FORMULA_ESCAPE + v, True + return v, False + + +_UMASK_LOCK = threading.Lock() + + +def _current_umask() -> int: + """The process umask. Python can only read it by setting it, so it is 0 + for an instant; the lock keeps two writers from interleaving that.""" + with _UMASK_LOCK: + umask = os.umask(0) + os.umask(umask) + return umask + + +@contextlib.contextmanager +def _atomic(path: str, newline: Optional[str] = None): + """Write to a temporary file beside `path`, then rename over it. + + Every write here replaces a file a previous run may have left, and the + invariant this module exists to protect is that a bad run never destroys + the last good data (`save` refuses an empty result for the same reason). + Writing in place gives that up at the worst moment: a kill, a full disk + or a crash halfway through `json.dump` leaves a TRUNCATED file where a + complete one was, and the sidecar beside it still describes the old run. + + The temporary file lives in the SAME directory, because a rename across + filesystems is not atomic and degrades to an interruptible copy. `fsync` + before the rename is what makes it hold after a power loss and not only + after a crash. + + NamedTemporaryFile creates the file 0600 and a rename keeps that. An + existing destination keeps its own mode (someone may have tightened it on + purpose); a new one gets exactly what `open(path, "w")` would have given + it, `0o666 & ~umask`. A fixed 0644 would be wrong both ways: wider than a + 077 umask asks for, narrower than a 002 one. + """ + directory = os.path.dirname(os.path.abspath(path)) or "." + handle = tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", newline=newline, dir=directory, + prefix=os.path.basename(path) + ".", suffix=".tmp", delete=False) + try: + with handle: + yield handle + handle.flush() + os.fsync(handle.fileno()) + try: + mode = os.stat(path).st_mode & 0o777 + except OSError: + mode = 0o666 & ~_current_umask() + os.chmod(handle.name, mode) + os.replace(handle.name, path) + except BaseException: + # Leave the destination untouched: a failed write must not be visible. + try: + os.unlink(handle.name) + except OSError: + pass + raise + + def write_json(rows: Sequence[Any], path: str) -> None: - with open(path, "w", encoding="utf-8") as f: + with _atomic(path) as f: json.dump([asdict(r) for r in rows], f, ensure_ascii=False, indent=2) -def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: +def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> int: + """Write the rows; return how many cells were formula-neutralised.""" # An empty result still gets the header row. A zero-byte file makes a # consumer fail on read (no columns to parse) instead of reading a valid # table with zero rows — and "an empty result is still a well-formed @@ -258,11 +345,23 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: # The header comes from `row_cls`, not from the first row, so an empty # run still writes the columns of the mode that produced it. fieldnames = [f.name for f in fields(row_cls)] - with open(path, "w", encoding="utf-8", newline="") as f: + escaped = 0 + # newline="" is the csv module's requirement and has to survive the move + # into the temp file, or every row gains a stray CR on Windows. + with _atomic(path, newline="") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() for r in rows: - writer.writerow({k: _csv_value(v) for k, v in asdict(r).items()}) + row = {} + for k, v in asdict(r).items(): + # Escape AFTER `_csv_value`: a list joined into one cell is + # text too, and its first element can be formula-shaped + # while the list itself is not a string. + value, was_escaped = _csv_escape(_csv_value(v)) + row[k] = value + escaped += was_escaped + writer.writerow(row) + return escaped # Exit code used when a run completes but produced nothing. Distinct from 1 @@ -274,19 +373,17 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: # search genuinely matched nothing" from "something stood between us and the # content". See product_parser.detect_bot_challenge. # -# On this site this code specifically does NOT cover the three ways to get a -# real page with no products on it: a `/p/` discovery hub, which -# answers 200 with banners and carousels and no stories; a tag whose feed -# matches nothing ("Oops, produk nggak ditemukan"); and one page past the -# end of a category listing. All three are EXIT_NO_PRODUCTS — the request -# was served exactly as asked and simply has no products on it. Reporting -# any of them as blocked would send a user hunting for a proxy problem that -# does not exist. +# On this site this code specifically does NOT cover a real page with no +# stories on it: a tag whose feed is empty, an author with nothing published, +# or a day archive the site redirected up to its month view. Those are +# EXIT_NO_PRODUCTS — the request was served exactly as asked and simply holds +# nothing. Reporting any of them as blocked would send a user hunting for a +# proxy problem that does not exist. # -# What EXIT_BLOCKED means here is unusually literal: this site refuses a -# address it has scored NOTHING at all. No status code, no interstitial, no -# vendor marker — the HTTP/2 stream is reset and the run sees a connection -# error rather than a page. +# What EXIT_BLOCKED means here is one of Medium's two Cloudflare refusals, +# both HTTP 403: the WAF block page ("Sorry, you have been blocked", which +# carries no widget at all) and the managed challenge ("Just a moment..."). +# See the README's "What Medium does when it refuses". EXIT_BLOCKED = 3 # Exit code for a run that gathered SOME rows and then stopped early — a @@ -347,7 +444,7 @@ def write_run_meta(out_prefix: str, meta: dict) -> str: both complete, and between runs of different `mode`. """ path = f"{out_prefix}.meta.json" - with open(path, "w", encoding="utf-8") as f: + with _atomic(path) as f: json.dump(meta, f, ensure_ascii=False, indent=2) print(f"[+] Wrote run metadata -> {path} (status={meta.get('status')})") return path @@ -382,7 +479,7 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, in archive mode — the DAYS a run covered and any day that redirected. A redirect is the thing worth recording: a tag day with no stories does not 404, it redirects up to the month view, which is a different - renderer holding a different set of stories (FINDINGS.md §4). + renderer holding a different set of stories. `pages_failed` lists the pages that did not yield data, by number. `pages_completed` alone was enough only while pages were fetched strictly @@ -406,14 +503,15 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, } if extra: # Merged rather than nested under a key, so a consumer reads - # `shop_rating` at the top level beside `products`. Run fields win a + # `tag_total_posts` at the top level beside `products`. Run fields win a # name collision: a caller cannot accidentally overwrite `status`. meta.update({k: v for k, v in extra.items() if k not in meta}) return meta def save(rows: Sequence[Any], out_prefix: str, fmt: str, - allow_empty: bool = False, row_cls: Type = Product) -> int: + allow_empty: bool = False, row_cls: Type = Product, + stats: Optional[dict] = None) -> int: """Write JSON/CSV and return a process exit code. Returns 0 when rows were written, EXIT_NO_PRODUCTS when there were none. @@ -429,6 +527,9 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, `allow_empty=True` is for the legitimate case: a filter that genuinely matches nothing, where an empty file is the correct output. + + Facts about the write go into `stats` rather than the return value, + because the return IS the exit code every engine exits with. """ if not rows and not allow_empty: print(f"[!] 0 products — refusing to write {out_prefix}.json/.csv, so an " @@ -440,8 +541,15 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, write_json(rows, f"{out_prefix}.json") print(f"[+] Saved {len(rows)} products -> {out_prefix}.json") if fmt in ("csv", "both"): - write_csv(rows, f"{out_prefix}.csv", row_cls=row_cls) + escaped = write_csv(rows, f"{out_prefix}.csv", row_cls=row_cls) print(f"[+] Saved {len(rows)} products -> {out_prefix}.csv") + if stats is not None: + stats["csv_cells_escaped"] = escaped + if escaped: + print(f"[!] {escaped} CSV cell(s) began with a formula character " + f"and were prefixed with {CSV_FORMULA_ESCAPE!r} so a " + f"spreadsheet reads them as text. The JSON output is " + f"unchanged — see csv_cells_escaped in the sidecar.") return 0 if rows else EXIT_NO_PRODUCTS @@ -508,7 +616,9 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, # names has a hole for every name nobody added to it. complete = stop_reason in COMPLETE_STOP_REASONS and not pages_failed row_cls = ROW_CLASS_BY_MODE.get(mode, Product) - rc = save(rows, out_prefix, fmt, allow_empty=allow_empty, row_cls=row_cls) + save_stats: dict = {} + rc = save(rows, out_prefix, fmt, allow_empty=allow_empty, row_cls=row_cls, + stats=save_stats) wrote_output = bool(rows) or allow_empty if wrote_output: @@ -519,12 +629,14 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, pages_requested=pages_requested, pages_completed=pages_completed, pages_failed=pages_failed, mode=mode, source=source, start_url=start_url, final_url=final_url, products=len(rows), - extra=extra)) + # The caller's `extra` wins a collision: housekeeping must never + # silently drop site data. + extra={**save_stats, **(extra or {})})) if not rows: # Nothing gathered at all, and WHY decides the code. The three # outcomes are genuinely different things and a pipeline branches on - # them (§8: blocked is not empty is not partial): + # them (blocked is not empty is not partial): # # blocked something stood between the run and the content # did not complete we never reached the site — a dead proxy, a diff --git a/page_flow.py b/page_flow.py index dcfb3f0..e33371e 100644 --- a/page_flow.py +++ b/page_flow.py @@ -2,7 +2,7 @@ Medium answers a request five ways, and four of them want a different response, which is why this module exists rather than the same triage being -written three times inside three engines and drifting apart (§1): +written three times inside three engines and drifting apart: content one of Medium's two payloads describes stories, or its own cards are rendered on a page built out of its own assets @@ -42,7 +42,7 @@ page_height() -> Optional[int] document.body.scrollHeight sleep(ms) -> None wait -No JavaScript crosses that boundary in either direction (§1): Selenium's +No JavaScript crosses that boundary in either direction: Selenium's `execute_script` takes a function BODY with an explicit `return` where Playwright and pyppeteer take `() => expr`, so this module names the OPERATION and each engine spells it in its own driver's dialect. @@ -69,7 +69,7 @@ # --------------------------------------------------------------------------- READY_SELECTOR = SELECTORS["item_card"] -# Above 1, per §5: waiting for a single match resolves on the page's own +# Above 1 on purpose: waiting for a single match resolves on the page's own # first card long before a feed paints. Measured first paints: 15 cards on a # tag feed, 10 on an author profile, 128 on a day archive, 1 on a post page — # which is why `min_matches` clamps for the mode that genuinely has one. @@ -117,7 +117,7 @@ def wait_for_count(count: Callable[[str], int], sleep: Callable[[int], None], under any CSP and spells the same in all three drivers. Returns the count it ended on, whether or not it reached the floor: - reporting a timeout is not the same as exiting on one (§8). + reporting a timeout is not the same as exiting on one. """ waited = 0 found = count(selector) @@ -134,7 +134,7 @@ def wait_for_count(count: Callable[[str], int], sleep: Callable[[int], None], # --------------------------------------------------------------------------- # Lazy loading # --------------------------------------------------------------------------- -# §8's second case. Scroll to `document.body.scrollHeight` rather than +# The lazy-loaded case. Scroll to `document.body.scrollHeight` rather than # wheeling a fixed distance — a fixed wheel stops short on a long grid and # the trigger is never reached — and require the count AND the height to hold # still for THREE rounds, because the next batch takes longer to arrive than @@ -189,7 +189,7 @@ def classify(html: Optional[str], status: Optional[int] = None, Getting that wrong is not a style question: a sibling repo shipped two of three engines calling this as `classify(html, url=...)`, both crashed on their first fetch, and nothing short of a live run or a signature-binding - check saw it (§17). This repo's smoke suite binds every shared-module + check saw it. This repo's smoke suite binds every shared-module call in every engine for that reason. """ if html is None: @@ -198,7 +198,7 @@ def classify(html: Optional[str], status: Optional[int] = None, # The retry/solve/blocked decision as DATA rather than as three copies of an -# if-chain in three engines (§1). +# if-chain in three engines. # # parse is there anything on this page worth writing down? # retry would fetching it again, in a fresh context, plausibly help? @@ -223,7 +223,7 @@ def classify(html: Optional[str], status: Optional[int] = None, # survives, because a challenge that never clears is a refusal, and # letting it through hands a 28 KB interstitial to the parser and reports # exit 4 — "ran fine, found nothing" — on a tag holding hundreds of - # thousands of stories. Blocked is not empty (§8). + # thousands of stories. Blocked is not empty. "challenge": {"parse": False, "retry": True, "solve": False, "blocked": True}, "blocked": {"parse": False, "retry": True, "solve": False, "blocked": True}, } @@ -267,15 +267,15 @@ def is_unpainted(state: str, html: Optional[str]) -> bool: # challenge `_cf_chl_opt = {… cType: 'managed' …}`. Transient. # # Three facts drive the constants below, and each is a constant rather than a -# paragraph an engine might not read (§17: a policy constant nothing consults +# paragraph an engine might not read (a policy constant nothing consults # is the same defect as dead code): # # * THE USER-AGENT IS THE DISCRIMINATOR, not the address and not headless # as such. A bare headless Chromium sending its default UA — the one # carrying the `HeadlessChrome` token — was WAF-blocked on 11 of 11 # attempts. The same browser with an ordinary Chrome UA was never -# WAF-blocked. The engines build their UA from the browser's own version -# (§8), so they never send that token; `curl` and a naive headless +# WAF-blocked. The engines build their UA from the browser's own version, +# so they never send that token; `curl` and a naive headless # script do, which is why they get nothing. Headful is still the default # on a smaller margin: 8/8 served against headless's 6/8, interleaved. # @@ -283,7 +283,7 @@ def is_unpainted(state: str, html: Optional[str]) -> bool: # seconds inside the same context never cleared it once. Tearing the # context down and opening a new one cleared it on the next attempt in 9 # of 9 cases. So a retry here must be a new context, which is the same -# shape as this family's "a rotation is a fresh browser" rule (§8), +# shape as this family's "a rotation is a fresh browser" rule, # applied to a case with no proxy in it. # # * IT IS NOT SOLVABLE. Measured on the interstitial Medium served: 0 @@ -296,9 +296,42 @@ def is_unpainted(state: str, html: Optional[str]) -> bool: BLOCK_RETRIES_WITH_POOL = 4 SOLVES_PER_PAGE = 1 + +class SolveBudget: + """The paid-solve allowance for ONE page, shared by every call site. + + An engine reaches the solver from two places per fetch attempt — the + check right after navigation, before the page is classified, and the + one after classification for a state that says the page is gated — and + both repeat on every block retry. A counter kept beside only the second + of them made `SOLVES_PER_PAGE` advisory: with detection forced and the + solver stubbed, one page with no proxy pool bought four solves against a + limit of one. So there is one object per page, created before the retry + loop, consulted immediately BEFORE each purchase and never reset by a + retry or a relaunch: a fresh context is a reason to re-fetch, not a + fresh allowance. + """ + + def __init__(self, limit: Optional[int] = None): + self.limit = SOLVES_PER_PAGE if limit is None else limit + self.spent = 0 + + def available(self) -> bool: + return self.spent < self.limit + + def spend(self) -> bool: + """Claim one purchase. False, and nothing claimed, when exhausted.""" + if not self.available(): + logger.info("Not buying another solve for this page: its " + "allowance of %d is spent.", self.limit) + return False + self.spent += 1 + return True + + # Whether a retry has to discard the browser context rather than reload the # page. True here on the measurement above. Consulted by all three engines; -# a constant no engine read would be the §17 defect this comment warns about. +# a constant no engine read would be dead policy. RETRY_NEEDS_FRESH_CONTEXT = True @@ -307,7 +340,7 @@ def cdp_connect_advice(error_text: str) -> str: The same sentence used to follow every connect failure, and it explained `profile_locked`. A 401 is not that: the credentials are expired or - wrong (a profile's last about a day), and a reader told to look for a + wrong, and a reader told to look for a run holding the pid goes looking in the wrong place. Classified by what the failure SAYS, not by which driver raised it. Pure text in, text out, and no secret ever goes into the answer. @@ -318,9 +351,10 @@ def cdp_connect_advice(error_text: str) -> str: """ text = (error_text or "").lower() if re.search(r"\b401\b", text) or "deny_no_user" in text or "unauthorized" in text: - return ("The endpoint refused the credentials (HTTP 401). A Scraping " - "Browser profile's login expires after about a day, so fetch " - "fresh ones from the 2Captcha dashboard and check the " + return ("The endpoint refused the credentials (HTTP 401): they are " + "expired, revoked or mistyped. How long a profile's login " + "lasts has not been measured as a fixed figure, so fetch " + "current ones from the 2Captcha dashboard and check the " "`ws://{login}-zone-scraping_browser-country-xx-pid-{id}:" "{password}@cb.2captcha.com:9222` shape. This is not " "`profile_locked`, and another pid will not help.") @@ -390,13 +424,13 @@ def block_advice(html: Optional[str], headless: bool, has_pool: bool) -> str: # `--mode archive` walks `/tag/{slug}/archive/{yyyy}/{mm}/{dd}` backwards a # day at a time. Those are real, independent addresses: 128 stories in one # fetch, `paging` empty, and page 5 knowable without fetching page 4 — which -# is what makes `--concurrency` meaningful there and nowhere else (§7). +# is what makes `--concurrency` meaningful there and nowhere else. # # The other three modes have no per-batch address at all. `?page=2` on a tag # feed is not an error, it is IGNORED, and a run built on it would add no new -# sku, call the listing exhausted and report COMPLETE holding one page (§18). +# sku, call the listing exhausted and report COMPLETE holding one page. # For those modes a "page" is one settled scroll batch, and the terminating -# condition is the DATA one §7 asks for: a batch that adds no unseen sku ends +# condition is a DATA one: a batch that adds no unseen sku ends # the listing. NEXT_BATCH_TIMEOUT_MS = 45_000 NEXT_BATCH_POLL_MS = 1_000 @@ -415,7 +449,7 @@ def advance_feed(scroll_to_bottom: Callable[[], None], Returns ADVANCED or NO_GROWTH, and the two must not collapse into one bool: a sibling repo mapped both to False, the engine mapped False to `pagination_exhausted`, and a run that was being rate-limited reported - status "complete" holding page 1 (§7). + status "complete" holding page 1. Readiness is the CARD COUNT growing. The document height is watched too but is not sufficient on its own — a page can grow by the height of a @@ -460,7 +494,7 @@ def page_gap(html: Optional[str], parsed: int) -> Optional[int]: rendered everything it had. None rather than 0, because an unknown gap is not a gap of zero and the - two must not read the same in a sidecar (§8). + two must not read the same in a sidecar. """ return None @@ -481,7 +515,7 @@ def tag_total_posts(html: Optional[str]) -> Optional[int]: # --------------------------------------------------------------------------- # A feed that did not extend is the site, not the scroll # --------------------------------------------------------------------------- -# §8's third case: "a different page was served to this SESSION — no amount +# The third kind of missing content: "a different page was served to this SESSION — no amount # of waiting or scrolling helps". On Medium the mechanism is blunter than a # session variant. The scroll fires a POST to `medium.com/_/graphql`, and # from the address these numbers were taken on every one of those POSTs came @@ -592,3 +626,26 @@ def attempts(value) -> int: f"With 0 the load loop never runs, nothing is fetched, and the " f"engine reports the browser's blank page as a result.") return number + + +def at_least(flag: str, minimum, cast=int): + """argparse `type=` for a numeric flag with a floor, naming the flag. + + `--pages 0` used to plan no page at all and `--delay -1` reached + `time.sleep` as a negative number, which raises mid-run. Both are usage + errors, and argparse should say so before anything is launched, in the + flag's own words rather than as a traceback. + """ + def parse(value): + try: + number = cast(value) + except (TypeError, ValueError): + raise _argparse.ArgumentTypeError( + f"{flag} must be a {'whole ' if cast is int else ''}number, " + f"not {value!r}") + if number < minimum: + raise _argparse.ArgumentTypeError( + f"{flag} must be at least {minimum}, not {number}") + return number + parse.__name__ = flag.lstrip("-").replace("-", "_") + return parse diff --git a/playwright_scraper.py b/playwright_scraper.py index 1769401..2e3e75a 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -26,7 +26,7 @@ WAF block page on 11 of 11 attempts. The same browser, same address, same minute, with an ordinary Chrome UA — which is what this engine builds, from the browser's OWN version — was never WAF-blocked once. This is the - family's "the user agent comes from the browser, not a literal" rule (§8) + family's "the user agent comes from the browser, not a literal" rule turning out to be the difference between data and a block page, and it is why `curl` and every naive headless script get nothing from this site. @@ -73,7 +73,7 @@ own Enterprise integration: 3-4 `g-recaptcha` and 4-17 `recaptcha` occurrences on pages known to be good, against 0 on the challenge page. The family's usual reCAPTCHA marker is a false positive here and is deliberately not in - the marker set (§18). + the marker set. * **A bundled Chromium is enough.** Headful, Playwright's own Chromium was served exactly as well as system Chrome (364 KB against 363 KB on the same @@ -196,7 +196,7 @@ class PageOutcome: # makes the difference between the two visible rather than assumed. tag_total_posts: Optional[int] = None # None, always, on this site: Medium publishes no per-page counter, and an - # unknown gap must not read the same as a gap of zero (§8). + # unknown gap must not read the same as a gap of zero. gap: Optional[int] = None # What the scroll did: how many cards it reached and whether it SETTLED. # A batch whose feed was still growing when the round budget ran out is @@ -225,7 +225,7 @@ def ok(self) -> bool: # engines make it identically. What lives here is only HOW to ask this # particular driver. # -# The primitives are NAMED OPERATIONS rather than JavaScript (§1). Selenium's +# The primitives are NAMED OPERATIONS rather than JavaScript. Selenium's # execute_script takes a function BODY with an explicit `return` while # Playwright and pyppeteer take `() => expr`, so a shared module handing JS # across this boundary would quietly acquire one driver's dialect. @@ -263,7 +263,7 @@ def _scroll_to_bottom(page) -> None: To `document.body.scrollHeight` rather than by a fixed wheel distance: a fixed wheel stopped three rounds short of the bottom on a sibling site's 7,600px grid, so the lazy-load trigger was never reached and a run took - 30 of 50 cards while looking settled (§8). + 30 of 50 cards while looking settled. Guarded for the same reason `_count` is: the page can navigate mid-loop on this site, and a scroll that raises turns a routine challenge into a @@ -288,14 +288,13 @@ def _page_height(page) -> Optional[int]: # endpoint can be refused while the HTML keeps answering 200. Counting the # refusals is what separates a feed that ran out (COMPLETE) from one whose # next batch was refused (PARTIAL) — without it the two are the same -# observation and a throttled run reports "complete" holding its first batch -# (§7). +# observation and a throttled run reports "complete" holding its first batch. # # Any status at or above 400, rather than one specific code, and the SAME # threshold in all three engines. A threshold that differed between them # would mean one engine reporting `complete` where its twins report `partial` # on the identical run, which is precisely the drift the shared modules exist -# to prevent (§6) — and this file carried a sibling repo's single-code check +# to prevent — and this file carried a sibling repo's single-code check # for several commits before the smoke suite was taught to compare the three. _GRAPHQL_PATH = "/graphql/" @@ -570,7 +569,7 @@ def _content_when_settled(page, attempts: int = 4, pause_ms: int = 700): return None -def handle_captcha_if_present(page, args) -> bool: +def handle_captcha_if_present(page, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Runs after EVERY navigation, for ANY page. The static-HTML and runtime @@ -586,7 +585,7 @@ def handle_captcha_if_present(page, args) -> bool: `challenge` as retry-but-do-not-solve and nothing is ever charged for it. The detector is still run on every navigation, and that asymmetry is - deliberate (§8). Medium HAS a captcha configured — Turnstile's loader is + deliberate. Medium HAS a captcha configured — Turnstile's loader is in the head of every page it serves and an empty `cf-turnstile-response` input sits in a 0x0 div — it simply does not render one to an anonymous reader. Which challenge a visitor meets @@ -628,6 +627,10 @@ def handle_captcha_if_present(page, args) -> bool: logger.warning("No 2captcha API key, so this challenge cannot be " "solved — continuing with whatever the page holds.") return False + if not budget.spend(): + # Checked BEFORE the purchase, and shared with the other call site: + # see page_flow.SolveBudget. + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -665,7 +668,7 @@ def _scroll_the_feed(session, args, html: str, page_num: int) -> dict: the 11 attempts measured, while the HTML itself came back 200. So the expected outcome here is "scrolled, nothing arrived", and that is reported rather than fought: more rounds against a feed the site will not - extend only spends time (§8). + extend only spends time. Deliberately no `target`. The only total Medium publishes is `tag.postCount` — 257,497 for `python` — which is the size of the tag's @@ -703,8 +706,7 @@ def _worker_pool(pool, worker_index: int): several — N workers all leaving from one address is just a faster way to burn that address. * No shared mutable state between threads, so rotation needs no lock: - the concurrency is safe by construction rather than by discipline - (§7). + the concurrency is safe by construction rather than by discipline. """ if not pool: return None @@ -778,7 +780,7 @@ def worker(index: int): # Anything still queued was never attempted (a worker died, or dispatch # stopped at the end of the listing). NOT reported as failed pages: they - # were not tried, and claiming otherwise would overstate the damage (§8). + # were not tried, and claiming otherwise would overstate the damage. unattempted = [] while True: try: @@ -808,7 +810,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: Optional[str]) -> P has_pool = bool(pool and len(pool) > 1) # `RETRY_ON_BLOCKED` is CONSULTED, not merely documented — a policy - # constant nothing reads is the same defect as dead code (§17). It is + # constant nothing reads is the same defect as dead code. It is # True on this site because a RESTED address recovers: the same URL that # was challenged was served in full on the next attempt about a minute # later. The budget is deliberately small, because a BUSY address does @@ -817,21 +819,27 @@ def _fetch_one_page(session, args, pool, page_num: int, url: Optional[str]) -> P block_retries = 0 if not page_flow.RETRY_ON_BLOCKED else ( args.proxy_block_retries if has_pool else page_flow.BLOCK_RETRIES_WITHOUT_POOL) - # Counted across the whole block-retry loop, not per attempt: a page that - # keeps coming back as a challenge would otherwise buy one solve per - # rotation, which is how a run quietly turns into a bill. - solves_bought = 0 + # ONE allowance for the whole block-retry loop and for BOTH call sites + # below: a page that keeps coming back gated would otherwise buy one + # solve per attempt, which is how a run quietly turns into a bill. + budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False for block_attempt in range(block_retries + 1): load_failed, exit_failed = False, None + # The navigation's HTTP status, handed to the classifier so a refusal + # with no recognisable body (a bare 403, 429 or 5xx) is read as one. + # None for a scroll batch, which is not a navigation. + nav_status = None if url is not None: logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) for attempt in range(1, args.retries + 1): try: - session.page.goto(url, wait_until="domcontentloaded", - timeout=60000) + response = session.page.goto( + url, wait_until="domcontentloaded", timeout=60000) + nav_status = (response.status if response is not None + else None) load_failed = False break except (PWTimeout, PWError) as e: @@ -868,7 +876,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: Optional[str]) -> P # refusing. Those are two different endings and must not # collapse into one: reporting this as the end of the # listing would make a throttled run say "complete" while - # holding its first batch (§7). + # holding its first batch. outcome.state = "no_turnover" outcome.load_failed = True outcome.final_url = session.page.url @@ -892,7 +900,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: Optional[str]) -> P # growing because the page was replaced has not run out. The # engines used to report that as `exhausted`, which is a # COMPLETE stop reason, so a run that was blocked halfway - # would have claimed the feed ended (§7). + # would have claimed the feed ended. current = _content_when_settled(session.page) or "" state_now = _classify(session.page, current) if page_flow.counts_as_blocked(state_now): @@ -923,14 +931,17 @@ def _fetch_one_page(session, args, pool, page_num: int, url: Optional[str]) -> P if load_failed: break - if handle_captcha_if_present(session.page, args): + if handle_captcha_if_present(session.page, args, budget): + # The page was reloaded after the solve, so the status belongs + # to a document that is no longer there. + nav_status = None session.page.wait_for_timeout(1000) html = _content_when_settled(session.page) or "" - state = _classify(session.page, html) + state = _classify(session.page, html, nav_status) # "Not painted yet" is not a fault, and telling it apart from one is - # the distinction §8 is about. A search page's first response is a + # the distinction that matters here. A search page's first response is a # shell — the grid arrives over client-side GraphQL — so classified # naively it reads as something to retry, and retrying a shell buys # another shell. Wait for the anchor and re-classify BEFORE the retry @@ -949,12 +960,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: Optional[str]) -> P logger.info("The grid still had not painted after %.0fs " "(%d match(es)).", wait_timeout / 1000, found) html = _content_when_settled(session.page) or html - state = _classify(session.page, html) + state = _classify(session.page, html, nav_status) - if (page_flow.should_solve(state) - and solves_bought < page_flow.SOLVES_PER_PAGE): - solves_bought += 1 - if handle_captcha_if_present(session.page, args): + if page_flow.should_solve(state) and budget.available(): + if handle_captcha_if_present(session.page, args, budget): session.page.wait_for_timeout(1000) html = _content_when_settled(session.page) or html state = _classify(session.page, html) @@ -994,7 +1003,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: Optional[str]) -> P # of them. So the retry relaunches rather than reloading — # this is what `page_flow.RETRY_NEEDS_FRESH_CONTEXT` says, # and a constant no engine consulted would be dead policy - # dressed up as enforcement (§17). + # dressed up as enforcement. # # NOT relaunched over --cdp-endpoint: a Scraping Browser # profile allows one live connection, so reconnecting risks @@ -1088,7 +1097,16 @@ def _fetch_one_page(session, args, pool, page_num: int, url: Optional[str]) -> P logger.info("Saved the snapshot the parser sees to %s (%d bytes).", dump_path, len(html or "")) - products = _parse_for_mode(html or "", session.page.url, args, page_num) + # The state policy decides whether this page is parsed at all, not only + # whether it is waited for and scrolled. A page classified `empty` is the + # site's own "nothing here" answer; parsing it anyway emitted rows from + # whatever else the page carried, and reported them as hits. + if page_flow.should_parse(state): + products = _parse_for_mode(html or "", session.page.url, args, page_num) + else: + products = [] + logger.info("Page %d is %s, which the state policy does not parse.", + page_num, state) logger.info("Parsed %d row(s) from page %d.", len(products), page_num) # The size of the tag's WHOLE catalogue, where Medium states one — 257,497 @@ -1130,7 +1148,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: Optional[str]) -> P # with reading time, word count and language); `apollo` carries claps # and responses but no reading time on a tag feed; `jsonld` and `dom` # carry neither. Reported rather than warned about, because which one - # answers is a property of the URL (§9). + # answers is a property of the URL. from collections import Counter views = Counter(p.data_source for p in products) logger.info("Rows by view on page %d: %s.", page_num, @@ -1167,7 +1185,7 @@ def scrape(args) -> int: "proxy on top would contradict it.") pool = None - # Two shapes of pagination live here, and the URL picks which (§7). + # Two shapes of pagination live here, and the URL picks which. # # archive /tag/{slug}/archive/{yyyy}/{mm}/{dd} is a real, independent # address holding a whole day — 128 stories in one fetch, @@ -1178,7 +1196,7 @@ def scrape(args) -> int: # everything else no address for batch 2 at all. A "page" is one # settled scroll batch, and `--concurrency` above 1 is # refused with that reason rather than quietly running one - # worker, which would look like the flag did something (§18). + # worker, which would look like the flag did something. by_url = paginates_by_url(args.url) concurrency = max(1, args.concurrency) @@ -1232,7 +1250,7 @@ def scrape(args) -> int: elif by_url: seen_keys.update(p.sku for p in first.products if p.sku is not None) - # VERIFY before planning (§7). A tag day with no stories does + # VERIFY before planning. A tag day with no stories does # not 404 — it redirects up to the month view, which is a # different renderer holding a different set of stories. HTTP # 200 and a page full of rows, for a day that has none. If @@ -1310,7 +1328,7 @@ def scrape(args) -> int: if p.sku is not None) if not fresh_count: # A property of the DATA, not of a selector that - # may have been renamed (§7). + # may have been renamed. logger.info("Day %s added no rows not already " "seen — treating that as the end of " "the listing.", url) @@ -1436,7 +1454,7 @@ def scrape(args) -> int: # What the TAG holds, where the site states it. Beside # `rows` in the sidecar rather than subtracted from it: the # difference is Medium rendering a fraction of a large tag, not - # a gap this run failed to close (§8 — an unknown gap is not a gap). + # a gap this run failed to close (an unknown gap is not a gap). "tag_total_posts": tag_total_posts, "inline_payload_rows": sum(1 for r in all_rows if r.data_source != "dom"), @@ -1478,10 +1496,11 @@ def parse_args(): "mode only. The data_source column says which view " "built each row.") p.add_argument("--category", default=None, - help="Label to tag output rows with. Filled from the URL " - "by default — the tag or the author — so it is rarely " - "empty.") - p.add_argument("--pages", type=int, default=1, + help="Accepted so a command line shared with the other " + "scrapers in this family still parses, and NOT used: " + "a Medium row has no category column (its tags are in " + "`tags`). A warning says so when it is passed.") + p.add_argument("--pages", type=page_flow.at_least("--pages", 1), default=1, help=f"Number of pages to walk (default 1, cap {PAGE_CAP}). " f"In --mode archive a page is one DAY, with its own " f"address, and page N is the day N-1 days earlier. " @@ -1489,13 +1508,13 @@ def parse_args(): f"`?page=2` is ignored and the feed returns its " f"first items again — so a page there is one settled " f"scroll batch.") - p.add_argument("--delay", type=float, default=3.0, + p.add_argument("--delay", type=page_flow.at_least("--delay", 0, float), default=3.0, help="Delay between batches, seconds (default 3.0). The " "rate matters more than the address on this site: " "fourteen fetches in twenty minutes from one exit " "took the Cloudflare challenge rate from one-in-four " "to three-in-three. This is the cheapest lever.") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--concurrency", type=page_flow.at_least("--concurrency", 1), default=1, metavar="N", help="Parallel workers, --mode archive only, where each " "day is an independent address (default 1). REFUSED " "above 1 everywhere else, with the reason: a tag or " @@ -1509,7 +1528,8 @@ def parse_args(): "that comes back EMPTY is not retried — see " "page_flow.STATE_POLICY — because an empty feed is " "a correct answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", + type=page_flow.at_least("--retry-delay", 0, float), default=2.0, help="Seconds before the first page-load retry, doubling " "thereafter (default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") @@ -1546,7 +1566,8 @@ def parse_args(): p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup, so concurrent runs do " "not all begin on the first exit in the file.") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", + type=page_flow.at_least("--proxy-block-retries", 0), default=2, help="When a page comes back refused, retry it from this " "many OTHER exits before giving up (default 2). Needs " "a pool of more than one; ignored otherwise. Worth " @@ -1569,7 +1590,7 @@ def parse_args(): # ONE OS-family tag, not a list — and the default is what makes # --fingerprint work at all. It shipped as "Windows,Chrome,Desktop" across # this family, which the API rejects with HTTP 400, so --fingerprint - # failed on every invocation in four repos at once (§17). + # failed on every invocation in four repos at once. p.add_argument("--fp-tags", default="Windows", help="ONE OS-family tag for the fingerprint filter: " "Windows, Microsoft Windows or Android. NOT a list — " @@ -1632,7 +1653,7 @@ def parse_args(): # `_chrome_ua` builds, from the browser's own version — was never # WAF-blocked once. This engine therefore works headless, and that is the # family's "the user agent comes from the browser, not a literal" rule - # (§8) being the difference between data and a block page. + # being the difference between data and a block page. # # Headful is still the default because it is still better: interleaved # over 8 rounds on one address, headful was served 8/8 and headless 6/8, @@ -1654,6 +1675,10 @@ def parse_args(): # Fill --twocaptcha-key / --cdp-endpoint / --proxy / --url from the # environment or .env when the flag was not given. An explicit flag wins. env_config.apply(args) + if args.category: + logger.warning("--category %r is accepted and ignored: a Medium row has " + "no category column. The story's own tags are in the " + "`tags` column.", args.category) if not args.url: p.error("no --url given, and MEDIUM_URL is not set in the environment " "or in .env.") diff --git a/product_parser.py b/product_parser.py index 8ba354a..897149a 100644 --- a/product_parser.py +++ b/product_parser.py @@ -2,7 +2,7 @@ product_parser.py ----------------- Medium extraction. THIS FILE IS THE SITE — everything else in the repo is -family core with a handful of named constants (CLAUDE.md §1). +family core with a handful of named constants. What Medium actually publishes, measured 2026-09-16 --------------------------------------------------- @@ -50,7 +50,7 @@ 4. **`challenge-platform` appears on every page Medium serves.** Twice, on good pages and refused ones alike — it is Cloudflare's ordinary script injection, not a signal. Every marker below was counted on a page known - to be good before it was allowed into the set (CLAUDE.md §18). + to be good before it was allowed into the set. """ from __future__ import annotations @@ -88,7 +88,7 @@ MEDIUM_HOSTS = ("medium.com", "www.medium.com") # Hosts that were Medium publications and are not any more. Refusing these by -# name, WITH the reason, is the §5 rule: "is not a Medium site" is false and +# name, WITH the reason, is deliberate: "is not a Medium site" is false and # sends the reader looking for a typo. LEFT_MEDIUM: Dict[str, str] = { # Measured 2026-09-16: HTTP 200, 210 KB, 25
, two JSON-LD blocks, @@ -276,7 +276,7 @@ def category_from_url(url: str) -> Optional[str]: def unsupported_reason(url: str) -> Optional[str]: """Why this URL cannot be run, in the site's own terms, or None. - Every refusal here names a MEASUREMENT. §5's rule: a wrong reason costs + Every refusal here names a MEASUREMENT. The rule: a wrong reason costs more than a missing one, because it sends the reader looking for a typo that is not there. """ @@ -318,7 +318,7 @@ def is_supported_host(url: str) -> bool: # =========================================================================== -# Pagination (§7) +# Pagination # =========================================================================== # Only the day archive is addressable, and the whole repo leans on it. # @@ -356,7 +356,7 @@ def page_url(url: str, page: int) -> Optional[str]: would LOOK like it worked: a `?page=2` on a tag feed is ignored by Medium and answers with the first items again, so a run that trusted it would find no new sku, call the listing exhausted, and report a complete run - holding one page (§18). + holding one page. """ if page <= 1: return url @@ -423,12 +423,12 @@ def concurrency_refusal(url: str) -> Optional[str]: # How many story links mean "this page has rendered". Must be > 1: waiting # for one match resolves on the site's own nav links long before a feed -# paints (§5). +# paints. MIN_CARD_MATCHES = 2 # Deliberately empty. Medium publishes no `link[rel=next]`, no numbered # anchors and no next control in any mode — the archive walk builds its own -# addresses from the date and `page_url` above, which is §7's second layer. +# addresses from the date and `page_url` above, which is the second pagination layer. NEXT_PAGE_SELECTOR = "" @@ -446,7 +446,7 @@ def count_cards(html: Optional[str]) -> int: DISTINCT, not links: a Medium card links its story twice (image and title), so counting links reports a two-card page as four and a readiness - wait for four matches resolves on two (§4). + wait for four matches resolves on two. """ if not html: return 0 @@ -473,7 +473,7 @@ def count_cards(html: Optional[str]) -> int: # `text -\x3e input_ids` — did not. The DOM fallback caught it and returned # 120 rows with titles and authors and NOTHING else, which is the shape of # this failure: not a crash, not an empty run, just a whole page quietly -# demoted to the thinnest of the four read paths (§15). +# demoted to the thinnest of the four read paths. # # Rewritten to the JSON spelling of the same character rather than decoded, # so string contents are untouched. The lookbehind counts backslashes: an @@ -539,8 +539,8 @@ def archive_years(html: Optional[str]) -> List[str]: """Which years this tag has stories in, from the site's own index. `archiveIndex.yearlyBuckets` on a day-archive page. The site publishing - its own index is why an archive walk can be planned rather than guessed - (§7) — though note the walk still stops on data, because a year having + its own index is why an archive walk can be planned rather than guessed — + though note the walk still stops on data, because a year having stories says nothing about any particular day in it. """ index = obvinit_payload(html).get("archiveIndex") or {} @@ -884,7 +884,7 @@ def _jsonld_entities(blocks: Iterable[Any]) -> List[dict]: continue # `@graph` is legal schema.org and carries the products on some # sites while `itemListElement` carries them on others; reading - # only one reports an empty category for an unread format (§4). + # only one reports an empty category for an unread format. for key in ("mainEntity", "itemListElement", "@graph"): value = item.get(key) if isinstance(value, list): @@ -896,7 +896,7 @@ def _jsonld_entities(blocks: Iterable[Any]) -> List[dict]: def _jsonld_image(value: Any) -> Optional[str]: - """All four legal shapes of schema.org `image` (§4).""" + """All four legal shapes of schema.org `image`.""" if isinstance(value, str): return value or None if isinstance(value, dict): @@ -969,8 +969,7 @@ def _card_of(anchor: Any) -> Any: a second LINK does: a Medium card links its story twice, so a walk that stopped at "more than one story link" would never leave the anchor and every row would come back with nothing but a url. Capped so a malformed - document cannot walk to and hand one card its neighbours' values - (§4). + document cannot walk to and hand one card its neighbours' values. """ node = anchor best = anchor @@ -1003,7 +1002,7 @@ def _in_card(anchor: Any) -> bool: publication's "About" tab is itself a Medium post and its nav link carries a 12-hex id, so an id-shaped href alone is NOT evidence of a story — reading one produced a row titled "About" pointing at - `medium.com/p/…` with every other column null (§4's junk-link case). + `medium.com/p/…` with every other column null (the junk-link case). """ node = getattr(anchor, "parent", None) for _ in range(_WIDEN_CAP + 4): @@ -1024,7 +1023,7 @@ def posts_from_dom(html: Optional[str], base_url: str = "") -> Dict[str, dict]: Deliberately reads only what a card actually prints. It does NOT guess at claps or dates from card text: Medium renders those inconsistently between the two renderers, and a wrong number in a populated column is - worse than a null (§8). + worse than a null. Only links INSIDE a card are read. A page whose cards have not painted contributes nothing here rather than contributing its navigation — which @@ -1100,7 +1099,7 @@ def _absolute(base_url: str, href: str) -> str: # =========================================================================== -# Page state (§8, §17, §18) +# Page state # =========================================================================== # Every marker below was COUNTED on a page known to be good before it was # allowed in. `challenge-platform` is the one that did not make it: it @@ -1126,7 +1125,7 @@ def _absolute(base_url: str, href: str) -> str: # signal. The family's usual reCAPTCHA # marker is a false positive on this site, and including it made a perfectly # good 364 KB tag feed report "challenge: recaptcha". A marker that matches -# every page is worse than no marker (§18); both entries below were counted +# every page is worse than no marker; both entries below were counted # at 0 on all four good pages before being allowed in. BOT_CHALLENGE_MARKERS: Tuple[str, ...] = ( "cf-turnstile", # 0 on good pages, 1 on the challenge @@ -1144,7 +1143,7 @@ def _absolute(base_url: str, href: str) -> str: # Medium's own asset hosts. A served page is built out of them; an # interstitial, a WordPress site that used to be a publication, and -# Chromium's own network-error page are not (§8, §18). +# Chromium's own network-error page are not. # # All three hosts are needed, not one: the legacy archive renderer ships ZERO # `cdn-client.medium.com` references, so anchoring on that alone would report @@ -1161,7 +1160,7 @@ def _absolute(base_url: str, href: str) -> str: # injects its own turnstile hunter into every page it loads, so `cf-turnstile` # appears in the markup of a perfectly good listing page fetched over # --cdp-endpoint. This repo's marker set CAN match one, which is the -# condition CLAUDE.md §8 sets for adding the filter — the siblings whose sets +# condition for adding the filter — the siblings whose sets # cannot were checked and deliberately left alone. _EXTENSION_SCRIPT_RE = re.compile( r"]+src=[\"'](?:chrome|moz)-extension://[^\"']*[\"'][^>]*>" @@ -1194,7 +1193,7 @@ def detect_bot_challenge(html: Optional[str], url: str = "") -> Optional[str]: Only ever REFINES a verdict the page-state policy has already reached: it is not consulted for a page that classified as `content` or `empty`, because a detection on a page whose stories are already parsed guards - nothing, and one on a correct empty answer is simply wrong (§18). + nothing, and one on a correct empty answer is simply wrong. """ lowered = strip_extension_scripts(html).lower() if any(marker in lowered for marker in BOT_CHALLENGE_MARKERS): @@ -1216,7 +1215,7 @@ def detect_block_marker(html: Optional[str]) -> Optional[str]: # Medium's own empty answers, in the copy it actually prints. A POSITIVE # signal only: a feed with no stories and none of this copy is a page still -# painting, and the two want opposite responses (§18). +# painting, and the two want opposite responses. NO_RESULTS_MARKERS: Tuple[str, ...] = ( "no stories", "there are no stories", @@ -1235,8 +1234,8 @@ def detect_page_state(html: Optional[str], status: Optional[int] = None, url: str = "") -> str: """Which of five states this response is. - Ordered by how much each signal PROVES, not by what is cheap to check - (§17). `status` is positional and second, matching `page_flow.classify`; + Ordered by how much each signal PROVES, not by what is cheap to check. + `status` is positional and second, matching `page_flow.classify`; two engines in a sibling repo passed it as a keyword and both crashed on their first fetch, invisibly to every offline check. @@ -1347,7 +1346,7 @@ def _merge(values: Dict[str, dict]) -> Tuple[dict, str]: richer view already set. `data_source` records every view that contributed, joined with '+', so a diff between two runs that read the same story through different views reports `source_changed` rather than - inventing a change (§8). + inventing a change. """ merged: dict = {} contributors: List[str] = [] @@ -1382,7 +1381,7 @@ def parse_posts(html: str, url: str, page: int = 1, except Exception as exc: # pragma: no cover # One unreadable view must not cost the other three. Logged # rather than swallowed: a path that silently returns {} on error - # is this codebase's most common historical bug class (§8). + # is this codebase's most common historical bug class. logger.warning("%s extraction failed on %s: %s", name, url, exc) views[name] = {} diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 196c5d4..9b751fe 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -12,7 +12,7 @@ * **PASS `--chromium-path`.** pyppeteer's own bundled Chromium is build 117.0.5938.0 — two years old — and because the User-Agent is built from the - browser's OWN version (§8: never a hardcoded literal), this engine + browser's OWN version (never a hardcoded literal), this engine truthfully announces `Chrome/117`. Medium's Cloudflare refused that three times out of three. Pointed at an installed Chrome with `--chromium-path` the same engine returned 53 rows on the first attempt. This is not a @@ -55,7 +55,7 @@ # that whole mechanism only works if importing this module actually requires # the driver. With the import hidden inside _Session.open(), the module # imports cleanly with no pyppeteer installed at all, the group never skips, -# and CI cannot notice a broken import (§10). +# and CI cannot notice a broken import. from pyppeteer import launch, connect from captcha_solver import (detect_recaptcha_v3, detect_recaptcha_in_page, @@ -324,7 +324,7 @@ def close(self): # --------------------------------------------------------------------------- # Only "how to ask this driver" lives here. pyppeteer takes `() => expr` like # Playwright and unlike Selenium, which is exactly why page_flow names -# operations rather than passing JavaScript across the boundary (§1). +# operations rather than passing JavaScript across the boundary. def _count(session, selector: str) -> int: try: return len(session.bridge.run(session.page.querySelectorAll(selector))) @@ -341,8 +341,8 @@ def _sleep(ms: int) -> None: # endpoint can be refused while the HTML keeps answering 200. Counting the # refusals is what separates a feed that ran out (COMPLETE) from one whose # next batch was refused (PARTIAL) — without it the two are the same -# observation and a throttled run reports "complete" holding its first batch -# (§7). All three engines count this, so all three reach the same verdict. +# observation and a throttled run reports "complete" holding its first batch. +# All three engines count this, so all three reach the same verdict. _GRAPHQL_PATH = "/graphql/" @@ -408,7 +408,7 @@ def _min_matches(args, html: str = "") -> int: def _classify(session, html: str, status=None) -> str: # `status` POSITIONAL and second, matching the other two engines and the - # callee's real signature (§17). + # callee's real signature. return page_flow.classify(html, status, _current_url(session)) @@ -416,7 +416,7 @@ def _parse_for_mode(html: str, url: str, args, page_num: int = 1) -> List: return parse_posts(html, url, page=page_num, mode=args.mode) -def handle_captcha_if_present(session, args) -> bool: +def handle_captcha_if_present(session, args, budget) -> bool: """Detect and solve a challenge. True if something was solved.""" html = _content(session) if html is None: @@ -447,6 +447,10 @@ def handle_captcha_if_present(session, args) -> bool: logger.warning("No 2captcha API key, so this challenge cannot be " "solved — continuing with whatever the page holds.") return False + if not budget.spend(): + # Checked BEFORE the purchase, and shared with the other call site: + # see page_flow.SolveBudget. + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -501,19 +505,25 @@ def _fetch_one_page(session, args, pool, page_num: int, block_retries = 0 if not page_flow.RETRY_ON_BLOCKED else ( args.proxy_block_retries if has_pool else page_flow.BLOCK_RETRIES_WITHOUT_POOL) - solves_bought = 0 + # One allowance for every attempt and both call sites (SolveBudget). + budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False for block_attempt in range(block_retries + 1): load_failed = False + # The navigation's HTTP status for the classifier; see the + # Playwright engine. None for a scroll batch. + nav_status = None if url is not None: logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) for attempt in range(1, args.retries + 1): try: - session.bridge.run(session.page.goto( + response = session.bridge.run(session.page.goto( url, {"waitUntil": "domcontentloaded", "timeout": 60000})) + nav_status = (getattr(response, "status", None) + if response is not None else None) load_failed = False break except Exception as e: # noqa: BLE001 @@ -577,11 +587,14 @@ def _fetch_one_page(session, args, pool, page_num: int, if load_failed: break - if handle_captcha_if_present(session, args): + if handle_captcha_if_present(session, args, budget): + # The page was reloaded after the solve, so the status belongs + # to a document that is no longer there. + nav_status = None _sleep(1000) html = _content(session) or "" - state = _classify(session, html) + state = _classify(session, html, nav_status) if page_flow.is_unpainted(state, html): wait_timeout = page_flow.content_timeout_ms(args.mode) @@ -593,12 +606,10 @@ def _fetch_one_page(session, args, pool, page_num: int, lambda sel: _count(session, sel), _sleep, _ready_selector(args), _min_matches(args, html), wait_timeout) html = _content(session) or html - state = _classify(session, html) + state = _classify(session, html, nav_status) - if (page_flow.should_solve(state) - and solves_bought < page_flow.SOLVES_PER_PAGE): - solves_bought += 1 - if handle_captcha_if_present(session, args): + if page_flow.should_solve(state) and budget.available(): + if handle_captcha_if_present(session, args, budget): _sleep(1000) html = _content(session) or html state = _classify(session, html) @@ -630,7 +641,7 @@ def _fetch_one_page(session, args, pool, page_num: int, # new context, while waiting inside the same context cleared # none. `page_flow.RETRY_NEEDS_FRESH_CONTEXT` says so, and a # constant no engine consulted would be dead policy dressed - # up as enforcement (§17). + # up as enforcement. # # NOT relaunched over --cdp-endpoint: a Scraping Browser # profile allows one live connection, so reconnecting risks @@ -714,7 +725,16 @@ def _fetch_one_page(session, args, pool, page_num: int, logger.info("Saved the snapshot the parser sees to %s (%d bytes).", dump_path, len(html or "")) - products = _parse_for_mode(html or "", _current_url(session), args, page_num) + # The state policy decides whether this page is parsed at all, not only + # whether it is waited for and scrolled. A page classified `empty` is the + # site's own "nothing here" answer; parsing it anyway emitted rows from + # whatever else the page carried, and reported them as hits. + if page_flow.should_parse(state): + products = _parse_for_mode(html or "", _current_url(session), args, page_num) + else: + products = [] + logger.info("Page %d is %s, which the state policy does not parse.", + page_num, state) logger.info("Parsed %d row(s) from page %d.", len(products), page_num) # The size of the tag's WHOLE catalogue, where Medium states one — 257,497 @@ -756,7 +776,7 @@ def _fetch_one_page(session, args, pool, page_num: int, # with reading time, word count and language); `apollo` carries claps # and responses but no reading time on a tag feed; `jsonld` and `dom` # carry neither. Reported rather than warned about, because which one - # answers is a property of the URL (§9). + # answers is a property of the URL. from collections import Counter views = Counter(p.data_source for p in products) logger.info("Rows by view on page %d: %s.", page_num, @@ -785,7 +805,7 @@ def scrape(args) -> int: "the remote browser has its own exit.") pool = None - # Two shapes of pagination, and the URL picks which (§7). A day archive + # Two shapes of pagination, and the URL picks which. A day archive # is a real independent address; the other three modes have no address # for batch 2 and advance by scrolling. by_url = paginates_by_url(args.url) @@ -797,7 +817,7 @@ def scrape(args) -> int: page_flow.concurrency_refusal(args.url)) else: # A DOCUMENTED ENGINE LIMIT, in the same class as this family's - # "Selenium cannot authenticate a remote CDP endpoint" (§6): the + # "Selenium cannot authenticate a remote CDP endpoint": the # worker pool is implemented in the Playwright engine only. Said # out loud rather than run as one worker quietly, which would # look like the flag did something. @@ -820,7 +840,7 @@ def scrape(args) -> int: elif by_url: seen_keys.update(p.sku for p in first.products if p.sku is not None) - # VERIFY before planning (§7). A tag day with no stories does not + # VERIFY before planning. A tag day with no stories does not # 404 — it redirects up to the month view, which is a different # renderer holding a different set of stories. drift = redirected_away(args.url, first.final_url or args.url) @@ -979,23 +999,26 @@ def parse_args(): help="Which view the URL is. Inferred from the URL by " "default; passing one that disagrees is an error.") p.add_argument("--category", default=None, - help="Label to tag output rows with. Filled from the URL " - "by default.") - p.add_argument("--pages", type=int, default=1, + help="Accepted so a command line shared with the other " + "scrapers in this family still parses, and NOT used: " + "a Medium row has no category column (its tags are in " + "`tags`). A warning says so when it is passed.") + p.add_argument("--pages", type=page_flow.at_least("--pages", 1), default=1, help=f"Number of pages to walk (default 1, cap {PAGE_CAP}). " f"In --mode archive a page is one day with its own " f"address; every other mode has no per-page address, " f"so a page there is one settled scroll batch.") - p.add_argument("--delay", type=float, default=3.0, + p.add_argument("--delay", type=page_flow.at_least("--delay", 0, float), default=3.0, help="Delay between batches, seconds (default 3.0).") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--concurrency", type=page_flow.at_least("--concurrency", 1), default=1, metavar="N", help="Accepted for family compatibility and not " "implemented in this engine: the worker pool lives in " "playwright_scraper.py, and this engine walks archive " "days one at a time with identical output.") p.add_argument("--retries", type=page_flow.attempts, default=3, help="Attempts per page load before giving up (default 3).") - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", + type=page_flow.at_least("--retry-delay", 0, float), default=2.0, help="Seconds before the first page-load retry, doubling " "thereafter (default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") @@ -1018,7 +1041,8 @@ def parse_args(): "mid-listing here.") p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup.") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", + type=page_flow.at_least("--proxy-block-retries", 0), default=2, help="Retries from other exits when a page is refused.") p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", @@ -1062,7 +1086,7 @@ def parse_args(): # `_chrome_ua` builds, from the browser's own version — was never # WAF-blocked once. This engine therefore works headless, and that is the # family's "the user agent comes from the browser, not a literal" rule - # (§8) being the difference between data and a block page. + # being the difference between data and a block page. # # Headful is still the default because it is still better: interleaved # over 8 rounds on one address, headful was served 8/8 and headless 6/8, @@ -1082,6 +1106,10 @@ def parse_args(): "with no display.") args = p.parse_args() env_config.apply(args) + if args.category: + logger.warning("--category %r is accepted and ignored: a Medium row has " + "no category column. The story's own tags are in the " + "`tags` column.", args.category) if not args.url: p.error("no --url given, and MEDIUM_URL is not set in the environment " "or in .env.") diff --git a/scraper_api_client.py b/scraper_api_client.py index af97acf..f904a23 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -63,6 +63,7 @@ BOT_CHALLENGE_MARKERS) from output_writer import save import env_config +import page_flow logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("scraper_api_client") @@ -325,7 +326,11 @@ def parse_args(): "author (/@{username}) or a story (/p/{id}). " "Required, unless MEDIUM_URL is set in the " "environment or in .env.") - p.add_argument("--category", default=None, help="Label to tag output rows with. Defaults to the category segment of the URL, so the column is never empty just because the flag was omitted.") + p.add_argument("--category", default=None, + help="Accepted so a command line shared with the other " + "scrapers in this family still parses, and NOT used: " + "a Medium row has no category column (its tags are in " + "`tags`). A warning says so when it is passed.") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="medium_posts_scraperapi", help="Output file prefix") p.add_argument("--timeout", type=int, default=60, @@ -345,11 +350,12 @@ def parse_args(): p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 products were parsed. Off by " "default so a failed fetch can't overwrite a good result.") - p.add_argument("--retries", type=int, default=1, + p.add_argument("--retries", type=page_flow.at_least("--retries", 0), default=1, help="Extra attempts if a bot-challenge page comes back. One retry is " "usually worth it. Each attempt is a separate billable task, so " "this defaults to 1.") - p.add_argument("--retry-delay", type=int, default=10, + p.add_argument("--retry-delay", type=page_flow.at_least("--retry-delay", 0), + default=10, help="Seconds between retries (default 10)") p.add_argument("--dump-html", default=None, help="Also write the raw returned HTML to this path (always, even on success)") @@ -364,6 +370,10 @@ def parse_args(): if not args.url: p.error("no --url given, and MEDIUM_URL is not set in the environment " "or in .env.") + if args.category: + logger.warning("--category %r is accepted and ignored: a Medium row has " + "no category column. The story's own tags are in the " + "`tags` column.", args.category) return args diff --git a/selenium_scraper.py b/selenium_scraper.py index a7c9d19..177b028 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -208,7 +208,7 @@ def open(self): # GraphQL responses — see `_graphql_refused_count`. It has to be # asked for at driver creation; there is no way to turn it on later. # Without it this engine would report `complete` where its twins - # report `partial` on the identical run (§6). + # report `partial` on the identical run. options.set_capability("goog:loggingPrefs", {"performance": "ALL"}) self.driver = webdriver.Chrome(options=options) @@ -243,7 +243,7 @@ def _apply_fingerprint(self): # which defeats the flag rather than breaking it, because the run # then presents a Windows fingerprint's screen, locale and timezone # over a local Chromium's UA. That is the identity MISMATCH the flag - # exists to avoid (§16, where the same defect was live in four + # exists to avoid (where the same defect was live in four # sibling repos at once). from fingerprint_client import (get_fingerprint, fingerprint_user_agent, playwright_init_script) @@ -290,7 +290,7 @@ def close(self): # Only "how to ask this driver" lives here. Note the JS dialect: Selenium's # execute_script runs a function BODY and needs an explicit `return`, unlike # the `() => expr` both other engines take — which is exactly why page_flow -# names operations instead of passing JavaScript across the boundary (§1). +# names operations instead of passing JavaScript across the boundary. def _count(session, selector: str) -> int: try: return len(session.driver.find_elements(By.CSS_SELECTOR, selector)) @@ -317,7 +317,7 @@ def _scroll_to_bottom(session) -> None: Note the dialect: Selenium's `execute_script` takes a function BODY with an explicit `return`, where Playwright and pyppeteer take `() => expr`. That difference is exactly why `page_flow` names the OPERATION and each - engine spells it itself (§1). + engine spells it itself. """ try: session.driver.execute_script( @@ -342,8 +342,8 @@ def _page_height(session) -> Optional[int]: # # This is NOT a cosmetic difference. If this engine always answered 0, it # would report `complete` where its twins report `partial` on the identical -# run, and that is precisely the drift the shared modules exist to prevent -# (§6). The log has to be ENABLED at driver creation; see `_Session.open`. +# run, and that is precisely the drift the shared modules exist to prevent. +# The log has to be ENABLED at driver creation; see `_Session.open`. _GRAPHQL_PATH = "/graphql/" @@ -399,7 +399,7 @@ def _current_url(session) -> str: def _classify(session, html: str, status=None) -> str: # `status` is POSITIONAL and second. Two engines in a sibling repo passed - # it as a keyword and both crashed on their first fetch (§17); this + # it as a keyword and both crashed on their first fetch; this # repo's smoke suite binds every shared-module call in every engine # against the callee's real signature for that reason. return page_flow.classify(html, status, _current_url(session)) @@ -409,7 +409,7 @@ def _parse_for_mode(html: str, url: str, args, page_num: int = 1) -> List: return parse_posts(html, url, page=page_num, mode=args.mode) -def handle_captcha_if_present(session, args) -> bool: +def handle_captcha_if_present(session, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Same contract and same reconciliation as the Playwright engine, including @@ -448,6 +448,10 @@ def handle_captcha_if_present(session, args) -> bool: logger.warning("No 2captcha API key, so this challenge cannot be " "solved — continuing with whatever the page holds.") return False + if not budget.spend(): + # Checked BEFORE the purchase, and shared with the other call site: + # see page_flow.SolveBudget. + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -477,7 +481,7 @@ def _scroll_the_feed(session, args, html: str, page_num: int) -> dict: the 11 attempts measured, while the HTML itself came back 200. So the expected outcome here is "scrolled, nothing arrived", and that is reported rather than fought: more rounds against a feed the site will not - extend only spends time (§8). + extend only spends time. Deliberately no `target`. The only total Medium publishes is `tag.postCount` — 257,497 for `python` — which is the size of the tag's @@ -516,7 +520,8 @@ def _fetch_one_page(session, args, pool, page_num: int, block_retries = 0 if not page_flow.RETRY_ON_BLOCKED else ( args.proxy_block_retries if has_pool else page_flow.BLOCK_RETRIES_WITHOUT_POOL) - solves_bought = 0 + # One allowance for every attempt and both call sites (SolveBudget). + budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False for block_attempt in range(block_retries + 1): @@ -553,7 +558,7 @@ def _fetch_one_page(session, args, pool, page_num: int, if refused: # NOT the end of the listing. Reporting a refusal as # "exhausted" would make a throttled run say "complete" - # while holding its first batch (§7). + # while holding its first batch. outcome.state = "no_turnover" outcome.load_failed = True outcome.final_url = _current_url(session) @@ -590,10 +595,14 @@ def _fetch_one_page(session, args, pool, page_num: int, if load_failed: break - if handle_captcha_if_present(session, args): + if handle_captcha_if_present(session, args, budget): _sleep(1000) html = _content(session) or "" + # No status here: `driver.get()` returns None and WebDriver exposes + # no HTTP status at all, so this engine classifies on the body alone. + # Both of Medium's refusals are recognised by body, so the gap is + # only a refusal with no recognisable page. state = _classify(session, html) if page_flow.is_unpainted(state, html): @@ -608,10 +617,8 @@ def _fetch_one_page(session, args, pool, page_num: int, html = _content(session) or html state = _classify(session, html) - if (page_flow.should_solve(state) - and solves_bought < page_flow.SOLVES_PER_PAGE): - solves_bought += 1 - if handle_captcha_if_present(session, args): + if page_flow.should_solve(state) and budget.available(): + if handle_captcha_if_present(session, args, budget): _sleep(1000) html = _content(session) or html state = _classify(session, html) @@ -643,7 +650,7 @@ def _fetch_one_page(session, args, pool, page_num: int, # new context, while waiting inside the same context cleared # none. `page_flow.RETRY_NEEDS_FRESH_CONTEXT` says so, and a # constant no engine consulted would be dead policy dressed - # up as enforcement (§17). + # up as enforcement. # # NOT relaunched over --cdp-endpoint: a Scraping Browser # profile allows one live connection, so reconnecting risks @@ -720,7 +727,16 @@ def _fetch_one_page(session, args, pool, page_num: int, logger.info("Saved the snapshot the parser sees to %s (%d bytes).", dump_path, len(html or "")) - products = _parse_for_mode(html or "", _current_url(session), args, page_num) + # The state policy decides whether this page is parsed at all, not only + # whether it is waited for and scrolled. A page classified `empty` is the + # site's own "nothing here" answer; parsing it anyway emitted rows from + # whatever else the page carried, and reported them as hits. + if page_flow.should_parse(state): + products = _parse_for_mode(html or "", _current_url(session), args, page_num) + else: + products = [] + logger.info("Page %d is %s, which the state policy does not parse.", + page_num, state) logger.info("Parsed %d row(s) from page %d.", len(products), page_num) # The size of the tag's WHOLE catalogue, where Medium states one — 257,497 @@ -762,7 +778,7 @@ def _fetch_one_page(session, args, pool, page_num: int, # with reading time, word count and language); `apollo` carries claps # and responses but no reading time on a tag feed; `jsonld` and `dom` # carry neither. Reported rather than warned about, because which one - # answers is a property of the URL (§9). + # answers is a property of the URL. from collections import Counter views = Counter(p.data_source for p in products) logger.info("Rows by view on page %d: %s.", page_num, @@ -795,7 +811,7 @@ def scrape(args) -> int: "the remote browser has its own exit.") pool = None - # Two shapes of pagination, and the URL picks which (§7). A day archive + # Two shapes of pagination, and the URL picks which. A day archive # is a real independent address; the other three modes have no address # for batch 2 and advance by scrolling. by_url = paginates_by_url(args.url) @@ -807,7 +823,7 @@ def scrape(args) -> int: page_flow.concurrency_refusal(args.url)) else: # A DOCUMENTED ENGINE LIMIT, in the same class as this family's - # "Selenium cannot authenticate a remote CDP endpoint" (§6): the + # "Selenium cannot authenticate a remote CDP endpoint": the # worker pool is implemented in the Playwright engine only. Said # out loud rather than run as one worker quietly, which would # look like the flag did something. @@ -829,7 +845,7 @@ def scrape(args) -> int: elif by_url: seen_keys.update(p.sku for p in first.products if p.sku is not None) - # VERIFY before planning (§7). A tag day with no stories does not + # VERIFY before planning. A tag day with no stories does not # 404 — it redirects up to the month view, which is a different # renderer holding a different set of stories. drift = redirected_away(args.url, first.final_url or args.url) @@ -987,27 +1003,30 @@ def parse_args(): "All four yield the same row — what differs is how " "much of it Medium sends (see the module docstring).") p.add_argument("--category", default=None, - help="Label to tag output rows with. Filled from the URL " - "by default.") - p.add_argument("--pages", type=int, default=1, + help="Accepted so a command line shared with the other " + "scrapers in this family still parses, and NOT used: " + "a Medium row has no category column (its tags are in " + "`tags`). A warning says so when it is passed.") + p.add_argument("--pages", type=page_flow.at_least("--pages", 1), default=1, help=f"Number of pages to walk (default 1, cap {PAGE_CAP}). " f"In --mode archive a page is one day with its own " f"address; every other mode has no per-page address " f"(`?page=2` is ignored and the feed returns its " f"first items again), so a page there is one settled " f"scroll batch.") - p.add_argument("--delay", type=float, default=3.0, + p.add_argument("--delay", type=page_flow.at_least("--delay", 0, float), default=3.0, help="Delay between batches, seconds (default 3.0). The " "rate matters more than the address on this site, so " "this is the lever that matters.") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--concurrency", type=page_flow.at_least("--concurrency", 1), default=1, metavar="N", help="Accepted for family compatibility and not " "implemented in this engine: the worker pool lives in " "playwright_scraper.py, and this engine walks archive " "days one at a time with identical output.") p.add_argument("--retries", type=page_flow.attempts, default=3, help="Attempts per page load before giving up (default 3).") - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", + type=page_flow.at_least("--retry-delay", 0, float), default=2.0, help="Seconds before the first page-load retry, doubling " "thereafter (default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") @@ -1029,7 +1048,8 @@ def parse_args(): "current browser session.") p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup.") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", + type=page_flow.at_least("--proxy-block-retries", 0), default=2, help="Retries from other exits when a page is refused.") p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", @@ -1073,7 +1093,7 @@ def parse_args(): # `_chrome_ua` builds, from the browser's own version — was never # WAF-blocked once. This engine therefore works headless, and that is the # family's "the user agent comes from the browser, not a literal" rule - # (§8) being the difference between data and a block page. + # being the difference between data and a block page. # # Headful is still the default because it is still better: interleaved # over 8 rounds on one address, headful was served 8/8 and headless 6/8, @@ -1093,6 +1113,10 @@ def parse_args(): "with no display.") args = p.parse_args() env_config.apply(args) + if args.category: + logger.warning("--category %r is accepted and ignored: a Medium row has " + "no category column. The story's own tags are in the " + "`tags` column.", args.category) if not args.url: p.error("no --url given, and MEDIUM_URL is not set in the environment " "or in .env.") diff --git a/smoke_test.py b/smoke_test.py index 34e6e2f..b831477 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -31,7 +31,7 @@ on the same story. Reading it would have filled the column completely and wrongly, and no coverage check would have said a word. - * `test_markers_do_not_match_a_good_page` is the §18 rule as a test. Medium + * `test_markers_do_not_match_a_good_page` is the "count a marker on a good page" rule as a test. Medium ships reCAPTCHA Enterprise on every page it serves, and `challenge-platform` appears twice on good and refused pages alike. A marker that matches every page is worse than no marker, so every @@ -103,7 +103,7 @@ SHARED_MODULES = {"page_flow": page_flow, "product_parser": product_parser} # Four fixtures cut from pages Medium actually served. Every marker check -# below asserts its markers are ABSENT from all four (§18). +# below asserts its markers are ABSENT from all four. GOOD_PAGES = ("tag_feed", "archive_day", "author", "post") _failures = [] @@ -492,7 +492,7 @@ def test_publication_pages_are_refused(): # A host that left Medium is refused with THAT reason rather than with # "is not a Medium site", which is false and sends the reader looking for - # a typo (§5). + # a typo. why = unsupported_reason("https://towardsdatascience.com/") check("a former publication is refused", why is not None) check("...with the reason it actually left", @@ -583,7 +583,7 @@ def test_pagination(): check("%s does not" % other, not paginates_by_url(other)) # None, not a built URL. `?page=2` on a tag feed is IGNORED rather # than failing, so a run built on it would add no new sku, call the - # listing exhausted and report COMPLETE holding one page (§18). + # listing exhausted and report COMPLETE holding one page. check("...and page_url returns None for it", page_url(other, 2) is None) check("page 1 is the URL given", page_url(archive, 1) == archive) @@ -607,7 +607,7 @@ def test_pagination(): page_url("https://medium.com/tag/python/archive/2026/02/31", 2) is None) # The site publishes its own archive index, which is what lets a walk be - # planned rather than guessed (§7). + # planned rather than guessed. years = archive_years(fixture("archive_day")) check("the archive index lists years with stories", len(years) > 10) check("...as four-digit strings", @@ -681,7 +681,7 @@ def test_page_state(): # The status is positional and SECOND, matching `page_flow.classify`. # Two engines in a sibling repo passed it as a keyword and both crashed - # on their first fetch (§17). + # on their first fetch. check("detect_page_state takes status positionally, second", list(inspect.signature(detect_page_state).parameters)[:2] == ["html", "status"]) @@ -691,7 +691,7 @@ def test_page_state(): # A payload proves content even where nothing has painted, and it is # asked BEFORE any threshold — ordered by what each signal proves rather - # than by what is cheap (§17). + # than by what is cheap. unpainted = re.sub(r"", "", fixture("tag_feed"), flags=re.S) check("a state with no painted cards is still content", @@ -710,7 +710,7 @@ def test_page_state(): "https://medium.com/tag/python") == "blocked") # Medium's own "no stories" copy is a POSITIVE signal and beats the # status check and every threshold: a correct empty answer must not be - # reported as a block (§17). + # reported as a block. empty = shell.replace("", "No stories") check("the site's own empty answer is `empty`, not `blocked`", detect_page_state(empty, 200, "https://medium.com/tag/x") == "empty") @@ -723,7 +723,7 @@ def test_page_state(): def test_markers_do_not_match_a_good_page(): - group("§18: a marker that matches every page is worse than no marker") + group("a marker that matches every page is worse than no marker") # EVERY marker in EVERY set, asserted absent from four pages Medium # actually served. This is the check that would have caught `g-recaptcha` @@ -782,7 +782,7 @@ def test_markers_do_not_match_a_good_page(): # The Scraping Browser's auto-solve extension injects its own turnstile # hunter into every page it loads. This repo's marker set CAN match one, - # which is the condition §8 sets for adding the filter. + # which is the condition for adding the filter. injected = (fixture("tag_feed") + '