diff --git a/.env.example b/.env.example index 9f77516..4fc0c0d 100644 --- a/.env.example +++ b/.env.example @@ -71,7 +71,7 @@ MEDIAMARKT_CDP_ENDPOINT=ws://{login}-zone-scraping_browser-country-de-pid-{profi MEDIAMARKT_PROXY=http://{user}:{password}@eu.proxy.2captcha.com:2334 # --- Default target ------------------------------------------------------- -# Saves retyping --url. Any of the eleven country sites; the hostname decides +# Saves retyping --url. Any of the ten country sites; the hostname decides # which, so there is no --country flag that could disagree with the URL. # # Careful with category URLs: some are HUB pages that list sub-categories and diff --git a/.github/canary_preflight.py b/.github/canary_preflight.py new file mode 100644 index 0000000..28c5682 --- /dev/null +++ b/.github/canary_preflight.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Check the canary's proxy BEFORE the browser run, and name what is wrong. + +Why this exists: from 2026-09-10 the daily canary failed on every run with +the same three lines, `Timeout loading ... (attempt 1/3)`, `2/3`, `Gave up`, +then exit 5. The scraper cannot say more than that: Chromium reports a proxy +that accepts a connection and never delivers anything as a plain navigation +timeout. This script asks the same proxy the same question with short +timeouts and turns the answer into one of a few named causes, so a red run +says "the proxy secret is dead" instead of "unexpected exit code". + +It reads the proxy from MEDIAMARKT_PROXY and never prints it: only the host +and port (which exit, not the secret) and the two-letter region code in the +login, if there is one. + +Exit codes: + 0 the proxy delivered a page from the site (the browser run goes ahead) + 1 the proxy is unusable, and the ::error:: line says how + 2 MEDIAMARKT_PROXY is not set + +A 403 from the site through the proxy is reported as a warning and exits 0: +the browser run is the real test of whether the exit is accepted, and a +plain HTTP client is a weaker witness than a browser. +""" + +import os +import re +import socket +import sys +from urllib.parse import urlparse + +TARGET = os.environ.get( + "CANARY_URL", + "https://www.mediamarkt.de/de/category/k%C3%BChlen-gefrieren-32.html") +CONNECT_TIMEOUT = float(os.environ.get("PREFLIGHT_CONNECT_TIMEOUT", "15")) +READ_TIMEOUT = float(os.environ.get("PREFLIGHT_READ_TIMEOUT", "45")) +UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36") + + +def error(msg): + print(f"::error::{msg}") + + +def where(proxy): + """host:port of the proxy, which is safe to print.""" + p = urlparse(proxy) + return f"{p.hostname}:{p.port}" + + +def region_in_login(proxy): + """The exit country the login asks for, e.g. 'de', or None.""" + user = urlparse(proxy).username or "" + m = re.search(r"(?:region|country)-([a-zA-Z]{2})\b", user) + return m.group(1).lower() if m else None + + +def main(): + proxy = (os.environ.get("MEDIAMARKT_PROXY") or "").strip() + if not proxy: + error("MEDIAMARKT_PROXY is not set. Add a residential German exit as " + "that repository secret.") + return 2 + if "://" not in proxy: + proxy = "http://" + proxy + p = urlparse(proxy) + if not p.hostname or not p.port: + error("MEDIAMARKT_PROXY is not a URL of the form " + "scheme://user:password@host:port (value not printed).") + return 1 + + region = region_in_login(proxy) + print(f"proxy {where(proxy)}; credentials " + f"{'present' if p.username else 'ABSENT'}; region in login: " + f"{region or 'none'}") + if region and region != "de": + print(f"::warning::The proxy login asks for region '{region}'. This " + f"canary scrapes mediamarkt.de, which refuses most non-German " + f"exits; the login should ask for 'de'.") + + # 1. Is anything listening at all? + try: + socket.create_connection((p.hostname, p.port), timeout=CONNECT_TIMEOUT).close() + except OSError as e: + error(f"Cannot open a TCP connection to the proxy {where(proxy)} " + f"({type(e).__name__}). The host or port in MEDIAMARKT_PROXY is " + f"wrong, or the gateway is down.") + return 1 + + import requests + + # 2. Does it carry a request to the site? + try: + r = requests.get(TARGET, proxies={"http": proxy, "https": proxy}, + headers={"User-Agent": UA, + "Accept-Language": "de-DE,de;q=0.9"}, + timeout=(CONNECT_TIMEOUT, READ_TIMEOUT), + allow_redirects=True) + except requests.exceptions.ProxyError as e: + text = str(e) + if "407" in text: + error(f"The proxy {where(proxy)} rejected the credentials (HTTP " + f"407). The login or password in MEDIAMARKT_PROXY is wrong " + f"or has expired: replace the secret.") + else: + error(f"The proxy {where(proxy)} accepted the connection but would " + f"not open a tunnel to the site ({_short(text)}). This is " + f"what an expired, suspended or out-of-traffic proxy " + f"account looks like: replace the MEDIAMARKT_PROXY secret.") + return 1 + except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout) as e: + error(f"The proxy {where(proxy)} accepted a TCP connection but the " + f"page did not arrive within {CONNECT_TIMEOUT:.0f}+" + f"{READ_TIMEOUT:.0f} s ({type(e).__name__}). The browser run " + f"would time out the same way (exit 5). A gateway that answers " + f"and then delivers nothing is what an expired or out-of-traffic " + f"proxy account looks like: replace the MEDIAMARKT_PROXY secret " + f"with a working residential German exit.") + return 1 + except requests.exceptions.RequestException as e: + error(f"Requesting the site through the proxy {where(proxy)} failed: " + f"{type(e).__name__}: {_short(str(e))}") + return 1 + + final_host = urlparse(r.url).hostname or "" + print(f"site answered HTTP {r.status_code}, {len(r.content)} bytes, " + f"final host {final_host}") + if r.status_code == 403: + print("::warning::MediaMarkt refused this plain HTTP request through the " + "proxy (403). If the browser run below is refused too (exit 3), " + "the exit is not a residential German address.") + elif r.status_code >= 400: + print(f"::warning::The site answered HTTP {r.status_code} through the " + f"proxy.") + elif "mediamarkt.de" not in final_host: + print(f"::warning::The request was redirected to {final_host}: the exit " + f"is not in Germany, and the canary's source check will fail.") + return 0 + + +def _short(text): + """An exception message without anything that could be a credential.""" + text = re.sub(r"[^\s/:@]+:[^\s/@]+@", "***@", text) + return text[:200] + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/ci_checks.py b/.github/ci_checks.py index 1151c35..6b6dbbc 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -76,7 +76,27 @@ HEX32 = re.compile(r"\b[0-9a-f]{32}\b") HEX32_ALLOWED = ("sha", "hash", "nonce", "example", "md5", "digest", "checksum") -SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example") +SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example", + # JSON too: fixtures and run outputs are where a captured + # page's front-end key would arrive. + ".json") + +# Directory names skipped wherever they appear INSIDE the repo. Judged on the +# path relative to the repo root, never on the absolute path: a clone that +# happens to live under a directory called `venv` or `.venv` otherwise skips +# every file and still prints "nothing credential-shaped". +SKIPPED_DIRS = {".git", "__pycache__", ".venv", "venv"} + + +def _is_env_backup(path): + """`.env.bak`, `.env.local`, `.env.save`, ... but not `.env` itself. + + A renamed copy of `.env` is the classic way a key ends up one `git add -A` + away from a commit, and its suffix (`.bak`, `.local`) is not one the suffix + list would ever include. `.env` itself is the user's own config, ignored by + git, and is supposed to hold a key; `.env.example` is matched by suffix. + """ + return path.name.startswith(".env.") and path.name != ".env.example" # A directory holding `pyvenv.cfg` is a virtualenv, whatever it is called. @@ -87,7 +107,7 @@ # into pip's vendored code and flags a 32-hex string in `_elffile.py` as # key-shaped. Correct about the string, wrong about the file — and it is the # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people -# have to argue with is one they learn to suppress (CLAUDE.md §22). +# have to argue with is one they learn to suppress. # # Deliberately NOT a narrowing of what gets scanned: an untracked file is # still read, because a key pasted into a scratch file beside the scripts is @@ -116,10 +136,11 @@ def _is_in_virtualenv(path): def scanned_files(): for path in sorted(REPO.rglob("*")): - if not path.is_file() or path.suffix not in SCANNED_SUFFIXES: + if not path.is_file(): + continue + if path.suffix not in SCANNED_SUFFIXES and not _is_env_backup(path): continue - if any(part in {".git", "__pycache__", ".venv", "venv"} - for part in path.parts): + if any(part in SKIPPED_DIRS for part in path.relative_to(REPO).parts): continue if _is_in_virtualenv(path): continue @@ -211,6 +232,11 @@ def secret_check(): failed.append(f"{rel}:{lineno} contains {match[:6]}… — a " f"32-char hex string, the shape of a 2captcha key") + if not scanned: + # Zero files is not "clean", it is "looked at nothing". Without this a + # broken exclusion reports exactly what a clean repo reports. + failed.append("0 files scanned — the scan looked at nothing, so it " + "cannot say the repo is clean") if not failed: print(f"ok {scanned} files scanned, nothing credential-shaped") return failed diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 1f3a1c9..2453878 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -17,12 +17,22 @@ name: canary # single day — and a check that is always red teaches everyone to ignore # checks, which is worse than having no check at all. # -# So the live steps below are SKIPPED, not failed, when the secret is absent. -# The job goes green with a notice saying why. Set the secret and the same -# workflow starts doing real work with no edit: +# So the live steps need the secret, and WITHOUT it the job FAILS, naming +# the missing secret. It used to skip green with a notice, and a green badge +# over a canary that tested nothing is worse than a red one: it says the site +# was checked when it was not. Set the secret and the same workflow starts +# doing real work with no edit: # # gh secret set MEDIAMARKT_PROXY --repo 2scraper/mediamarkt-scraper # +# A secret that is SET but dead fails too, and says so. From 2026-09-10 every +# scheduled run timed out loading page 1 through the proxy and ended "exit 5"; +# the run itself cannot tell a dead proxy from a slow site. The preflight step +# (.github/canary_preflight.py) asks the proxy directly with short timeouts and +# names the cause: unreachable gateway, rejected credentials (407), a tunnel +# that never opens, or a connection that delivers nothing (the shape of an +# expired or out-of-traffic proxy account). Only then does the browser run. +# # It must be a RESIDENTIAL exit, and for this URL a German one — the site # geo-redirects a cookie-less visit on the exit IP, and a run redirected to # another country site fails the `source` check below on purpose. @@ -51,29 +61,53 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Explain a skipped run + - name: Require the proxy secret if: env.HAVE_PROXY != 'true' run: | - echo "::notice::Skipping the live run: no MEDIAMARKT_PROXY secret is set." + echo "::error::Missing secret MEDIAMARKT_PROXY: the canary cannot test the site without it." echo "MediaMarkt answers every datacentre address with HTTP 403, and a" - echo "GitHub runner is one, so a run from here would be red every day" - echo "whatever the state of the code. Set a residential German exit as" - echo "the MEDIAMARKT_PROXY repository secret to turn this canary on:" + echo "GitHub runner is one. Set a residential GERMAN exit as the" + echo "MEDIAMARKT_PROXY repository secret:" echo echo " gh secret set MEDIAMARKT_PROXY --repo 2scraper/mediamarkt-scraper" - echo - echo "The offline suite (tests.yml) covers everything that does not" - echo "need the network, and it runs on every push." + exit 1 + + - name: Can this runner reach the site without a proxy? + if: always() + # Informational, never fails the job. It separates "the site is down" + # from "the site refuses this address": a 403 here is the expected + # datacentre refusal, a 000 means the site did not answer at all. + run: | + code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 30 \ + -A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" \ + "https://www.mediamarkt.de/de/category/k%C3%BChlen-gefrieren-32.html" || true) + case "$code" in + 403) echo "::notice::Bare runner: HTTP 403, the site's datacentre refusal. The site is up; the proxy is what has to get past it." ;; + 200) echo "::notice::Bare runner: HTTP 200. The site served a datacentre address today, so a failure below is about the proxy, not the site." ;; + 000) echo "::warning::Bare runner: no HTTP answer from mediamarkt.de within 30 s. The site or the network may be down." ;; + *) echo "::notice::Bare runner: HTTP $code." ;; + esac - uses: actions/setup-python@v5 if: env.HAVE_PROXY == 'true' with: python-version: "3.12" - - name: Install Playwright engine + - name: Check the proxy before the browser run if: env.HAVE_PROXY == 'true' + # Fails the job with an ::error:: that names the cause when the proxy + # cannot deliver a page, and then the browser run is not attempted: + # it would only time out three times and report exit 5. + env: + MEDIAMARKT_PROXY: ${{ secrets.MEDIAMARKT_PROXY }} run: | python -m pip install --upgrade pip + pip install -r requirements.txt + python .github/canary_preflight.py + + - name: Install Playwright engine + if: env.HAVE_PROXY == 'true' + run: | pip install -r requirements.txt -r requirements-playwright.txt playwright install --with-deps chromium diff --git a/.gitignore b/.gitignore index 9cb035e..306e8ff 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,20 @@ live_results.zip *_debug.html *_debug.png *.html.page* +# --dump-html X --pages N writes X.page1..X.pageN whatever X is called, so a +# dump named canary_dump.json becomes canary_dump.json.page3, which neither +# `*.json` nor `*.html.page*` covers. +*.page[0-9]* +# A bare dump at the repo root (canary_run.html, dump.html). No page of this +# project is an HTML file at the root. +/*.html +# Working directories by KIND, not by one chosen name: a raw capture reached +# a sibling repo's first commit through `live/`, a directory nobody had listed. +live/ +out/ +output/ +dumps/ +scratch/ # --- captures kept for parser work --------------------------------------- # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py); @@ -73,7 +87,6 @@ secrets* # recoverable only because that repo was private at the time. On 2026-09-17 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC # repository, one `git add -A` from publication. Craigslist was fixed then -# and the fix never reached anyone else, which is §16's rule unapplied: when -# you fix one, check the siblings. +# and the fix never reached anyone else: when you fix one, check the siblings. .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f4fa4d..4718956 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,67 @@ with it, so nobody discovers it from a bill or from a diff. ## [Unreleased] +> **Behaviour changes in this batch:** +> * `--pages 0`, `--concurrency 0` and negative values of `--pages`, +> `--concurrency`, `--proxy-block-retries`, `--delay` and `--retry-delay` +> are now usage errors (exit 2) naming the flag, in all three engines. A +> negative `--retries` or `--retry-delay` is rejected by +> `scraper_api_client.py` too (0 stays legal there: it counts extra tries). +> * A page buys at most ONE paid captcha solve, however many times it is +> retried or rotated to another exit. Before, a pool of 4 exits could buy 4. +> * Output files are now written to a temporary file and renamed into place. +> A new file gets the same mode `open()` gave it (the umask decides); an +> existing file keeps its own mode. +> * CSV cells that start with `=`, `+`, `-`, `@`, a tab, CR or LF get a +> leading apostrophe so a spreadsheet shows them as text. Only text cells, +> only in the CSV; the sidecar records how many as `csv_cells_escaped`. +> * A complete run saved with `--allow-empty` and 0 rows is now +> `status: complete` in the sidecar, not `failed`. +> * The canary FAILS when the `MEDIAMARKT_PROXY` secret is missing, instead +> of skipping green. + +### Fixed (audit 2026-10-14) + +- **A failed write destroyed the previous good output.** `open(path, "w")` + truncates first, so an error halfway through `json.dump` left a broken + file where a good one had been. JSON, CSV and the `.meta.json` sidecar now + go through one atomic writer (temp file in the same directory, fsync, + rename, temp file removed on error). +- **CSV formula injection.** Titles, categories and other site text are + written by the shop or a seller; a cell beginning with a formula character + is now neutralised in the CSV. JSON keeps the site's exact text. +- **The per-page solve allowance did not exist.** Every engine now creates + one `page_flow.SolveBudget` per page, before the retry loop, and checks it + before buying a solve. +- **A reCAPTCHA challenge frame with no `size` and no `render` was taken for + v3**, so a paid solve would have bought the wrong task type. A challenge + frame now always means v2 (v3 never renders one). +- **An Enterprise reCAPTCHA was handed to the ordinary solver.** This repo + does not implement the enterprise task type yet, and an ordinary token is + rejected by an enterprise widget, so no solve is bought for one now. +- **The HTTP status never reached the page classifier** in the browser + engines, although it is this site's strongest block signal. Playwright and + pyppeteer now pass `goto()`'s status; Selenium cannot see one and says so. +- **The credential scan could scan nothing and report clean.** Its + directory exclusions were tested against the ABSOLUTE path, so a clone + inside any directory named `venv` or `.venv` scanned 0 files. It now + judges paths relative to the repo, fails when it scanned nothing, and also + reads `.json` files and renamed `.env` copies such as `.env.bak`. +- **`.gitignore`** now also covers `--dump-html` page dumps of any name + (`*.page[0-9]*`), bare HTML dumps at the repo root, and working + directories such as `live/` and `out/`. +- **The canary said "Unexpected exit code" for five weeks.** Every scheduled + run since 2026-09-10 timed out loading page 1 through the proxy secret. A + preflight step now asks the proxy directly and names the cause (unreachable + gateway, rejected credentials, a tunnel that never opens, or a connection + that delivers nothing), and a bare-runner request shows whether the site + itself is up. +- Wording: "ten country sites" (not eleven), the README no longer quotes a + check count that goes stale, and the Scraper API `--wait-element` example + uses this site's product-card selector. +- References to private notes were removed from comments, the suite and this + changelog; a check keeps them out. + > **Behaviour change:** `--retries 0` is now a usage error (exit 2). It used > to be accepted and made the page-load loop unreachable, so nothing was > fetched and the engine reported the browser's own blank page as a result: @@ -61,7 +122,7 @@ with it, so nobody discovers it from a bill or from a diff. ### Added - **A check that binds every call into a shared module against the callee's - real signature** (§17's check #1), which this repo did not have. It catches + real signature**, which this repo did not have. It catches two things nothing else here can: a call whose arguments do not fit the signature, and a call to a name the shared module does not define at all — both of which reach a live run as a crash on the first fetch while import, @@ -89,9 +150,9 @@ with it, so nobody discovers it from a bill or from a diff. - **`fingerprint_client.py` could not read the key from `.env`.** `--key` defaulted to `os.environ.get("TWOCAPTCHA_KEY")` and only that, so a key put - in `.env` — exactly as §3, the README and `.env.example` instruct — worked + in `.env` — exactly as the README and `.env.example` instruct — worked for every engine and failed HERE with "No API key". A documented mechanism - not applied on one path, which is the shape of half the defects §16 lists. + not applied on one path, a common shape of defect in copied code. It now reads through `env_config.env_value`, calling `load_env()` itself because this is a standalone entry point that no engine has necessarily run @@ -103,7 +164,7 @@ with it, so nobody discovers it from a bill or from a diff. "still set to the placeholder from .env.example" instead. Found on a sibling repo's first live `--fingerprint` run, then checked - across the family before patching, per §16: five repos had it and one had + across the family before patching: five repos had it and one had already fixed it. Pinned by a check verified to fail on the old code — including that the help string does not interpolate its default, which is one substring away from printing a live credential to anyone who types diff --git a/README.md b/README.md index 6328367..dbfb37f 100644 --- a/README.md +++ b/README.md @@ -409,7 +409,9 @@ percent sign written before its number, and an instalment line in the same price block) — so a change in the site's markup fails a test rather than quietly emptying a column. -`smoke_test.py`: **339 checks**, no network, no browser, no credentials. +`python3 smoke_test.py` runs the offline suite: no network, no browser, no +credentials. Each check prints a PASS or FAIL line, and an engine group whose +library is not installed is reported as SKIPPED at the end. --- diff --git a/captcha_solver.py b/captcha_solver.py index a65660a..6761ef0 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -99,6 +99,11 @@ class CaptchaChallenge: # Raw `size` from the reCAPTCHA client config, when available: # "invisible" for v2-invisible and for v3, absent for a v2 checkbox. size: Optional[str] = None + # True when the live page runs reCAPTCHA ENTERPRISE. Enterprise needs its + # own task type, which this repo does not implement, and an ordinary + # v2/v3 token is rejected by an enterprise widget, so solve_recaptcha + # refuses to buy one rather than paying for a token the site will refuse. + enterprise: bool = False @property def is_v3(self) -> bool: @@ -306,6 +311,8 @@ def detect_recaptcha_in_page(evaluate, page_url: str = "") -> Optional[CaptchaCh * `size == "invisible"` with no challenge frame is treated as v3, which is v3's normal appearance (badge only). * a `normal`/`compact` size is a v2 checkbox. + * a challenge frame with no `size` and no `render` is a v2 checkbox: + v3 renders no frame at all, so a frame rules v3 out. When the two signals disagree the ambiguity is recorded in the returned challenge's `kind` and in the log, rather than silently guessing — pick the wrong one and 2captcha returns a token the site rejects. @@ -340,8 +347,12 @@ def detect_recaptcha_in_page(evaluate, page_url: str = "") -> Optional[CaptchaCh kind = "recaptcha_v2_invisible" if size == "invisible" else "recaptcha_v2" elif size in ("normal", "compact"): kind = "recaptcha_v2" - elif info.get("challengeFrame") and size == "invisible": - kind = "recaptcha_v2_invisible" + elif info.get("challengeFrame"): + # A challenge frame settles it as v2, whatever `size` says or omits: + # v3 never renders one. Before this rung, a frame with no `size` and + # no `render` fell through to v3 and a paid solve bought the wrong + # task type. + kind = "recaptcha_v2_invisible" if size == "invisible" else "recaptcha_v2" else: kind = "recaptcha_v3" @@ -351,8 +362,9 @@ def detect_recaptcha_in_page(evaluate, page_url: str = "") -> Optional[CaptchaCh kind, info["sitekey"], size, render, info.get("challengeFrame"), info.get("containerId"), "; ".join(hints)) if info.get("enterprise"): - logger.warning("This is a reCAPTCHA ENTERPRISE widget — 2captcha needs its " - "enterprise method, which this project does not implement.") + logger.warning("This is a reCAPTCHA ENTERPRISE widget. Solving it needs " + "the enterprise task type, which this repo does not " + "implement yet, so no solve will be bought for it.") return CaptchaChallenge( kind=kind, @@ -361,6 +373,7 @@ def detect_recaptcha_in_page(evaluate, page_url: str = "") -> Optional[CaptchaCh page_url=page_url, source="runtime", size=size, + enterprise=bool(info.get("enterprise")), ) @@ -414,6 +427,8 @@ def reconcile_detections(html_challenge: Optional[CaptchaChallenge], # path often can't see it. if html_challenge.action and html_challenge.action != "verify": runtime_challenge.action = html_challenge.action + runtime_challenge.enterprise = (runtime_challenge.enterprise + or html_challenge.enterprise) return runtime_challenge @@ -590,6 +605,12 @@ def solve_recaptcha(challenge: CaptchaChallenge, twocaptcha_api_key: Optional[st "API you may not need either: Captcha.setAutoSolve can clear it " "inside the browser." ) + if getattr(challenge, "enterprise", False): + # Checked before any request is made, so nothing is billed. + raise RuntimeError( + "This is a reCAPTCHA Enterprise widget. This repo does not implement " + "the enterprise task type yet, and an ordinary reCAPTCHA token would " + "be rejected by it, so no solve was bought.") solver = _solve_with_2captcha_v1 if api_version == "v1" else _solve_with_2captcha_v2 return solver(twocaptcha_api_key, challenge, min_score=min_score) @@ -653,4 +674,4 @@ def solve_recaptcha(challenge: CaptchaChallenge, twocaptcha_api_key: Optional[st # what the address has been doing — a narrow list is how a challenge gets # reported as an empty page months later. A solver for a challenge this site # has never been observed to serve is dead code; a DETECTOR for one is cheap -# insurance. See the family note in CLAUDE.md. +# insurance. diff --git a/fingerprint_client.py b/fingerprint_client.py index e84ce1e..0543e1e 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -331,7 +331,7 @@ def main() -> int: # loader. It used to read `os.environ` alone, which meant a key put in # `.env` exactly as the README and .env.example instruct worked for every # engine and failed HERE with "No API key" — a documented mechanism not - # applied on one path, which is the shape of half the defects §16 lists. + # applied on one path, a common shape of defect in copied code. # # `.env` has to be LOADED before it can be read: `env_value` looks at # os.environ, and `load_env` is what fills that from the file. Calling it diff --git a/output_writer.py b/output_writer.py index e4c1685..2cec704 100644 --- a/output_writer.py +++ b/output_writer.py @@ -24,14 +24,17 @@ gets the right header for the mode that produced it. """ +import contextlib import csv import json +import os +import tempfile from dataclasses import dataclass, asdict, field, fields from datetime import datetime, timezone from typing import Optional, List, Set, Sequence, Any, Type -# The hostname a row came from. MediaMarkt is one platform across eleven +# The hostname a row came from. MediaMarkt is one platform across ten # country sites (plus MediaWorld in Italy, which is the same company and the # same markup under a different brand), and which one produced a row is not # derivable from the sku: the same article number exists on several sites at @@ -170,6 +173,109 @@ def dedupe_by_sku(rows: Sequence[Any], seen: Set[str]) -> List[Any]: LIST_CSV_SEPARATOR = " | " +# =========================================================================== +# Replacing a file without destroying the one that is already there +# =========================================================================== +@contextlib.contextmanager +def _atomic(path: str, newline: Optional[str] = None): + """Write to a temporary file beside `path`, then rename over it. + + `open(path, "w")` truncates before a single byte is written, so a crash, + a kill, a full disk or a Ctrl-C halfway through `json.dump` leaves a + SHORTER file where a complete one was. That breaks exactly the promise + `save()` keeps by refusing to overwrite good output with an empty + result: the previous good run is destroyed by the ATTEMPT to replace it + rather than by its outcome. Reproduced on a sibling repo before the fix + — a 2,084-byte good `out.json` came back 0 bytes and invalid JSON. + + The sidecar is the part that matters most. `.meta.json` is the file + a consumer branches on, so a truncated one beside good rows reads as a + broken run over data that is fine. + + Three details, each of which is the reason for a line here: + + * the temp file goes in the target's OWN directory, because `os.replace` + is atomic only within one filesystem and a temp file under /tmp can be + on another, where the rename degrades to an interruptible copy; + * `fsync` before the rename is what makes the bytes durable rather than + merely visible, so a power loss cannot leave a renamed empty file; + * `newline` is passed through because the csv module requires + `newline=""` and that requirement has to survive the move into the + temp file, or every row gains a stray CR on Windows. + """ + directory = os.path.dirname(os.path.abspath(path)) or "." + handle = tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", newline=newline, dir=directory, + prefix=os.path.basename(path) + ".", suffix=".tmp", delete=False) + try: + with handle: + yield handle + handle.flush() + os.fsync(handle.fileno()) + # `NamedTemporaryFile` creates at 0600 and `os.replace` keeps the + # temp file's mode, so without this every output comes out readable + # by its owner alone — a regression from `open(path, "w")`, which + # honours the umask, and one that only appears once the write became + # atomic. Measured across the family on 2026-10-08 by CALLING each + # repo's sidecar writer: of 43 repos, 8 produce a 0600 sidecar, and + # they are exactly the ones that took `_atomic` without this. + # + # An existing target keeps its OWN mode, because someone may have + # tightened it on purpose and a save should not undo that. A new one + # gets exactly what `open()` would have given it. Lifted from + # hackernews-scraper, which fixed this first — the first version + # written here used a flat `0o644 & ~umask`, which quietly narrows + # 0664 to 0644 under umask 002 and clobbers a deliberately tightened + # file. + try: + mode = os.stat(path).st_mode & 0o777 + except OSError: + umask = os.umask(0) + os.umask(umask) + mode = 0o666 & ~umask + os.chmod(handle.name, mode) + os.replace(handle.name, path) + except BaseException: + # Including KeyboardInterrupt and SystemExit: an interrupted run + # must not leave `out.json.ab12cd.tmp` behind beside the good file. + try: + os.unlink(handle.name) + except OSError: + pass + raise + + +# A spreadsheet treats a cell beginning with one of these as a FORMULA +# rather than as text, and every string in these rows was written by +# the site or by a seller — a title, a category, a badge, a delivery note. `=HYPERLINK(...)`, `+cmd|...` and `@SUM(...)` are the classic +# shapes; the tab and the two newlines are here because some readers strip +# a leading one and then execute whatever followed it. +CSV_FORMULA_LEADS = ("=", "+", "-", "@", "\t", "\r", "\n") + +# Excel, Google Sheets and LibreOffice all read a leading apostrophe as +# "the rest of this cell is text" and do not display it. A consumer parsing +# the file with the `csv` module DOES see it, which is why the count goes +# into the sidecar rather than staying silent. +CSV_FORMULA_ESCAPE = "'" + + +def _csv_escape(v: Any): + """Neutralise a formula-shaped cell. Returns (value, was_escaped). + + Only STRINGS are touched. Escaping a number would turn `-5` into text + and break every sum a consumer writes over the column, and this schema + has numbers a consumer sums: `price`, `original_price`, `discount_pct`. + + CSV only, on purpose. The JSON output keeps the site's bytes exactly as + served, so the two files deliberately differ, and `csv_cells_escaped` + in the sidecar is what declares that divergence rather than leaving it + to be discovered as a stray apostrophe. + """ + if isinstance(v, str) and v.startswith(CSV_FORMULA_LEADS): + return CSV_FORMULA_ESCAPE + v, True + return v, False + + def _csv_value(v: Any) -> Any: if isinstance(v, (list, tuple)): return LIST_CSV_SEPARATOR.join(str(x) for x in v) @@ -177,11 +283,11 @@ def _csv_value(v: Any) -> Any: def write_json(rows: Sequence[Any], path: str) -> None: - with open(path, "w", encoding="utf-8") as f: + with _atomic(path) as f: json.dump([asdict(r) for r in rows], f, ensure_ascii=False, indent=2) -def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: +def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> int: # An empty result still gets the header row. A zero-byte file makes a # consumer fail on read (no columns to parse) instead of reading a valid # table with zero rows — and "an empty result is still a well-formed @@ -190,11 +296,23 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: # The header comes from `row_cls`, not from the first row, so an empty # run still writes the columns of the mode that produced it. fieldnames = [f.name for f in fields(row_cls)] - with open(path, "w", encoding="utf-8", newline="") as f: + escaped = 0 + with _atomic(path, newline="") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() for r in rows: - writer.writerow({k: _csv_value(v) for k, v in asdict(r).items()}) + row = {} + for k, v in asdict(r).items(): + # AFTER the list serialiser, not before it. A list is not a + # string, so `_csv_escape` would skip it — and the joined + # cell it becomes can start with a formula character + # ("@SUM(1) | Free Delivery"), which is the case a naive + # ordering misses. + value, was_escaped = _csv_escape(_csv_value(v)) + row[k] = value + escaped += was_escaped + writer.writerow(row) + return escaped # Exit code used when a run completes but produced nothing. Distinct from 1 @@ -262,7 +380,7 @@ def write_run_meta(out_prefix: str, meta: dict) -> str: both complete, and between runs of different `mode`. """ path = f"{out_prefix}.meta.json" - with open(path, "w", encoding="utf-8") as f: + with _atomic(path) as f: json.dump(meta, f, ensure_ascii=False, indent=2) print(f"[+] Wrote run metadata -> {path} (status={meta.get('status')})") return path @@ -271,7 +389,8 @@ def write_run_meta(out_prefix: str, meta: dict) -> str: def run_meta(status: str, stop_reason: str, pages_requested: int, pages_completed: int, start_url: str, final_url: str, products: int, pages_failed: Optional[List[int]] = None, - mode: str = "listing", source: str = SOURCE_DEFAULT) -> dict: + mode: str = "listing", source: str = SOURCE_DEFAULT, + extra: Optional[dict] = None) -> dict: """Build the metadata dict for a finished run. `status` is the field a consumer branches on: @@ -292,8 +411,12 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, a description once pages can be fetched independently and page 3 can fail while 4 and 5 succeed. Recording the numbers keeps the sidecar honest about WHICH part of the catalogue is missing, not just how much. + + `extra` carries facts about the run that are not about any single row, + such as `csv_cells_escaped`. It is merged at the top level, and the run + fields above win a name collision, so a caller cannot overwrite `status`. """ - return { + meta = { "source": source, "mode": mode, "status": status, @@ -306,10 +429,14 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, "final_url": final_url, "finished_at": datetime.now(timezone.utc).isoformat(), } + if extra: + meta.update({k: v for k, v in extra.items() if k not in meta}) + return meta def save(rows: Sequence[Any], out_prefix: str, fmt: str, - allow_empty: bool = False, row_cls: Type = Product) -> int: + allow_empty: bool = False, row_cls: Type = Product, + stats: Optional[dict] = None) -> int: """Write JSON/CSV and return a process exit code. Returns 0 when rows were written, EXIT_NO_PRODUCTS when there were none. @@ -336,8 +463,19 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, write_json(rows, f"{out_prefix}.json") print(f"[+] Saved {len(rows)} products -> {out_prefix}.json") if fmt in ("csv", "both"): - write_csv(rows, f"{out_prefix}.csv", row_cls=row_cls) + escaped = write_csv(rows, f"{out_prefix}.csv", row_cls=row_cls) print(f"[+] Saved {len(rows)} products -> {out_prefix}.csv") + # Reported through `stats` rather than as a return value, because + # this function's return IS the exit code. The caller threads it into + # the sidecar, where a consumer comparing the CSV against the JSON + # can see that the two diverge and by how much. + if stats is not None: + stats["csv_cells_escaped"] = escaped + if escaped: + print(f"[!] {escaped} CSV cell(s) began with a formula character " + f"and were prefixed with {CSV_FORMULA_ESCAPE!r} so a " + f"spreadsheet reads them as text. The JSON output is " + f"unchanged — see csv_cells_escaped in the sidecar.") return 0 if rows else EXIT_NO_PRODUCTS @@ -364,7 +502,8 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, pages_requested: int, pages_completed: int, start_url: str, final_url: str, pages_failed: Optional[List[int]] = None, - mode: str = "listing", source: str = SOURCE_DEFAULT) -> int: + mode: str = "listing", source: str = SOURCE_DEFAULT, + extra: Optional[dict] = None) -> int: """Write output + the run-metadata sidecar; return the exit code. Shared by all three browser engines so the status/exit-code mapping @@ -391,17 +530,28 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, # names has a hole for every name nobody added to it. complete = stop_reason in COMPLETE_STOP_REASONS and not pages_failed row_cls = ROW_CLASS_BY_MODE.get(mode, Product) - rc = save(rows, out_prefix, fmt, allow_empty=allow_empty, row_cls=row_cls) + write_stats: dict = {} + rc = save(rows, out_prefix, fmt, allow_empty=allow_empty, row_cls=row_cls, + stats=write_stats) wrote_output = bool(rows) or allow_empty if wrote_output: - status = "complete" if (rows and complete) else ( + # A zero-row run the user opted into with --allow-empty, which ended + # for a COMPLETE reason with no failed page, is an empty listing read + # in full: `complete`, not `failed`. Only a zero-row run that did not + # finish is `failed`. + status = "complete" if complete else ( "partial" if rows else "failed") + # UNDER the caller's `extra`, never over it: a collision would be + # housekeeping silently dropping something the engine measured. + if write_stats: + extra = {**write_stats, **(extra or {})} write_run_meta(out_prefix, run_meta( status=status, stop_reason=stop_reason, pages_requested=pages_requested, pages_completed=pages_completed, pages_failed=pages_failed, mode=mode, source=source, - start_url=start_url, final_url=final_url, products=len(rows))) + start_url=start_url, final_url=final_url, products=len(rows), + extra=extra)) if not rows: # Nothing gathered at all, and WHY decides the code. The three diff --git a/page_flow.py b/page_flow.py index e93ad68..5164484 100644 --- a/page_flow.py +++ b/page_flow.py @@ -275,3 +275,85 @@ def attempts(value) -> int: f"With 0 the load loop never runs, nothing is fetched, and the " f"engine reports the browser's blank page as a result.") return number + + +def _bounded_int(flag: str, minimum: int, why: str): + """An argparse `type=` that rejects integers below `minimum`, naming the flag.""" + def parse(value) -> int: + try: + number = int(value) + except (TypeError, ValueError): + raise _argparse.ArgumentTypeError( + f"{flag} must be an integer, not {value!r}") + if number < minimum: + raise _argparse.ArgumentTypeError( + f"{flag} must be at least {minimum}, not {number}. {why}") + return number + parse.__name__ = flag.lstrip("-").replace("-", "_") + return parse + + +def _non_negative_float(flag: str): + """An argparse `type=` for a delay in seconds: 0 or more, naming the flag.""" + def parse(value) -> float: + try: + number = float(value) + except (TypeError, ValueError): + raise _argparse.ArgumentTypeError( + f"{flag} must be a number of seconds, not {value!r}") + if not number >= 0: # also rejects NaN + raise _argparse.ArgumentTypeError( + f"{flag} must be 0 or more seconds, not {value}. " + f"time.sleep() raises on a negative value.") + return number + parse.__name__ = flag.lstrip("-").replace("-", "_") + return parse + + +# A negative --pages planned no page at all and wrote nothing; a negative +# --concurrency or --proxy-block-retries made `range()` empty; a negative +# delay crashed in time.sleep() after page 1 had been fetched. Each is a typo +# that deserves a usage error (exit 2) naming the flag, in every engine. +pages_arg = _bounded_int("--pages", 1, "It is the number of listing pages to fetch.") +concurrency_arg = _bounded_int("--concurrency", 1, "It is the number of workers.") +block_retries_arg = _bounded_int( + "--proxy-block-retries", 0, + "It counts EXTRA exits to try after a refusal; 0 means do not rotate.") +delay_arg = _non_negative_float("--delay") +# scraper_api_client.py counts EXTRA attempts (each one billable), so 0 is +# legal there and means "no retry"; only a negative value is a typo. +extra_retries_arg = _bounded_int( + "--retries", 0, "Here it counts EXTRA attempts after the first; 0 means none.") +api_retry_delay_arg = _bounded_int("--retry-delay", 0, "It is a wait in seconds.") +retry_delay_arg = _non_negative_float("--retry-delay") + + +# --------------------------------------------------------------------------- +# Paid captcha solves: one allowance per page +# --------------------------------------------------------------------------- +# A solve is a purchase. Every engine checks the captcha once per fetch +# attempt, and a refused page is fetched again from another exit up to +# --proxy-block-retries times, so without an allowance one page could buy a +# solve on every attempt: measured before this existed, a pool of 4 exits +# with --proxy-block-retries 3 bought 4 solves for one page. +SOLVES_PER_PAGE = 1 + + +class SolveBudget: + """How many paid solves one page may still buy. + + Created once per page, BEFORE the block-retry loop, and passed to every + place that can buy a solve. It is not reset by a retry or by a rotation: + a fresh exit is a reason to fetch again, not a fresh allowance. The check + happens before the purchase, so a refused `take()` costs nothing. + """ + + def __init__(self, limit: int = SOLVES_PER_PAGE): + self.limit = limit + self.spent = 0 + + def take(self) -> bool: + if self.spent >= self.limit: + return False + self.spent += 1 + return True diff --git a/playwright_scraper.py b/playwright_scraper.py index db00588..453f2b0 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -484,7 +484,7 @@ def _content_when_settled(page, attempts: int = 4, pause_ms: int = 700): return None -def handle_captcha_if_present(page, args) -> bool: +def handle_captcha_if_present(page, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Runs after EVERY navigation, for ANY page — not scoped to one URL. The @@ -500,6 +500,11 @@ def handle_captcha_if_present(page, args) -> bool: for the account, newsletter and checkout flows where the site does use reCAPTCHA, and because the family's rule is that detection stays broad: different geos and scenarios surface different challenges. + + `budget` is the page's paid-solve allowance (page_flow.SolveBudget). It + is required, and checked immediately before the purchase, so a page that + is fetched again after a refusal or from another exit cannot buy a second + solve. """ html = _content_when_settled(page) if html is None: @@ -538,6 +543,10 @@ def handle_captcha_if_present(page, args) -> bool: "solved — continuing with whatever the page already " "holds.") return False + if not budget.take(): + logger.warning("This page has already used its %d paid solve(s) — " + "not buying another one for it.", budget.limit) + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -587,15 +596,24 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # the same address just burns it further. block_retries = args.proxy_block_retries if (pool and len(pool) > 1) else 0 html, state, load_failed = None, "ok", False + # One paid-solve allowance for the whole page, across every retry and + # every rotation below. + budget = page_flow.SolveBudget() for block_attempt in range(block_retries + 1): logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) # Retry a navigation timeout rather than ending the run on it. One # network flap on page 12 of 50 should not break the loop. load_failed, exit_failed = False, None + status = None for attempt in range(1, args.retries + 1): try: - session.page.goto(url, wait_until="domcontentloaded", timeout=60000) + response = session.page.goto(url, wait_until="domcontentloaded", + timeout=60000) + # The HTTP status is the strongest block signal this site + # gives (its refusal is its own branded page under a 403), + # so it goes to the classifier rather than being dropped. + status = response.status if response is not None else None load_failed = False break except (PWTimeout, PWError) as e: @@ -627,13 +645,15 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if load_failed: break - if handle_captcha_if_present(session.page, args): + if handle_captcha_if_present(session.page, args, budget): # A solve navigated the page. Give the destination a moment - # before judging what came back. + # before judging what came back, and forget the status of the + # page the solve replaced: it describes the challenge, not this. session.page.wait_for_timeout(1000) + status = None html = _content_when_settled(session.page) or "" - state = _classify(session.page, html) + state = _classify(session.page, html, status=status) if not page_flow.should_retry(state): # "content" and "empty" are both final answers. An empty page is @@ -1140,11 +1160,11 @@ def parse_args(): "because the flag was omitted. A search URL has no " "category in its path, so pass one if you want the " "column filled on a search run.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.pages_arg, default=1, help="Number of listing pages to crawl. Ignored outside " "--mode listing.") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--delay", type=page_flow.delay_arg, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.concurrency_arg, default=1, metavar="N", help="Fetch pages through N parallel workers (default 1 — " "unchanged sequential behaviour). Each worker runs its " "own browser and holds its own proxy exit, so N>1 " @@ -1156,7 +1176,7 @@ def parse_args(): "that comes back EMPTY is not retried — see " "page_flow.STATE_POLICY — because an empty hub " "category is a correct answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", type=page_flow.retry_delay_arg, default=2.0, help="Seconds before the first page-load retry, doubling " "thereafter (default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") @@ -1182,7 +1202,7 @@ def parse_args(): p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup, so concurrent runs do not " "all begin on the first exit in the file.") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", type=page_flow.block_retries_arg, default=2, help="When a page comes back refused (HTTP 403) or behind " "a captcha, retry it from this many OTHER exits before " "giving up (default 2). Needs a pool of more than one; " diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index ad2fe8c..83509ec 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -364,7 +364,7 @@ def _next_page_href(session) -> Optional[str]: NEXT_PAGE_SELECTOR)) -def handle_captcha_if_present(session, args) -> bool: +def handle_captcha_if_present(session, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Same two families, same order, same "detected is not blocking" rule as @@ -396,6 +396,10 @@ def handle_captcha_if_present(session, args) -> bool: if not args.twocaptcha_key: logger.warning("No 2captcha API key, so this challenge cannot be solved.") return False + if not budget.take(): + logger.warning("This page has already used its %d paid solve(s) — " + "not buying another one for it.", budget.limit) + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -423,15 +427,22 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome d = _driver(session) block_retries = args.proxy_block_retries if (pool and len(pool) > 1) else 0 + # One paid-solve allowance for the whole page, across every retry and + # every rotation below. + budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False for block_attempt in range(block_retries + 1): logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) load_failed = False + status = None for attempt in range(1, args.retries + 1): try: - bridge.run(page.goto(url, {"waitUntil": "domcontentloaded", - "timeout": 60000})) + response = bridge.run(page.goto(url, {"waitUntil": "domcontentloaded", + "timeout": 60000})) + # The HTTP status is the strongest block signal this site + # gives, so it goes to the classifier rather than being dropped. + status = response.status if response is not None else None load_failed = False break except Exception as e: # noqa: BLE001 — pyppeteer raises many types @@ -462,11 +473,12 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome break - if handle_captcha_if_present(session, args): + if handle_captcha_if_present(session, args, budget): time.sleep(1) + status = None # the status described the page the solve replaced html = _content(session) or "" - state = page_flow.classify(html, url=page.url) + state = page_flow.classify(html, status=status, url=page.url) if not page_flow.should_retry(state): @@ -761,16 +773,16 @@ def parse_args(): "the full image list — the columns a listing row " "cannot carry. No --pages in product mode.") p.add_argument("--category", default=None, help="Label to tag output rows with.") - p.add_argument("--pages", type=int, default=1, help="Listing pages to crawl") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--pages", type=page_flow.pages_arg, default=1, help="Listing pages to crawl") + p.add_argument("--delay", type=page_flow.delay_arg, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.concurrency_arg, default=1, metavar="N", help="Accepted for flag parity and IGNORED here: parallel " "page fetching lives in playwright_scraper.py.") p.add_argument("--retries", type=page_flow.attempts, default=3, help="Attempts per page load before giving up (default 3). " "A page that comes back EMPTY is not retried: an empty " "hub category is a correct answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", type=page_flow.retry_delay_arg, default=2.0, help="Seconds before the first retry, doubling thereafter") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="mediamarkt_products", help="Output file prefix") @@ -783,7 +795,7 @@ def parse_args(): "Wins over --proxy.") p.add_argument("--proxy-rotate", choices=list(ROTATE_MODES), default="per-run") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2) + p.add_argument("--proxy-block-retries", type=page_flow.block_retries_arg, default=2) p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 rows were found.") diff --git a/scraper_api_client.py b/scraper_api_client.py index 4221b64..437b265 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -109,6 +109,7 @@ detect_page_state, BOT_CHALLENGE_MARKERS) from output_writer import save import env_config +import page_flow logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("scraper_api_client") @@ -368,17 +369,17 @@ def parse_args(): "on protected sites — a DOM/load wait is satisfied instantly by " "the challenge page itself.") wait.add_argument("--wait-element", default=None, - help="Wait until this CSS selector is visible, e.g. 'a[href*=\"-item-\"]'") + help="Wait until this CSS selector is visible, e.g. '[data-test=\"mms-product-card\"]'") wait.add_argument("--wait-state", choices=["load", "domcontentloaded"], default=None, help="Wait for a page load state instead of specific content") p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 products were parsed. Off by " "default so a failed fetch can't overwrite a good result.") - p.add_argument("--retries", type=int, default=1, + p.add_argument("--retries", type=page_flow.extra_retries_arg, default=1, help="Extra attempts if a bot-challenge page comes back. One retry is " "usually worth it. Each attempt is a separate billable task, so " "this defaults to 1.") - p.add_argument("--retry-delay", type=int, default=10, + p.add_argument("--retry-delay", type=page_flow.api_retry_delay_arg, default=10, help="Seconds between retries (default 10)") p.add_argument("--dump-html", default=None, help="Also write the raw returned HTML to this path (always, even on success)") diff --git a/selenium_scraper.py b/selenium_scraper.py index 05fb8f8..b716aba 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -378,7 +378,7 @@ def _next_page_href(session) -> Optional[str]: return None -def handle_captcha_if_present(session, args) -> bool: +def handle_captcha_if_present(session, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Same detectors, same reconciliation and the same "detected is not @@ -416,6 +416,10 @@ def handle_captcha_if_present(session, args) -> bool: if not args.twocaptcha_key: logger.warning("No 2captcha API key, so this challenge cannot be solved.") return False + if not budget.take(): + logger.warning("This page has already used its %d paid solve(s) — " + "not buying another one for it.", budget.limit) + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -446,6 +450,9 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome html, state, load_failed = None, "ok", False block_retries = args.proxy_block_retries if (pool and len(pool) > 1) else 0 + # One paid-solve allowance for the whole page, across every retry and + # every rotation below. + budget = page_flow.SolveBudget() for block_attempt in range(block_retries + 1): logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) @@ -481,11 +488,15 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome break - if handle_captcha_if_present(session, args): + if handle_captcha_if_present(session, args, budget): time.sleep(1) html = d["content"]() or "" - state = page_flow.classify(html, url=d["current_url"]()) + # status=None on purpose: driver.get() returns nothing and Selenium + # exposes no HTTP status, so this engine relies on the text and + # own-asset-host signals in detect_page_state, which catch both + # measured refusal pages without one. + state = page_flow.classify(html, status=None, url=d["current_url"]()) if not page_flow.should_retry(state): @@ -771,16 +782,16 @@ def parse_args(): "the full image list — the columns a listing row " "cannot carry. No --pages in product mode.") p.add_argument("--category", default=None, help="Label to tag output rows with.") - p.add_argument("--pages", type=int, default=1, help="Listing pages to crawl") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--pages", type=page_flow.pages_arg, default=1, help="Listing pages to crawl") + p.add_argument("--delay", type=page_flow.delay_arg, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.concurrency_arg, default=1, metavar="N", help="Accepted for flag parity and IGNORED here: parallel " "page fetching lives in playwright_scraper.py.") p.add_argument("--retries", type=page_flow.attempts, default=3, help="Attempts per page load before giving up (default 3). " "A page that comes back EMPTY is not retried: an empty " "hub category is a correct answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", type=page_flow.retry_delay_arg, default=2.0, help="Seconds before the first retry, doubling thereafter") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="mediamarkt_products", help="Output file prefix") @@ -794,7 +805,7 @@ def parse_args(): "Wins over --proxy.") p.add_argument("--proxy-rotate", choices=list(ROTATE_MODES), default="per-run") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2) + p.add_argument("--proxy-block-retries", type=page_flow.block_retries_arg, default=2) p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 rows were found.") diff --git a/smoke_test.py b/smoke_test.py index 472ec15..ff1f7fa 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1202,6 +1202,33 @@ def test_captcha(): ok = True from captcha_solver import CaptchaChallenge + # A challenge frame settles the variant as v2: v3 never renders one. A + # frame with no `size` and no `render` used to fall through to v3, and a + # paid solve would have bought the wrong task type. + frame = {"found": True, "sitekey": "k" * 40, "size": None, + "renderParam": None, "challengeFrame": True} + got = detect_recaptcha_in_page(lambda js: frame) + ok &= check("a challenge frame with no size/render is v2, not v3 (got %s)" + % (got and got.kind), got is not None and got.kind == "recaptcha_v2") + got = detect_recaptcha_in_page(lambda js: dict(frame, size="invisible")) + ok &= check("a challenge frame with size=invisible is v2-invisible", + got is not None and got.kind == "recaptcha_v2_invisible") + got = detect_recaptcha_in_page(lambda js: dict(frame, size="invisible", + challengeFrame=False)) + ok &= check("no frame and size=invisible is still read as v3", + got is not None and got.kind == "recaptcha_v3") + # Enterprise: this repo does not implement that task type, so nothing is + # bought for it. Raised before any request, so no key is ever sent. + from captcha_solver import solve_recaptcha + ent = detect_recaptcha_in_page(lambda js: dict(frame, enterprise=True)) + try: + solve_recaptcha(ent, "not-a-key") + refused = False + except RuntimeError as e: + refused = "Enterprise" in str(e) + ok &= check("an Enterprise reCAPTCHA is flagged and refused before any " + "solve is bought", ent.enterprise and refused) + # Format 2: the site's own wrapper element carries the config as # attributes, with the execute() call inside a bundled file that never # appears as readable inline script. @@ -1770,10 +1797,10 @@ def test_fingerprint_client_reads_env(): # THE DEFECT THIS PINS, found the first time --fingerprint was run live # here and confirmed present in five sibling repos: `--key` defaulted to # `os.environ.get("TWOCAPTCHA_KEY")` alone. So a key put in `.env` — - # which is exactly what §3, the README and .env.example instruct — worked + # which is exactly what the README and .env.example instruct — worked # for every engine and failed HERE with "No API key". A documented - # mechanism not applied on one path, which is the shape of half the - # defects §16 lists. + # mechanism not applied on one path, a common shape of defect in + # copied code. src = inspect.getsource(fpc.main) ok &= check("it loads .env itself, rather than hoping an engine did", "env_config.load_env()" in src) @@ -2088,7 +2115,7 @@ def reached(module, seen=None): def test_shared_calls_bind_against_the_real_signature(): - """§17's check #1: bind every call into a shared module against the + """Bind every call into a shared module against the callee's real signature. This exists because of a defect that no other check in this suite can @@ -2409,6 +2436,417 @@ def test_retries_is_validated_in_every_entry_point(): return check("--retries is validated in every entry point, so 0 is a usage error", not problems) + +# --------------------------------------------------------------------------- +# Output files are replaced atomically, with the modes open() would give +# --------------------------------------------------------------------------- +def test_atomic_writes(): + group("output writes are atomic and keep the umask's modes") + ok = True + import output_writer as ow + with tempfile.TemporaryDirectory() as d: + prefix = os.path.join(d, "out") + good = [Product(sku="1", url="u1", price=1.0)] + save(good, prefix, "both") + before_json = open(prefix + ".json").read() + before_csv = open(prefix + ".csv").read() + + class Boom(Exception): + pass + + def exploding_asdict(_row): + raise Boom("disk full halfway through") + + real_asdict = ow.asdict + ow.asdict = exploding_asdict + try: + for writer, path in ((lambda: ow.write_json(good * 3, prefix + ".json"), + prefix + ".json"), + (lambda: ow.write_csv(good * 3, prefix + ".csv"), + prefix + ".csv")): + try: + writer() + except Boom: + pass + finally: + ow.asdict = real_asdict + ok &= check("a write that fails halfway leaves the previous JSON intact", + open(prefix + ".json").read() == before_json) + ok &= check("a write that fails halfway leaves the previous CSV intact", + open(prefix + ".csv").read() == before_csv) + leftovers = [n for n in os.listdir(d) if n.endswith(".tmp")] + ok &= check("a failed write leaves no temp file behind (%s)" % leftovers, + not leftovers) + + # The sidecar goes through the same writer. + meta_path = ow.write_run_meta(prefix, {"status": "complete"}) + ok &= check("the sidecar is written through the atomic writer", + "_atomic(" in inspect.getsource(ow.write_run_meta) + and json.load(open(meta_path))["status"] == "complete") + + # Modes: a NEW file gets exactly what open() would give it under the + # current umask, and an EXISTING file keeps its own mode. A flat + # 0o644 would pass under 022 and fail under 002, so both are tried. + if os.name == "posix": + modes = {} + for umask in (0o022, 0o002, 0o077): + old = os.umask(umask) + try: + p = os.path.join(d, "new_%o.json" % umask) + ow.write_json(good, p) + modes[umask] = os.stat(p).st_mode & 0o777 + finally: + os.umask(old) + ok &= check("a new output file honours the umask (022->644, " + "002->664, 077->600): got %s" + % {oct(k): oct(v) for k, v in modes.items()}, + modes == {0o022: 0o644, 0o002: 0o664, 0o077: 0o600}) + p = os.path.join(d, "kept.json") + ow.write_json(good, p) + os.chmod(p, 0o640) + ow.write_json(good, p) + ok &= check("an existing output file keeps its own mode", + os.stat(p).st_mode & 0o777 == 0o640) + return ok + + +def test_csv_formula_guard(): + group("CSV cells that a spreadsheet would execute are neutralised") + ok = True + row = Product(sku="1", url="u", title="=HYPERLINK(\"http://x\")", + category="+cmd", price=-5.0, images=["@SUM(1)", "b"]) + with tempfile.TemporaryDirectory() as d: + prefix = os.path.join(d, "out") + rc = finish_run([row], prefix, "both", False, blocked=False, + stop_reason="completed", pages_requested=1, + pages_completed=1, start_url="u", final_url="u", + extra={"total_results": 7}) + import csv as _csv + cells = next(_csv.DictReader(open(prefix + ".csv", newline="", + encoding="utf-8"))) + ok &= check("a formula-shaped title is prefixed with an apostrophe in CSV", + cells["title"] == "'=HYPERLINK(\"http://x\")") + ok &= check("a +cmd category is prefixed too", cells["category"] == "'+cmd") + ok &= check("a joined list starting with @ is escaped AFTER joining", + cells["images"].startswith("'@SUM(1)")) + ok &= check("a negative NUMBER stays a number (no apostrophe)", + cells["price"] == "-5.0") + data = json.load(open(prefix + ".json"))[0] + ok &= check("the JSON output keeps the site's bytes unescaped", + data["title"].startswith("=HYPERLINK") and data["images"][0] == "@SUM(1)") + meta = json.load(open(prefix + ".meta.json")) + ok &= check("csv_cells_escaped is recorded in the sidecar (3)", + meta.get("csv_cells_escaped") == 3 and rc == 0) + ok &= check("the caller's own extra still reaches the sidecar", + meta.get("total_results") == 7) + # Under the caller's extra, never over it. + finish_run([row], prefix, "both", False, blocked=False, + stop_reason="completed", pages_requested=1, + pages_completed=1, start_url="u", final_url="u", + extra={"csv_cells_escaped": "caller"}) + meta = json.load(open(prefix + ".meta.json")) + ok &= check("csv_cells_escaped is merged UNDER the caller's extra", + meta.get("csv_cells_escaped") == "caller") + # A complete run that the user allowed to be empty is complete. + finish_run([], prefix, "json", True, blocked=False, + stop_reason="completed", pages_requested=1, + pages_completed=1, start_url="u", final_url="u") + meta = json.load(open(prefix + ".meta.json")) + ok &= check("an --allow-empty run that finished is 'complete', not 'failed'", + meta["status"] == "complete") + return ok + + +# --------------------------------------------------------------------------- +# One paid solve per page, in every engine, read off disk +# --------------------------------------------------------------------------- +ENGINE_FILES = ("playwright_scraper.py", "puppeteer_scraper.py", + "selenium_scraper.py") + + +def _solve_budget_problems(root=REPO_ROOT): + """Source-level reasons an engine could buy more than one solve per page. + + Read from disk with `ast`, never through an import, so it runs for every + engine whether or not its driver is installed. + """ + problems = [] + for name in ENGINE_FILES: + tree = ast.parse(open(os.path.join(root, name), encoding="utf-8").read()) + funcs = {n.name: n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef)} + handler = funcs.get("handle_captcha_if_present") + fetch = funcs.get("_fetch_one_page") + if handler is None or fetch is None: + problems.append("%s: handle_captcha_if_present or _fetch_one_page is missing" % name) + continue + # 1. Every purchase in the handler sits after a `budget.take()` guard. + take_lines = [n.lineno for n in ast.walk(handler) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) + and n.func.attr == "take" + and getattr(n.func.value, "id", None) == "budget"] + buys = [n.lineno for n in ast.walk(handler) + if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "solve_recaptcha"] + if not buys: + problems.append("%s: no solve_recaptcha call found (check is stale)" % name) + for line in buys: + if not any(t < line for t in take_lines): + problems.append("%s:%d buys a solve without budget.take() first" % (name, line)) + # 2. Every call of the handler anywhere in the file passes a budget. + for n in ast.walk(tree): + if (isinstance(n, ast.Call) + and getattr(n.func, "id", None) == "handle_captcha_if_present"): + names = {getattr(a, "id", None) for a in n.args} | { + getattr(k.value, "id", None) for k in n.keywords} + if "budget" not in names: + problems.append("%s:%d calls the captcha handler without " + "the page's budget" % (name, n.lineno)) + # 3. The budget is created once per page, BEFORE the block-retry + # loop, so a retry or a rotation cannot reset it. + made = [n.lineno for n in ast.walk(fetch) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) + and n.func.attr == "SolveBudget"] + loops = [n.lineno for n in ast.walk(fetch) if isinstance(n, ast.For)] + if len(made) != 1 or not loops or made[0] > min(loops): + problems.append("%s: the SolveBudget is not created exactly once " + "before the retry loop (made at %s, first loop %s)" + % (name, made, min(loops) if loops else None)) + return problems + + +class _Anything: + """A stand-in driver object: every attribute and call yields another one, + and it is an empty sequence, so `len(page.query_selector_all(...))` is 0.""" + + def __getattr__(self, _name): + return _Anything() + + def __call__(self, *a, **k): + return _Anything() + + def __len__(self): + return 0 + + def __iter__(self): + return iter(()) + + def __str__(self): + return "https://www.mediamarkt.de/de/category/x-1.html" + + +def _solves_bought_by_handler(mod): + """Call an engine's real captcha handler twice with ONE page budget, with a + challenge always present and the solver stubbed; return the purchases.""" + from captcha_solver import CaptchaChallenge + bought = [] + patched = { + "solve_recaptcha": lambda *a, **k: bought.append(1) or "tok", + "reconcile_detections": lambda *a, **k: CaptchaChallenge( + kind="recaptcha_v2", sitekey="k" * 40), + "detect_recaptcha_v3": lambda *a, **k: None, + "detect_recaptcha_in_page": lambda *a, **k: None, + "_content_when_settled": lambda *a, **k: "", + "_content": lambda *a, **k: "", + "_driver": lambda *a, **k: {"content": lambda: "", + "count": lambda sel: 0, + "current_url": lambda: "u"}, + "time": type("T", (), {"sleep": staticmethod(lambda s: None)}), + } + saved = {k: getattr(mod, k) for k in patched if hasattr(mod, k)} + try: + for k, v in patched.items(): + if k in saved: + setattr(mod, k, v) + args = type("A", (), dict(mode="listing", solve_captcha="always", + twocaptcha_key="x", captcha_api="v2", + min_score=0.7))() + budget = page_flow.SolveBudget() + for _ in range(3): + mod.handle_captcha_if_present(_Anything(), args, budget) + finally: + for k, v in saved.items(): + setattr(mod, k, v) + return len(bought) + + +def test_solve_budget(): + group("a page can buy at most SOLVES_PER_PAGE solves, in every engine") + ok = True + for name in ENGINE_FILES: + modname = name[:-3] + try: + mod = __import__(modname) + except ImportError: + continue # recorded as a skip by test_engines + n = _solves_bought_by_handler(mod) + ok &= check("%s: three challenged attempts on one page buy %d solve(s), " + "not more" % (modname, page_flow.SOLVES_PER_PAGE), + n == page_flow.SOLVES_PER_PAGE) + b = page_flow.SolveBudget() + ok &= check("SOLVES_PER_PAGE is 1", page_flow.SOLVES_PER_PAGE == 1) + ok &= check("the first take() is allowed and the second refused", + b.take() is True and b.take() is False and b.spent == 1) + problems = _solve_budget_problems() + for line in problems: + print(" " + line) + ok &= check("every engine routes every solve through the per-page budget", + not problems) + return ok + + +def _status_problems(root=REPO_ROOT): + """Classify call sites in the engines that drop the HTTP status.""" + problems = [] + for name in ENGINE_FILES: + tree = ast.parse(open(os.path.join(root, name), encoding="utf-8").read()) + calls = [n for n in ast.walk(tree) if isinstance(n, ast.Call) + and ((isinstance(n.func, ast.Attribute) and n.func.attr == "classify") + or getattr(n.func, "id", None) == "_classify")] + # The helper _classify(page, html, status=None) forwards its own + # parameter; that one call is the definition's body, not a call site. + calls = [c for c in calls + if not (isinstance(c.func, ast.Attribute) and c.func.attr == "classify" + and any(k.arg == "status" and getattr(k.value, "id", None) == "status" + for k in c.keywords) + and name == "playwright_scraper.py" + and c.lineno < 200)] + if not calls: + problems.append("%s: no classify call found (check is stale)" % name) + for c in calls: + kw = {k.arg: k.value for k in c.keywords} + if "status" not in kw: + problems.append("%s:%d classifies without passing status=" % (name, c.lineno)) + elif name != "selenium_scraper.py" and isinstance(kw["status"], ast.Constant): + problems.append("%s:%d passes a constant status; this driver " + "reports a real one" % (name, c.lineno)) + return problems + + +def test_status_reaches_the_classifier(): + group("the HTTP status reaches the page classifier") + ok = True + problems = _status_problems() + for line in problems: + print(" " + line) + ok &= check("Playwright and pyppeteer pass goto()'s status to classify; " + "Selenium passes an explicit None", not problems) + ok &= check("a 403 classifies as blocked whatever the body says", + page_flow.classify('x', + status=403) == "blocked") + # Behaviour, where the driver is installed: a 403 from goto() must decide + # the page even when the body looks like content. + try: + import playwright_scraper as pw + except ImportError: + return ok # recorded as a skip by test_engines + + class Resp: + status = 403 + + class Page: + url = "https://www.mediamarkt.de/de/category/x-1.html" + + def goto(self, *a, **k): + return Resp() + + def __getattr__(self, _name): + return lambda *a, **k: None + + class Session: + page = Page() + + body = 'x' + saved = (pw._content_when_settled, pw.handle_captcha_if_present) + pw._content_when_settled = lambda page, *a, **k: body + pw.handle_captcha_if_present = lambda *a, **k: False + try: + with tempfile.TemporaryDirectory() as d: + args = type("A", (), dict(proxy_block_retries=0, pages=1, retries=1, + retry_delay=0, mode="listing", + out=os.path.join(d, "o"), dump_html=None, + category=None))() + with redirect_stdout(io.StringIO()): + outcome = pw._fetch_one_page(Session(), args, None, 1, Page.url) + finally: + pw._content_when_settled, pw.handle_captcha_if_present = saved + ok &= check("playwright: a 403 from goto() makes the page 'blocked' even " + "with a product link in the body (got %r)" % outcome.state, + outcome.state == "blocked") + return ok + + +def test_numeric_flags_are_validated(): + group("numeric flags reject values that make no sense, naming the flag") + ok = True + bad = {"--pages": ["0", "-1"], "--concurrency": ["0", "-2"], + "--proxy-block-retries": ["-1"], "--delay": ["-1"], + "--retry-delay": ["-0.5"]} + for name in ENGINE_FILES: + src = open(os.path.join(REPO_ROOT, name), encoding="utf-8").read() + for flag, typ in (("--pages", "pages_arg"), ("--concurrency", "concurrency_arg"), + ("--proxy-block-retries", "block_retries_arg"), + ("--delay", "delay_arg"), ("--retry-delay", "retry_delay_arg")): + ok &= check("%s: %s goes through page_flow.%s" % (name, flag, typ), + re.search(r'add_argument\("%s",\s*type=page_flow\.%s\b' + % (re.escape(flag), typ), src) is not None) + import argparse as _ap + for flag, values in bad.items(): + typ = {"--pages": page_flow.pages_arg, "--concurrency": page_flow.concurrency_arg, + "--proxy-block-retries": page_flow.block_retries_arg, + "--delay": page_flow.delay_arg, + "--retry-delay": page_flow.retry_delay_arg}[flag] + for v in values: + try: + typ(v) + raised = "" + except _ap.ArgumentTypeError as e: + raised = str(e) + ok &= check("%s %s is rejected with a message naming the flag" % (flag, v), + raised.startswith(flag)) + ok &= check("valid values pass: --pages 3, --proxy-block-retries 0, --delay 0", + page_flow.pages_arg("3") == 3 and page_flow.block_retries_arg("0") == 0 + and page_flow.delay_arg("0") == 0.0) + api = open(os.path.join(REPO_ROOT, "scraper_api_client.py"), encoding="utf-8").read() + ok &= check("scraper_api_client: --retries (extra attempts) rejects a " + "negative value and still allows 0", + "type=page_flow.extra_retries_arg" in api + and page_flow.extra_retries_arg("0") == 0) + return ok + + +def test_no_private_citations(): + group("public files cite no private instruction file") + ok = True + # Assembled from pieces so this file does not match itself. + banned = ["CLAUDE" + ".md", "claude_" + "2scraper", "2scraper_" + "repo_rules", + "play" + "book"] + section = re.compile("§" + r"\s*\d") + try: + tracked = subprocess.run(["git", "ls-files"], cwd=REPO_ROOT, + capture_output=True, text=True) + except OSError: + tracked = None + if tracked is None or tracked.returncode != 0: + print(" (not a git checkout: scanning nothing, which is the " + "Docker build's case)") + return ok + hits = [] + for rel in tracked.stdout.split(): + path = os.path.join(REPO_ROOT, rel) + if not os.path.isfile(path) or rel.endswith((".png", ".ico")): + continue + text = open(path, encoding="utf-8", errors="replace").read() + for token in banned: + if token.lower() in text.lower(): + hits.append("%s: %s" % (rel, token)) + if section.search(text): + hits.append("%s: a bare section reference" % rel) + for h in hits: + print(" " + h) + ok &= check("no tracked file cites a private instruction file or its " + "section numbers", not hits) + return ok + + def main() -> int: ok = True # Checks that could not run because an optional engine library is absent. @@ -2450,6 +2888,12 @@ def main() -> int: ok &= test_sample_output() ok &= test_x_debug_header_is_redacted() ok &= test_scraper_api_payload_and_status() + ok &= test_atomic_writes() + ok &= test_csv_formula_guard() + ok &= test_solve_budget() + ok &= test_status_reaches_the_classifier() + ok &= test_numeric_flags_are_validated() + ok &= test_no_private_citations() print() if _failures: