From e3e579d3bc8c13e8374420747dc797e37b11c7ab Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:49:53 +0000 Subject: [PATCH] Blocks no longer crash the mirror engines, solves stay within budget, and the canary is red unless it tested Etsy - pyppeteer/Selenium: a t=bv block or a dead exit with no proxy pool (the --cdp-endpoint case) re-fetches instead of crashing on pool=None. - One per-page solve allowance covers the reCAPTCHA and DataDome call sites in all three engines, checked before each purchase. - A served page that states listings and parses to none is parser_found_nothing (exit 5/6, not 4 or a complete run); an HTTP 404/410 reaches the classifier as not_found (Playwright, pyppeteer). - New outputs follow the umask; existing files keep their mode. - Credential scan: exclusions relative to the repo, an empty scan fails, data files and .env.* backups are read (ignored ones warn). - .gitignore covers top-level and multi-page dumps and live/. - tests.yml runs ci_checks.py instead of inline copies. - Private-note citations removed from public files, with a guard. - Canary: weekly + on demand; exit 3/5/6 and a missing ETSY_CDP_ENDPOINT secret fail the job instead of going green. Co-Authored-By: Claude Opus 5.5 --- .github/ci_checks.py | 65 ++++- .github/workflows/canary.yml | 113 +++----- .github/workflows/tests.yml | 89 +----- .gitignore | 11 +- CHANGELOG.md | 68 ++++- CONTRIBUTING.md | 11 +- README.md | 62 ++-- captcha_solver.py | 16 +- fingerprint_client.py | 2 +- output_writer.py | 21 +- page_flow.py | 64 ++++- playwright_scraper.py | 97 +++++-- product_parser.py | 10 +- puppeteer_scraper.py | 146 +++++++--- selenium_scraper.py | 93 ++++-- smoke_test.py | 536 +++++++++++++++++++++++++++++++---- 16 files changed, 1034 insertions(+), 370 deletions(-) diff --git a/.github/ci_checks.py b/.github/ci_checks.py index bb4ddfb..12dfd23 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -34,7 +34,8 @@ ENGINE_LIBS = ("playwright", "pyppeteer", "selenium", "webdriver_manager") CLIS = ["playwright_scraper.py", "puppeteer_scraper.py", "selenium_scraper.py", - "scraper_api_client.py", "fingerprint_client.py", "env_config.py"] + "scraper_api_client.py", "fingerprint_client.py", "env_config.py", + "diff_runs.py"] SAMPLE_FILES = ("sample_output.json", "sample_output.csv") @@ -85,7 +86,7 @@ # into pip's vendored code and flags a 32-hex string in `_elffile.py` as # key-shaped. Correct about the string, wrong about the file — and it is the # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people -# have to argue with is one they learn to suppress (CLAUDE.md §22). +# have to argue with is one they learn to suppress. # # Deliberately NOT a narrowing of what gets scanned: an untracked file is # still read, because a key pasted into a scratch file beside the scripts is @@ -112,18 +113,55 @@ def _is_in_virtualenv(path): return False +# Read too, though nothing here is source: data and dumps are where a key or +# a session arrives by accident (a fixture captured with a key in a request, +# a `--dump-html` of a page that echoed one), and `.env.bak`/`.env.local` +# are what a working `.env` gets renamed to. These shapes are git-ignored by +# design, so a hit in an IGNORED one is reported as a warning — the ignore +# rule is what keeps it out of a commit, and a local key in a local file is +# not a defect. A hit in one git would NOT ignore fails like any other. +DATA_SUFFIXES = (".json", ".html", ".csv", ".toml", ".cfg", ".ini") +DATA_NAMES = ("Dockerfile",) + +# Skipped by name, and only RELATIVE to the repository: tested against the +# path's absolute parts, a clone that merely lives under a directory called +# `venv` (or `.git`) excluded every file, and the scan reported "0 files +# scanned, nothing credential-shaped" — a clean pass over nothing. +SKIPPED_DIRS = {".git", "__pycache__", ".venv", "venv", ".pytest_cache"} + + +def _is_data_file(path): + name = path.name + return (path.suffix in DATA_SUFFIXES or name in DATA_NAMES + or (name.startswith(".env.") and name != ".env.example")) + + def scanned_files(): for path in sorted(REPO.rglob("*")): - if not path.is_file() or path.suffix not in SCANNED_SUFFIXES: + if not path.is_file(): continue - if any(part in {".git", "__pycache__", ".venv", "venv"} - for part in path.parts): + if path.suffix not in SCANNED_SUFFIXES and not _is_data_file(path): + continue + if any(part in SKIPPED_DIRS for part in path.relative_to(REPO).parts): continue if _is_in_virtualenv(path): continue yield path +def _git_ignored(paths): + """The subset of `paths` git would ignore; empty outside a checkout.""" + if not paths: + return set() + try: + done = subprocess.run( + ["git", "check-ignore", "--stdin"], cwd=REPO, capture_output=True, + text=True, input="\n".join(str(p.relative_to(REPO)) for p in paths)) + except FileNotFoundError: + return set() + return {REPO / line for line in done.stdout.splitlines() if line} + + def help_check(): failed = [] for name in CLIS: @@ -190,25 +228,34 @@ def sample_check(): def secret_check(): - failed = [] + failed, warned = [], [] scanned = 0 - for path in scanned_files(): + files = list(scanned_files()) + ignored_data = _git_ignored([p for p in files if _is_data_file(p)]) + for path in files: scanned += 1 + sink = warned if path in ignored_data else failed for lineno, line in enumerate( path.read_text(encoding="utf-8", errors="replace").splitlines(), 1): rel = path.relative_to(REPO) if CREDENTIALLED_URL.search(line) and not any( token in line for token in CREDENTIAL_ALLOWED): - failed.append(f"{rel}:{lineno} looks like a URL with real " + sink.append(f"{rel}:{lineno} looks like a URL with real " f"credentials in it") for match in HEX32.findall(line): if any(token in line.lower() for token in HEX32_ALLOWED): continue - failed.append(f"{rel}:{lineno} contains {match[:6]}… — a " + sink.append(f"{rel}:{lineno} contains {match[:6]}… — a " f"32-char hex string, the shape of a 2captcha key") + for line in warned: + print(f"WARNING {line} (git-ignored, so not a commit risk while the " + f"ignore rule holds; delete it if it is not yours)") + if scanned == 0: + failed.append("0 files scanned — a scan that read nothing cannot say " + "the repository is clean") if not failed: print(f"ok {scanned} files scanned, nothing credential-shaped") return failed diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 75901bc..d3d03fa 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -4,8 +4,8 @@ name: canary # why. That leaves a real gap: a markup change on the site itself only shows # up when a user hits it and files an issue, while the offline badge stays # green throughout. This workflow is the other half — a real run against a -# real search URL — so a site-side break is caught here first. It runs ON -# DEMAND rather than on a schedule, for the reason set out below. +# real search URL — so a site-side break is caught here first. It runs +# weekly and on demand. # # THIS CANARY CANNOT RUN FROM A BARE RUNNER, AND THAT IS NOT PESSIMISM. # ------------------------------------------------------------------------ @@ -22,10 +22,10 @@ name: canary # browser here. So this canary gates on ETSY_CDP_ENDPOINT — the Scraping # Browser API — rather than on a proxy secret like its sibling repos do. # -# The live steps are SKIPPED, not failed, when the secret is absent. The job -# goes green with a notice saying why. A check that is always red teaches -# everyone to ignore checks, which is worse than having no check at all. Set -# the secret and the same workflow starts doing real work with no edit: +# WITHOUT THAT SECRET THIS JOB FAILS, with an error naming it. It used to +# skip and go green, and a green badge over a run that never reached the site +# reads as "the parser still works" when nothing was tested. Red until the +# secret is set is the honest state: # # gh secret set ETSY_CDP_ENDPOINT --repo 2scraper/etsy-scraper # @@ -40,27 +40,14 @@ name: canary # its own CI; and an unset secret arrives as an empty string, which the # loader already treats as "not configured", so there is no commented-out # line for someone to forget to uncomment. -# ON DEMAND, NOT ON A SCHEDULE — and that is forced by the credential rather -# than chosen. -# -# A Scraping Browser endpoint on this account does not survive a day. A daily -# cron against a credential with that lifetime produces one of two useless -# outcomes: a red badge every day (if a dead endpoint fails the job), or a -# green badge every day that has tested nothing (if it does not). The second -# is the more dangerous, because green reads as "the parser still works". -# -# So the schedule is off. Dispatch it by hand, with a fresh secret, when the -# answer matters — before a release, or when someone reports the parser -# missing a column. That is honest about what the check covers: the offline -# suite runs on every push and pins the parser against real captures; this -# workflow is the only thing that can catch ETSY changing its markup, and it -# runs when someone asks it to. -# -# Put the schedule back the day a long-lived credential exists: -# schedule: -# - cron: "23 6 * * *" +# WEEKLY. How long a Scraping Browser endpoint stays valid has not been +# measured for this account, so the schedule is not built around a lifetime +# guess: if the secret stops authenticating, the run goes red and says so, +# and that is the signal to refresh it. on: workflow_dispatch: + schedule: + - cron: "23 6 * * 1" jobs: live-run: @@ -72,20 +59,18 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Explain a skipped run + - name: Fail when the live run cannot happen if: env.HAVE_CDP != 'true' run: | - echo "::notice::Skipping the live run: no ETSY_CDP_ENDPOINT secret is set." + echo "::error::The ETSY_CDP_ENDPOINT secret is not set, so this canary cannot reach etsy.com and has tested nothing." echo "Etsy's DataDome refuses datacentre addresses, and a GitHub runner" - echo "is one — a residential proxy alone was measured as not enough for" - echo "an automated browser either. A run from here would be red every" - echo "day whatever the state of the code. Set a Scraping Browser API" - echo "endpoint (with a pid reserved for CI) to turn this canary on:" + echo "is one; a residential proxy alone was measured as not enough for" + echo "an automated browser either. Set a Scraping Browser API endpoint" + echo "(with a pid reserved for CI):" echo echo " gh secret set ETSY_CDP_ENDPOINT --repo 2scraper/etsy-scraper" - echo - echo "The offline suite (tests.yml) covers everything that does not" - echo "need the network, and it runs on every push." + echo "### Canary did NOT test the site: ETSY_CDP_ENDPOINT is not set" >> "$GITHUB_STEP_SUMMARY" + exit 1 - uses: actions/setup-python@v5 if: env.HAVE_CDP == 'true' @@ -136,58 +121,34 @@ jobs: - name: Interpret the exit code id: verdict if: env.HAVE_CDP == 'true' - # ACCESS CONDITIONS ARE NOT DEFECTS, and on this site that - # distinction decides whether the badge means anything. - # - # An earlier version failed the job on ANY non-zero exit. That makes - # the canary red for two things that are nobody's bug: - # - # * the SECRET EXPIRED. A Scraping Browser credential is not - # forever; when it stops authenticating the run exits 5, and a - # daily cron then paints the badge red every day until someone - # refreshes it. A check that is always red teaches everyone to - # ignore checks, which is worse than having no check. - # * ACCESS IS INTERMITTENT PER PROFILE. Measured 2026-09-09/10 on - # one account: a fresh profile served three full pages in the - # morning, and that evening two fresh profiles were refused with - # t=bv through all their retries while a third served a shop - # front in full, twice. Same code, same zone, same URLs. - # - # So: 3 (blocked), 5 (remote API error) and 6 (partial) are reported - # as WARNINGS and do not fail the job. What stays red is a run that - # got in and found the wrong thing, or one that broke: - # - # 1 crashed - # 2 the workflow's own arguments are wrong - # 4 served a page and parsed ZERO rows from a search that returns - # tens of thousands — the tile anchor moved, which is exactly - # what this canary exists to catch - # - # A warning must not read as a pass, either. Each one writes to the - # step summary as well, so the run page says in words that the canary - # did not actually test anything this time. + # ANY RUN THAT DID NOT TEST THE SITE IS RED. Only exit 0 goes on to + # the data checks below. A block (3), a run that never obtained the + # page or whose parser read nothing from a served page (5), a partial + # run (6) and zero rows (4) all fail the job: each means this + # workflow's one purpose — showing that a live Etsy search still + # parses — was not achieved, whatever the reason. The message says + # which reason, so an access problem is not mistaken for a parser + # regression. run: | set -e code="${{ steps.run.outputs.exit_code }}" - tested=true + tested=false case "$code" in 0) + tested=true echo "OK — listings parsed from a live run." ;; 3) - tested=false - echo "::warning::Blocked before parsing (exit 3) — an ACCESS condition, not a code defect, so this run is not failed. Read the log for the DataDome 't' value: t=bv means the profile or its exit is banned and no captcha solve can help; t=fe means a solvable challenge appeared and no key was configured. Try a different --cdp-endpoint pid." - echo "### Canary did NOT test anything (exit 3, blocked)" >> "$GITHUB_STEP_SUMMARY" - echo "Etsy refused the request before any markup was parsed. The parser was not exercised. This is access, not a regression — but it also means the site-side check did not happen." >> "$GITHUB_STEP_SUMMARY" ;; + echo "::error::Blocked before parsing (exit 3): the site was not tested. Read the log for the DataDome 't' value: t=bv means the profile or its exit is refused and no solve helps; t=fe is a solvable challenge with no key configured. Try a different pid in ETSY_CDP_ENDPOINT." + echo "### Canary did NOT test the site (exit 3, blocked)" >> "$GITHUB_STEP_SUMMARY" + exit 1 ;; 5) - tested=false - echo "::warning::Remote API error (exit 5) — the Scraping Browser endpoint refused the connection, so nothing was tested and this run is not failed. If the log says 'profile_locked', another run still holds this pid: give CI its own. Otherwise the ETSY_CDP_ENDPOINT secret has most likely expired and needs refreshing." - echo "### Canary did NOT test anything (exit 5, endpoint refused)" >> "$GITHUB_STEP_SUMMARY" - echo "The most likely cause is an expired \`ETSY_CDP_ENDPOINT\` secret. Refresh it, or remove it to go back to the explicit SKIP path." >> "$GITHUB_STEP_SUMMARY" ;; + echo "::error::Nothing was obtained (exit 5). The log names the cause: the endpoint refused the connection (401: check or refresh ETSY_CDP_ENDPOINT; profile_locked: another run holds this pid), a navigation that never completed, or parser_found_nothing — a served page that states listings and parsed to none, which means the tile markup moved. See stop_reason in the log and the HTML dump." + echo "### Canary did NOT test the site (exit 5)" >> "$GITHUB_STEP_SUMMARY" + exit 1 ;; 6) - tested=false - echo "::warning::Partial run (exit 6) — some pages were gathered and the run then stopped early, usually a mid-run block. canary_run.meta.json names which pages failed. Not failed, because on this site that is an access condition; if it repeats on every run, suspect pagination instead." + echo "::error::Partial run (exit 6): some pages were gathered and the run then stopped early. canary_run.meta.json names which pages failed and why (stop_reason)." echo "### Canary tested only PART of the run (exit 6)" >> "$GITHUB_STEP_SUMMARY" - echo "See \`pages_failed\` in the sidecar artefact." >> "$GITHUB_STEP_SUMMARY" ;; + exit 1 ;; 4) echo "::error::Ran fine but parsed 0 rows (exit 4) on a search that returns tens of thousands of listings. The tile anchor (data-listing-id) has probably moved — this is the regression this canary exists to catch. The HTML dump in the artefacts tells you." exit 1 ;; diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 05aa489..b7ed00c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,82 +49,15 @@ jobs: pip install pytest pytest -q - - name: Every CLI answers --help + - name: Every CLI answers --help, and the committed sample is real + # One implementation, in .github/ci_checks.py, which the offline suite + # runs too. Inline copies of these two checks lived here and had + # already drifted (this step listed diff_runs.py, the script did not). # A crash in argparse setup or a bad local import is invisible to the - # test suite but breaks the first thing any user types. The engine - # libraries are deliberately NOT installed here, so an ImportError - # naming one of them is expected; anything else is a failure. - run: | - python - <<'EOF' - import subprocess, sys - - ENGINE_LIBS = ("playwright", "pyppeteer", "selenium", "webdriver_manager") - failed = [] - - for script in ["playwright_scraper.py", "puppeteer_scraper.py", - "selenium_scraper.py", "scraper_api_client.py", - "fingerprint_client.py", "env_config.py", - "diff_runs.py"]: - r = subprocess.run([sys.executable, script, "--help"], - capture_output=True, text=True) - if r.returncode == 0: - print(f"ok {script}") - continue - blob = r.stdout + r.stderr - if "ModuleNotFoundError" in blob and any(l in blob for l in ENGINE_LIBS): - print(f"skipped {script} (engine library not installed here)") - continue - print(f"FAILED {script}\n{blob}") - failed.append(script) - - sys.exit(1 if failed else 0) - EOF - - - name: Committed sample output is real - # The README links to these two files, so a push without them ships - # broken links. And a sample invented by hand drifts from the schema and - # teaches readers the wrong field names, so these are cut from a real - # run rather than written by hand. - run: | - python - <<'EOF' - import csv, json, sys - from pathlib import Path - - MARKERS = ("sample-product-", "example brand", "sample product", - "product description text", "lorem ipsum") - - fail = [] - for name in ("sample_output.json", "sample_output.csv"): - if not Path(name).is_file(): - fail.append(f"{name} is missing — regenerate it from a real run") - - if not fail: - rows = json.loads(Path("sample_output.json").read_text()) - if not rows: - fail.append("sample_output.json is empty") - blob = json.dumps(rows).lower() - hit = [m for m in MARKERS if m in blob] - if hit: - fail.append(f"sample_output.json looks fabricated: {hit}") - - # Column set must match what the writers actually produce. - from output_writer import Product - from dataclasses import asdict - expected = list(asdict(Product()).keys()) - for i, row in enumerate(rows): - if list(row.keys()) != expected: - fail.append(f"row {i} columns drifted from output_writer.Product") - break - - with open("sample_output.csv", newline="", encoding="utf-8") as f: - header = next(csv.reader(f)) - if header != expected: - fail.append("sample_output.csv header drifted from output_writer.Product") - - for line in fail: - print("FAILED:", line) - sys.exit(1 if fail else 0) - EOF + # test suite but breaks the first thing any user types; a sample + # invented by hand drifts from the schema. Engine libraries are not + # installed here, so an ImportError naming one is expected. + run: python3 .github/ci_checks.py --help-check --sample-check - name: Config reporter runs with no .env present run: python env_config.py @@ -151,12 +84,6 @@ jobs: # endpoint, a planted 32-hex key and a planted http proxy URL. run: python3 .github/ci_checks.py --secret-check - - name: The repo's own --help and sample checks agree with this workflow - # ci_checks.py implements all three checks. The two above are also - # done as workflow steps for the clearer failure output; running them - # here as well is what keeps the two from drifting apart again. - run: python3 .github/ci_checks.py --help-check --sample-check - engine-smoke: # The offline job above installs NO engine library on purpose (that is what # makes it offline), so every engine-guarded check in smoke_test.py skips diff --git a/.gitignore b/.gitignore index b58fa90..d900a0b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,13 @@ live_results.zip *_debug.html *_debug.png *.html.page* +# What `--dump-html X --pages N` writes for an X that does not end in .html +# (`out.page3`, `canary_dump.json.page3`), a one-page `--dump-html page.html` +# at the top level (the shape TROUBLESHOOTING.md suggests), and a scratch +# directory of live captures. Keyed on the KIND of file, not on one name. +*.page[0-9]* +/*.html +live/ # --- captures kept for parser work --------------------------------------- # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py); @@ -73,7 +80,7 @@ secrets* # recoverable only because that repo was private at the time. On 2026-09-17 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC # repository, one `git add -A` from publication. Craigslist was fixed then -# and the fix never reached anyone else, which is §16's rule unapplied: when -# you fix one, check the siblings. +# and the fix never reached anyone else. When you fix one, check the +# siblings. .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index d2efb92..3257b3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,25 @@ with it, so nobody discovers it from a bill or from a diff. ## [Unreleased] +> **Behaviour changes in this section, read these first:** +> +> * **A served listing page that states it holds listings and parses to +> none now ends the run `parser_found_nothing`** — exit 5 when nothing was +> gathered, exit 6 after earlier pages — instead of exit 4 ("empty +> catalogue"), or, on page 2 and later, a `complete` run with exit 0 that +> held page 1 only. +> * **An HTTP 404/410 for the URL you gave is now `not_found`, exit 5** +> (Playwright and pyppeteer; Selenium exposes no status). It used to be +> classified by its markup, and an error page linking to recommended +> listings could be parsed as results. A 404 past page 1 still ends the +> listing normally. +> * **New output files follow your umask** (`0o666 & ~umask`, what +> `open()` gives) instead of a fixed 0644; an existing file keeps its mode. +> Under umask 077 outputs are now 0600, under 002 they are 0664. +> * **The canary is red whenever it did not test the site** — blocked, +> nothing obtained, partial, or no `ETSY_CDP_ENDPOINT` secret — and it now +> runs weekly as well as on demand. +> > **Behaviour change:** `--retries 0` is now a usage error (exit 2). It used > to be accepted and made the page-load loop unreachable, so nothing was > fetched and the engine reported the browser's own blank page as a result: @@ -30,6 +49,41 @@ with it, so nobody discovers it from a bill or from a diff. ### Fixed +- **The pyppeteer and Selenium engines crashed on a block when no proxy + pool was configured** — the ordinary case over `--cdp-endpoint`. Etsy's + usual `t=bv` refusal reached `pool.current` / `pool.advance` on `None` and + the run ended with a traceback (exit 1) instead of a block (exit 3); the + pyppeteer engine did the same on a repeated load failure. Both now re-fetch + through the same access path without a relaunch, as the Playwright engine + already did, and rotate only when a pool exists. A pyppeteer timeout no + longer rotates exits either: like its twins it rotates only for an + unusable exit, a timeout having had its `--retries`. +- **Paid solves could exceed the per-page limit.** The reCAPTCHA check runs + on every fetch attempt and was not counted, so one page with no pool could + buy four solves against a limit of one. Every purchase, reCAPTCHA and + DataDome, in all three engines, now goes through one per-page allowance + checked before buying and never reset by a retry or a rotation. +- **New output files ignored the umask** (always 0644). See above. +- **The credential scan could pass over nothing.** Its directory exclusions + were tested against the ABSOLUTE path, so a clone under any directory + named `venv` or `.git` scanned 0 files and reported clean; it now judges + paths relative to the repository and fails a scan that read nothing. It + also reads `.json`, `.html`, `.csv`, `.toml`, `Dockerfile` and `.env.*` + backups now; a hit in a file git ignores is reported as a warning, a hit + anywhere else fails. +- **Raw dumps were not all git-ignored**: a top-level `page.html` (the shape + TROUBLESHOOTING suggests), `out.page3`-style multi-page dumps and a + `live/` directory are now ignored, with a check that shipped files are not. +- **The CI workflow carried inline copies of the `--help` and sample checks** + that had drifted from `.github/ci_checks.py`; it now runs the script, which + also checks `diff_runs.py --help`. +- **Garbled log text in the pyppeteer and Selenium block message** ("DataDome + under HTTP 403 under HTTP 403"), and a comment in `captcha_solver.py` that + described another site's hosts as Etsy's. +- **Public files cited private maintainer notes** by name and section + number; the rules are now stated in place, and a check fails if such a + citation comes back. + - **`--retries 0` fetched nothing and called it a result.** The load loop is `range(1, retries + 1)`, so 0 never runs it and the engine judged `about:blank`. Three third-party audits of this family ran exactly that and @@ -41,8 +95,8 @@ with it, so nobody discovers it from a bill or from a diff. - **Output files were written in place, so a crash mid-write truncated the previous good run.** `write_json`, `write_csv` and the `.meta.json` sidecar now write to a temporary file in the same directory, `fsync`, and - `os.replace` it. A new output is mode 0644 and a rewritten one keeps its - existing mode. + `os.replace` it. A new output follows the umask exactly as `open()` + would, and a rewritten one keeps its existing mode. - **HTML dumps (`--dump-html`, the `_debug.html` files) are now mode 0600.** They are what the site served this session and are what people paste into issues. @@ -86,7 +140,7 @@ with it, so nobody discovers it from a bill or from a diff. ### Added - **A check that binds every call into a shared module against the callee's - real signature** (§17's check #1), which this repo did not have. It catches + real signature**, which this repo did not have. It catches two things nothing else here can: a call whose arguments do not fit the signature, and a call to a name the shared module does not define at all — both of which reach a live run as a crash on the first fetch while import, @@ -101,9 +155,9 @@ with it, so nobody discovers it from a bill or from a diff. - **`fingerprint_client.py` could not read the key from `.env`.** `--key` defaulted to `os.environ.get("TWOCAPTCHA_KEY")` and only that, so a key put - in `.env` — exactly as §3, the README and `.env.example` instruct — worked + in `.env` — exactly as the README and `.env.example` instruct — worked for every engine and failed HERE with "No API key". A documented mechanism - not applied on one path, which is the shape of half the defects §16 lists. + not applied on one path. It now reads through `env_config.env_value`, calling `load_env()` itself because this is a standalone entry point that no engine has necessarily run @@ -115,7 +169,7 @@ with it, so nobody discovers it from a bill or from a diff. "still set to the placeholder from .env.example" instead. Found on a sibling repo's first live `--fingerprint` run, then checked - across the family before patching, per §16: five repos had it and one had + across the family before patching: five repos had it and one had already fixed it. Pinned by a check verified to fail on the old code — including that the help string does not interpolate its default, which is one substring away from printing a live credential to anyone who types @@ -195,7 +249,7 @@ all: it got in, and parsed zero rows. ### The second storefront, verified — and a guard the family's own one cannot provide -§15 of the family notes asks for a SECOND country site, and the first +The family's practice is to verify a SECOND country site, and the first Scraping Browser zone could not give one: three fresh `pid`s with `country-us` all returned German exits. A second zone honoured it, so the US storefront is now captured and pinned. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab9ceaa..640864f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,13 +87,10 @@ history needs a decision, not a red check on every push. Then the rest of the presentation, in the order that matters: -1. `python3 smoke_test.py` green, and the canary dispatched at least once — - including its SKIP branch, which is what runs when the - `ETSY_CDP_ENDPOINT` secret is absent. The canary has **no schedule**: a - Scraping Browser credential on this account does not survive a day, so it - runs on demand with a fresh secret rather than painting a badge that has - tested nothing. Put a schedule back the day a long-lived credential - exists. +1. `python3 smoke_test.py` green, and the canary dispatched at least once. + It runs weekly and on demand, through the `ETSY_CDP_ENDPOINT` secret, and + it is red whenever it did not test the site — including when that secret + is absent, which it names in the error. 2. The repo description, homepage and topics set (see the family notes on what those should say). 3. Only then the row in the org profile README — and check it with an diff --git a/README.md b/README.md index 24c559e..fd821f6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ blocking, JSON/CSV output and a run-metadata sidecar. [![release](https://img.shields.io/github/v/release/2scraper/etsy-scraper?sort=semver)](https://github.com/2scraper/etsy-scraper/releases) [![tests](https://github.com/2scraper/etsy-scraper/actions/workflows/tests.yml/badge.svg)](https://github.com/2scraper/etsy-scraper/actions/workflows/tests.yml) -[![canary (on demand)](https://github.com/2scraper/etsy-scraper/actions/workflows/canary.yml/badge.svg)](https://github.com/2scraper/etsy-scraper/actions/workflows/canary.yml) +[![canary](https://github.com/2scraper/etsy-scraper/actions/workflows/canary.yml/badge.svg)](https://github.com/2scraper/etsy-scraper/actions/workflows/canary.yml) ![Python](https://img.shields.io/badge/python-3.9%2B-blue?logo=python&logoColor=white) ![licence](https://img.shields.io/badge/licence-MIT-green) ![engines](https://img.shields.io/badge/engines-Playwright%20%7C%20Selenium%20%7C%20Puppeteer-blueviolet) @@ -288,8 +288,10 @@ never touched. Not measured on Etsy yet: how often this fires on real data. ### Exit codes `0` ok · `1` crash · `2` bad usage · `3` blocked · `4` zero products · -`5` remote API error (a locked Scraping Browser profile is the common one) · -`6` partial +`5` the content was never obtained (a refused or locked Scraping Browser +profile, a navigation that never completed, a 404 for the URL itself — +`stop_reason: not_found` — or a served page that states listings and parsed +to none — `stop_reason: parser_found_nothing`) · `6` partial **A run that finds nothing writes nothing** — last night's good output is not replaced with `[]`. Pass `--allow-empty` if an empty result is what you want @@ -433,39 +435,31 @@ All four are separate 2Captcha products behind one key. `tests.yml` is offline and green means green, on every push. -The **canary** is a real run against etsy.com, and it runs **on demand -rather than on a schedule** — dispatch it from the Actions tab with a fresh -`ETSY_CDP_ENDPOINT` secret when the answer matters. The reason is the -credential: a Scraping Browser endpoint on this account does not survive a -day, and a daily cron against it would give either a permanently red badge or -a permanently green one that had tested nothing. The second is worse, because -green reads as "the parser still works". +The **canary** is a real 3-page run against etsy.com, weekly and on demand. +Etsy's DataDome refuses a GitHub runner's datacentre address, so it runs +through the 2Captcha Scraping Browser API and needs an `ETSY_CDP_ENDPOINT` +repository secret (use a `pid` reserved for CI: a profile allows one live +connection). -When it does run, it has to distinguish two things that look the same from -the outside: +**Green means the site was reached and parsed, and nothing else does.** -| the run | the badge | why | -|---|---|---| -| got in, data is right | green | | -| got in, **parsed 0 rows** from a search that returns tens of thousands | **red** | the tile anchor moved. This is the regression the canary exists to catch | -| got in, a column collapsed below its floor | **red** | same reason | -| crashed, or the workflow's own arguments are wrong | **red** | | -| **blocked** before parsing (`t=bv`) | green, with a **warning** | access, not code — measured intermittent per profile | -| the **endpoint refused the connection** | green, with a **warning** | usually an expired secret | -| no `ETSY_CDP_ENDPOINT` secret at all | green, with a **notice** | nothing to run | - -The three warning rows also write to the run's step summary in words: *the -canary did not test anything this time.* That matters — a warning must not -read as a pass, and a green badge from a blocked run is not evidence the -parser still works. - -**Why not fail on a block?** Because it would be red most days for reasons -that are nobody's bug. Measured 2026-09-09/10 on one account: a fresh profile -served three full pages in the morning; that evening two fresh profiles were -refused through all their retries while a third served a shop front in full, -twice. Same code, same zone, same URLs. And a credential expires eventually, -which would pin the badge red until someone noticed. A check that is always -red teaches everyone to ignore checks. +| the run | the badge | +|---|---| +| got in, rows parsed, every data check holds | green | +| got in, **parsed 0 rows**, or a served page parsed to nothing (`parser_found_nothing`) | **red** — the tile markup moved | +| got in, a column collapsed below its floor | **red** | +| **blocked** before parsing (`t=bv`) | **red** — the site was not tested | +| the endpoint refused the connection, or nothing was obtained | **red** — the log names the cause | +| partial run | **red** | +| no `ETSY_CDP_ENDPOINT` secret at all | **red**, naming the missing secret | + +A block or a refused endpoint is an access condition rather than a parser +bug, and the error message says which it was. It is still red: a green badge +over a run that never reached the site would read as "the parser still +works" when nothing was tested. Access through one profile was measured +intermittent on 2026-09-09/10 (a fresh profile served three full pages in +the morning; that evening two were refused through all their retries), so a +single red run from a block is worth one re-dispatch before anything else. ## Comparing two runs diff --git a/captcha_solver.py b/captcha_solver.py index 95efbf2..828a66d 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -963,13 +963,13 @@ def _raise_for_datadome_error(payload: dict) -> None: # a JPEG of distorted text and a GET form). Roughly 190 lines of it, and none # of it is ported here, because Etsy has no such page. # -# What Etsy does instead is answer HTTP 403 with a DataDome shell, whose -# error page — no challenge, no form, nothing a solver can answer. Captured -# 2026-09-09 from a datacentre exit on five hosts (.de, .es, .nl, .pl and -# mediaworld.it): zero captcha markers of any kind, zero forms, zero images -# to read. The response to that page is a different exit address, not a -# solve, and product_parser.detect_page_state reports it as "blocked" rather -# than "captcha" precisely so no solve is attempted. +# What Etsy does instead is answer a refused request with HTTP 403 and a +# DataDome shell. When that shell says `t=bv` it carries no form and nothing +# a solver can answer — DataDome's own verdict is that the address or the +# browser is banned. The response to that page is a different exit address, +# not a solve, and product_parser.detect_page_state reports it as "blocked" +# rather than "captcha" precisely so no solve is attempted. The one solvable +# shape, `t=fe`, goes to solve_datadome above. # # The reCAPTCHA / hCaptcha / Turnstile machinery above IS kept, and that is a # deliberate asymmetry rather than an inconsistency. Detection stays broad @@ -977,4 +977,4 @@ def _raise_for_datadome_error(payload: dict) -> None: # what the address has been doing — a narrow list is how a challenge gets # reported as an empty page months later. A solver for a challenge this site # has never been observed to serve is dead code; a DETECTOR for one is cheap -# insurance. See the family note in CLAUDE.md. +# insurance. diff --git a/fingerprint_client.py b/fingerprint_client.py index e84ce1e..3210607 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -331,7 +331,7 @@ def main() -> int: # loader. It used to read `os.environ` alone, which meant a key put in # `.env` exactly as the README and .env.example instruct worked for every # engine and failed HERE with "No API key" — a documented mechanism not - # applied on one path, which is the shape of half the defects §16 lists. + # applied on one path. # # `.env` has to be LOADED before it can be read: `env_value` looks at # os.environ, and `load_env` is what fills that from the file. Calling it diff --git a/output_writer.py b/output_writer.py index 76ef4d8..b9a7c44 100644 --- a/output_writer.py +++ b/output_writer.py @@ -39,6 +39,7 @@ import json import os import tempfile +import threading from dataclasses import dataclass, asdict, field, fields from datetime import datetime, timezone from typing import Optional, List, Set, Sequence, Any, Type, Tuple @@ -281,6 +282,18 @@ def _csv_escape(v: Any) -> Tuple[Any, bool]: return v, False +_UMASK_LOCK = threading.Lock() + + +def _current_umask() -> int: + """The process umask. Python can only read it by setting it, so it is 0 + for an instant; the lock keeps two writers from interleaving that.""" + with _UMASK_LOCK: + umask = os.umask(0) + os.umask(umask) + return umask + + @contextlib.contextmanager def _atomic(path: str, newline: Optional[str] = None): """Write to a temporary file beside `path`, then rename over it. @@ -300,7 +313,11 @@ def _atomic(path: str, newline: Optional[str] = None): NamedTemporaryFile creates the file 0600 and a rename keeps that, so the output would come out unreadable to the group and to other users where - `open()` gave 0644. Restore the destination's own mode, or 0644. + `open()` honoured the umask. An existing destination keeps its own mode + (someone may have tightened it on purpose); a new one gets exactly what + `open(path, "w")` would have given it, `0o666 & ~umask`. A fixed 0644 + would be wrong both ways: wider than a 077 umask asks for, narrower + than a 002 one. """ directory = os.path.dirname(os.path.abspath(path)) or "." handle = tempfile.NamedTemporaryFile( @@ -314,7 +331,7 @@ def _atomic(path: str, newline: Optional[str] = None): try: mode = os.stat(path).st_mode & 0o777 except OSError: - mode = 0o644 + mode = 0o666 & ~_current_umask() os.chmod(handle.name, mode) os.replace(handle.name, path) except BaseException: diff --git a/page_flow.py b/page_flow.py index 7314115..12f1326 100644 --- a/page_flow.py +++ b/page_flow.py @@ -59,7 +59,8 @@ from product_parser import (TRACKING_PARAMS, datadome_verdict, detect_page_state, is_datadome_interstitial, - page_number_from_url, page_url, total_pages) + page_number_from_url, page_url, total_pages, + total_results) logger = logging.getLogger("page_flow") @@ -304,6 +305,12 @@ def classify(html: Optional[str], status: Optional[int] = None, # listing has none either. Retrying it would spend the user's budget on # getting the same right answer again. "empty": {"retry": False, "solve": False, "blocked": False}, + # The site answered 404/410 for this address. Final, and NOT blocked: + # exit 3 would send the reader looking for a proxy problem when the URL + # is simply wrong (a removed listing, a mistyped shop). Not parsed + # either: an error page can still link to recommended listings, and + # those are not the rows that were asked for. + "not_found": {"retry": False, "solve": False, "blocked": False}, } @@ -335,6 +342,61 @@ def counts_as_blocked(state: str) -> bool: SOLVES_PER_PAGE = 1 +class SolveBudget: + """The paid-solve allowance for ONE page, shared by every call site. + + An engine reaches the solver from two places per fetch attempt — the + reCAPTCHA check right after navigation, and the DataDome solve after the + page is classified — and both repeat on every block retry. A counter + kept beside only one of them made `SOLVES_PER_PAGE` advisory: with the + solver stubbed, one page with no proxy pool bought four reCAPTCHA solves + against a limit of one. So there is one object per page, created before + the retry loop, consulted immediately BEFORE each purchase and never + reset by a retry or a rotation: a fresh exit is a reason to re-fetch, + not a fresh allowance. + """ + + def __init__(self, limit: Optional[int] = None): + self.limit = SOLVES_PER_PAGE if limit is None else limit + self.spent = 0 + + def available(self) -> bool: + return self.spent < self.limit + + def spend(self) -> bool: + """Claim one purchase. False, and nothing claimed, when exhausted.""" + if not self.available(): + logger.info("Not buying another solve for this page: its " + "allowance of %d is spent.", self.limit) + return False + self.spent += 1 + return True + + +def parser_found_nothing(html: Optional[str], rows: int, page_num: int) -> bool: + """True when a SERVED listing page says it holds listings and none parsed. + + Zero rows from a page Etsy served means one of two opposite things: the + listing is genuinely empty (a taxonomy hub, a search with no results, + one page past the end), or the tile markup moved and the parser can no + longer read it. Reported as "0 products" the second sends the reader to + check the URL instead of the parser, and on page 2 or later it ended the + run as a COMPLETE one holding page 1 only. + + Etsy states its own arithmetic, which is what tells the two apart: the + listing's JSON-LD `numberOfItems` (any page that carries it) and a search + page's `initial_total_pages`. A page inside the site's own stated range + that yields nothing is the parser's failure, not the catalogue's. + """ + if rows or not html: + return False + stated_pages = total_pages(html) + if stated_pages and page_num <= stated_pages: + return True + stated_items = total_results(html) + return bool(page_num == 1 and stated_items and stated_items > 0) + + # The DataDome solve is OPT-IN on this site, and that is a measurement rather # than caution. Six frame-aware attempts on 2026-09-10, each from a fresh # residential exit: diff --git a/playwright_scraper.py b/playwright_scraper.py index c21249f..73cf394 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -151,10 +151,25 @@ class PageOutcome: # the length of a run to re-read six fields at the end would cost more # memory than the whole result set. shop_facts: Optional[dict] = None + # A page that was served and still produced no usable answer: + # "not_found" (the site said 404/410) or "parser_found_nothing" (the + # page states it holds listings and none parsed). Neither is blocked and + # neither is an empty catalogue, so each gets its own stop_reason. + failure: Optional[str] = None @property def ok(self) -> bool: - return not self.load_failed and self.blocked_by is None + return (not self.load_failed and self.blocked_by is None + and self.failure is None) + + @property + def stop_reason(self) -> Optional[str]: + """Why this page ended the run, or None if it did not.""" + if self.load_failed: + return "page_load_timeout" + if self.blocked_by is not None: + return f"blocked_{self.blocked_by}" + return self.failure ITEM_LINK_SELECTOR = SELECTORS["item_link"] @@ -588,7 +603,7 @@ def _content_when_settled(page, attempts: int = 4, pause_ms: int = 700): return None -def handle_datadome_if_present(page, args, html: str) -> bool: +def handle_datadome_if_present(page, args, html: str, budget) -> bool: """Buy a DataDome solve when — and only when — one can work. True if solved. The preconditions are checked in the order that costs least. A `t=bv` @@ -637,6 +652,8 @@ def handle_datadome_if_present(page, args, html: str) -> bool: # The browser's OWN user agent. The cookie is bound to it, so a # literal here would buy one the site rejects. user_agent=page.evaluate("navigator.userAgent")) + if not budget.spend(): + return False try: name, value = solve_datadome(challenge, args.twocaptcha_key, args.proxy, attempts=2) @@ -659,7 +676,7 @@ def handle_datadome_if_present(page, args, html: str) -> bool: return True -def handle_captcha_if_present(page, args) -> bool: +def handle_captcha_if_present(page, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Runs after EVERY navigation, for ANY page — not scoped to one URL. The @@ -713,6 +730,8 @@ def handle_captcha_if_present(page, args) -> bool: "solved — continuing with whatever the page already " "holds.") return False + if not budget.spend(): + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -779,20 +798,24 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome block_retries = 0 if not page_flow.RETRY_ON_BLOCKED else ( args.proxy_block_retries if has_pool else page_flow.BLOCK_RETRIES_WITHOUT_POOL) - # Counted across the whole block-retry loop, not per attempt: a page that - # keeps coming back as a challenge would otherwise buy one solve per - # rotation, which is how a run quietly turns into a bill. - solves_bought = 0 + # One allowance for the whole block-retry loop and for BOTH solve call + # sites: a page that keeps coming back as a challenge would otherwise buy + # one solve per attempt, which is how a run quietly turns into a bill. + budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False for block_attempt in range(block_retries + 1): logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) # Retry a navigation timeout rather than ending the run on it. One # network flap on page 12 of 50 should not break the loop. - load_failed, exit_failed = False, None + load_failed, exit_failed, status = False, None, None for attempt in range(1, args.retries + 1): try: - session.page.goto(url, wait_until="domcontentloaded", timeout=60000) + response = session.page.goto(url, wait_until="domcontentloaded", + timeout=60000) + # The status the site answered with, for the classifier: a + # 404 is "this address does not exist", which no retry fixes. + status = response.status if response is not None else None load_failed = False break except (PWTimeout, PWError) as e: @@ -824,21 +847,27 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if load_failed: break - if handle_captcha_if_present(session.page, args): + if handle_captcha_if_present(session.page, args, budget): # A solve navigated the page. Give the destination a moment # before judging what came back. session.page.wait_for_timeout(1000) + status = None # the status belonged to the page that was replaced html = _content_when_settled(session.page) or "" # A DataDome interstitial has not decided yet. Wait it out before # classifying: measured, one turns into a served page and another # into a solvable `t=fe` challenge, and reading the markup at # `domcontentloaded` sees neither. - html = page_flow.settle_datadome( + settled = page_flow.settle_datadome( lambda: _content_or_none(session.page), lambda ms: session.page.wait_for_timeout(ms), - frames=lambda: _frame_urls(session.page)) or html - state = _classify(session.page, html) + frames=lambda: _frame_urls(session.page)) + if settled and settled != html: + # The interstitial turned into another document; the status of + # the navigation describes the interstitial, not this. + status = None + html = settled or html + state = _classify(session.page, html, status) # The paid path, and the only state that reaches it. `should_solve` # is True for "captcha" and False for "blocked", which is where the @@ -848,10 +877,8 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Bounded by SOLVES_PER_PAGE: at most one purchase per page, because # the vendor's remedy for a failed solve is a different exit and the # rotation below already provides one. - if (page_flow.should_solve(state) - and solves_bought < page_flow.SOLVES_PER_PAGE): - solves_bought += 1 - if handle_datadome_if_present(session.page, args, html): + if page_flow.should_solve(state) and budget.available(): + if handle_datadome_if_present(session.page, args, html, budget): html = _content_or_none(session.page) or html state = _classify(session.page, html) # The VERIFIED outcome, and the only one worth reporting: a @@ -915,6 +942,23 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.state = state + if state == "not_found": + if page_num > 1: + # One page past the end of a listing answering 404 is the end of + # the listing, the same answer an empty page gives. + logger.info("Page %d answered %s — treating that as the end of " + "the listing.", page_num, status) + outcome.state = "empty" + outcome.final_url = session.page.url + return outcome + logger.error("Etsy answered HTTP %s for %s: that address does not " + "exist (a removed listing, a closed shop or a mistyped " + "URL). Not a block, so a proxy will not change it.", + status, url) + outcome.failure = "not_found" + outcome.final_url = session.page.url + return outcome + if state == "blocked": # Reported here rather than left to the generic challenge check # below, because what a caller needs to know is not "there is a @@ -1026,6 +1070,16 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome products = _parse_for_mode(html, session.page.url, args) logger.info("Parsed %d row(s) from page %d.", len(products), page_num) + if (state == "content" and args.mode in ("listing", "shop") + and page_flow.parser_found_nothing(html, len(products), page_num)): + logger.error("Page %d was served and states it holds listings, but " + "the parser read none from it. That is this scraper " + "failing to read the page, not an empty catalogue — " + "re-run with --dump-html and check the tile markup.", + page_num) + outcome.failure = "parser_found_nothing" + outcome.final_url = session.page.url + return outcome if args.mode == "shop" and page_num == 1: outcome.shop_facts = shop_metadata(html, session.page.url) @@ -1269,8 +1323,7 @@ def scrape(args) -> int: outcomes.append(first) if not first.ok: - stop_reason = ("page_load_timeout" if first.load_failed - else f"blocked_{first.blocked_by}") + stop_reason = first.stop_reason blocked = first.blocked_by is not None elif args.mode == "product": pass # one page is the whole run @@ -1301,8 +1354,7 @@ def scrape(args) -> int: failed = [o for o in rest if not o.ok] if failed: worst = min(failed, key=lambda o: o.page_num) - stop_reason = ("page_load_timeout" if worst.load_failed - else f"blocked_{worst.blocked_by}") + stop_reason = worst.stop_reason blocked = any(o.blocked_by for o in rest) elif exhausted: stop_reason = "no_new_products" @@ -1325,8 +1377,7 @@ def scrape(args) -> int: outcome = _fetch_one_page(session, args, pool, page_num, url) outcomes.append(outcome) if not outcome.ok: - stop_reason = ("page_load_timeout" if outcome.load_failed - else f"blocked_{outcome.blocked_by}") + stop_reason = outcome.stop_reason blocked = outcome.blocked_by is not None break diff --git a/product_parser.py b/product_parser.py index 9e3297c..ff3eadd 100644 --- a/product_parser.py +++ b/product_parser.py @@ -765,13 +765,15 @@ def datadome_captcha_url(html: str, frame_urls=None) -> Optional[str]: def detect_page_state(html: str, status: Optional[int] = None, url: Optional[str] = None, frame_urls=None) -> str: - """One of "blocked", "captcha", "empty", "content". + """One of "blocked", "captcha", "empty", "content", "not_found". `status` is optional because not every engine path can see it — a page fetched over CDP reports one, a page read back out of a `--dump-html` file does not. """ if not html: + if status in (404, 410): + return "not_found" return "blocked" if status and status >= 400 else "empty" verdict = datadome_verdict(html, frame_urls) @@ -787,6 +789,12 @@ def detect_page_state(html: str, status: Optional[int] = None, # `t=fe` challenge worth solving. return "captcha" if verdict == "fe" else "blocked" + if status in (404, 410): + # The site's own answer that this address does not exist. Checked + # before the content test, because an error page can still link to + # recommended listings and "/listing/" alone would call it content. + return "not_found" + if status == 403: return "blocked" diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 987f740..e4d3302 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -219,9 +219,23 @@ class PageOutcome: # memory than the whole result set. shop_facts: Optional[dict] = None + # "not_found" or "parser_found_nothing": served, but no usable answer. + # See playwright_scraper.PageOutcome. + failure: Optional[str] = None + @property def ok(self) -> bool: - return not self.load_failed and self.blocked_by is None + return (not self.load_failed and self.blocked_by is None + and self.failure is None) + + @property + def stop_reason(self) -> Optional[str]: + """Why this page ended the run, or None if it did not.""" + if self.load_failed: + return "page_load_timeout" + if self.blocked_by is not None: + return f"blocked_{self.blocked_by}" + return self.failure # Every `scheme://user:pass@` in a string, however many times it occurs. @@ -440,7 +454,7 @@ def _next_page_candidates(session, page_num: int) -> List[str]: return page_flow.next_page_candidates(page.url, hrefs or []) -def handle_datadome_if_present(session, args, html: str) -> bool: +def handle_datadome_if_present(session, args, html: str, budget) -> bool: """Buy a DataDome solve when — and only when — one can work. True if solved. Mirrors playwright_scraper.handle_datadome_if_present exactly: the @@ -479,6 +493,8 @@ def handle_datadome_if_present(session, args, html: str) -> bool: challenge = DataDomeChallenge( captcha_url=captcha_url, page_url=page.url, user_agent=bridge.run(page.evaluate("() => navigator.userAgent"))) + if not budget.spend(): + return False try: name, value = solve_datadome(challenge, args.twocaptcha_key, args.proxy, attempts=2) @@ -502,7 +518,7 @@ def handle_datadome_if_present(session, args, html: str) -> bool: return True -def handle_captcha_if_present(session, args) -> bool: +def handle_captcha_if_present(session, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Same two families, same order, same "detected is not blocking" rule as @@ -534,6 +550,8 @@ def handle_captcha_if_present(session, args) -> bool: if not args.twocaptcha_key: logger.warning("No 2captcha API key, so this challenge cannot be solved.") return False + if not budget.spend(): + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -572,19 +590,20 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome block_retries = 0 if not page_flow.RETRY_ON_BLOCKED else ( args.proxy_block_retries if has_pool else page_flow.BLOCK_RETRIES_WITHOUT_POOL) - # Counted across the whole block-retry loop, not per attempt: a page that - # keeps coming back as a challenge would otherwise buy one solve per - # rotation, which is how a run quietly turns into a bill. - solves_bought = 0 + # One allowance for the whole block-retry loop and for BOTH solve call + # sites — see page_flow.SolveBudget. + budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False for block_attempt in range(block_retries + 1): logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) - load_failed = False + load_failed, exit_failed, status = False, None, None for attempt in range(1, args.retries + 1): try: - bridge.run(page.goto(url, {"waitUntil": "domcontentloaded", - "timeout": 60000})) + response = bridge.run(page.goto( + url, {"waitUntil": "domcontentloaded", "timeout": 60000})) + # The status the site answered with, for the classifier. + status = response.status if response is not None else None load_failed = False break except Exception as e: # noqa: BLE001 — pyppeteer raises many types @@ -595,9 +614,8 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # responses, so they are told apart by that text. text = str(e) if any(marker in text for marker in _PROXY_ERROR_MARKERS): - logger.warning("Exit %s is unusable (%s).", - mask(pool.current) if pool else "(none)", text[:120]) - break + exit_failed = text[:120] + break # a different exit is the only thing that helps if attempt < args.retries: pause = args.retry_delay * (2 ** (attempt - 1)) logger.warning("Failed to load %s (attempt %d/%d: %s) — " @@ -605,8 +623,15 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome args.retries, text[:120], pause) time.sleep(pause) - if load_failed and block_attempt < block_retries: - pool.advance("unusable exit or repeated load failure") + # Rotate only when there is somewhere to rotate TO, and only for an + # unusable exit (a timeout already had its --retries). With no pool — + # the ordinary case over --cdp-endpoint — this used to call + # pool.advance on None and end the run with a traceback. + if exit_failed and has_pool and block_attempt < block_retries: + logger.warning("Exit %s is unusable (%s) — rotating to another " + "one (%d/%d).", mask(pool.current), exit_failed, + block_attempt + 1, block_retries) + pool.advance(f"unusable exit: {exit_failed}") session.relaunch() bridge, page = session.bridge, session.page d = _driver(session) @@ -615,26 +640,28 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome break - if handle_captcha_if_present(session, args): + if handle_captcha_if_present(session, args, budget): time.sleep(1) + status = None # the status belonged to the page that was replaced html = _content(session) or "" # A DataDome interstitial has not decided yet — wait it out before # classifying, exactly as the Playwright engine does. - html = page_flow.settle_datadome( + settled = page_flow.settle_datadome( lambda: _content(session), lambda ms: bridge.run(page.waitFor(ms)), - frames=lambda: _frame_urls(session)) or html - state = page_flow.classify(html, url=page.url, + frames=lambda: _frame_urls(session)) + if settled and settled != html: + status = None # it described the interstitial, not this document + html = settled or html + state = page_flow.classify(html, status=status, url=page.url, frame_urls=_frame_urls(session)) # The paid path, and the only state that reaches it. `should_solve` is # True for "captcha" and False for "blocked", which is where the # t=fe / t=bv distinction turns into a decision about money. - if (page_flow.should_solve(state) - and solves_bought < page_flow.SOLVES_PER_PAGE): - solves_bought += 1 - if handle_datadome_if_present(session, args, html): + if page_flow.should_solve(state) and budget.available(): + if handle_datadome_if_present(session, args, html, budget): html = _content(session) or html state = page_flow.classify(html, url=page.url, frame_urls=_frame_urls(session)) @@ -667,13 +694,26 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # so a different exit is the only thing that plausibly changes the # outcome. if block_attempt < block_retries: - logger.warning("Page %d came back as %s from %s — retrying from " - "another exit (%d/%d).", page_num, state, - mask(pool.current), block_attempt + 1, block_retries) - pool.advance(f"{state} on page {page_num}") - session.relaunch() - bridge, page = session.bridge, session.page - d = _driver(session) + if has_pool: + logger.warning("Page %d came back as %s from %s — retrying " + "from another exit (%d/%d).", page_num, state, + mask(pool.current), block_attempt + 1, + block_retries) + pool.advance(f"{state} on page {page_num}") + session.relaunch() + bridge, page = session.bridge, session.page + d = _driver(session) + else: + # No pool, so nowhere else to go — a plain re-fetch through + # the same access path, as the Playwright engine does. Not a + # relaunch: over --cdp-endpoint a profile allows one live + # connection, and reconnecting risks profile_locked. + pause = args.retry_delay * (block_attempt + 1) + logger.warning("Page %d came back as %s — re-fetching through " + "the same access path in %.1fs (%d/%d).", + page_num, state, pause, block_attempt + 1, + block_retries) + time.sleep(pause) if load_failed: logger.error("Gave up loading %s after %d attempt(s).", url, args.retries) @@ -682,15 +722,33 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.state = state + if state == "not_found": + if page_num > 1: + # One page past the end of a listing answering 404 is the end of + # the listing, the same answer an empty page gives. + logger.info("Page %d answered %s — treating that as the end of " + "the listing.", page_num, status) + outcome.state = "empty" + outcome.final_url = page.url + return outcome + logger.error("Etsy answered HTTP %s for %s: that address does not " + "exist (a removed listing, a closed shop or a mistyped " + "URL). Not a block, so a proxy will not change it.", + status, url) + outcome.failure = "not_found" + outcome.final_url = page.url + return outcome + if state == "blocked": - # DataDome's `t=bv` says the challenge is unsolvable, so the generic - # challenge check below to find — it is the shop's own error page - # under a 403. Saying so plainly, and saying what actually clears it, - # is more use than a captcha hint that does not apply. + # DataDome's `t=bv` says the challenge is unsolvable, so this is + # reported here rather than left to the generic challenge check + # below: there is nothing on the page worth paying for. Saying so + # plainly, and saying what actually clears it, is more use than a + # captcha hint that does not apply. debug_html = f"{args.out}_page{page_num}_debug.html" write_private_text(debug_html, html) - logger.error("Etsy refused this request (DataDome under HTTP 403 " - "under HTTP 403) — saved to %s. There is no challenge on " + logger.error("Etsy refused this request (DataDome, HTTP 403) — " + "saved to %s. There is no challenge on " "that page to solve, so a 2Captcha key does not help; a " "residential exit does. This is exit 3, distinct from a " "genuinely empty result (exit 4).", debug_html) @@ -757,6 +815,16 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome products = _parse_for_mode(html, page.url, args) logger.info("Parsed %d row(s) from page %d.", len(products), page_num) + if (state == "content" and args.mode in ("listing", "shop") + and page_flow.parser_found_nothing(html, len(products), page_num)): + logger.error("Page %d was served and states it holds listings, but " + "the parser read none from it. That is this scraper " + "failing to read the page, not an empty catalogue — " + "re-run with --dump-html and check the tile markup.", + page_num) + outcome.failure = "parser_found_nothing" + outcome.final_url = page.url + return outcome if args.mode == "shop" and page_num == 1: outcome.shop_facts = shop_metadata(html, page.url) @@ -864,8 +932,7 @@ def scrape(args) -> int: outcomes.append(first) if not first.ok: - stop_reason = ("page_load_timeout" if first.load_failed - else f"blocked_{first.blocked_by}") + stop_reason = first.stop_reason blocked = first.blocked_by is not None elif args.mode in ("listing", "shop"): seen_keys.update(p.sku for p in first.products if p.sku is not None) @@ -907,8 +974,7 @@ def scrape(args) -> int: outcome = _fetch_one_page(session, args, pool, page_num, url) outcomes.append(outcome) if not outcome.ok: - stop_reason = ("page_load_timeout" if outcome.load_failed - else f"blocked_{outcome.blocked_by}") + stop_reason = outcome.stop_reason blocked = outcome.blocked_by is not None break diff --git a/selenium_scraper.py b/selenium_scraper.py index bc029c3..cc9fd14 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -130,9 +130,23 @@ class PageOutcome: # memory than the whole result set. shop_facts: Optional[dict] = None + # "not_found" or "parser_found_nothing": served, but no usable answer. + # See playwright_scraper.PageOutcome. + failure: Optional[str] = None + @property def ok(self) -> bool: - return not self.load_failed and self.blocked_by is None + return (not self.load_failed and self.blocked_by is None + and self.failure is None) + + @property + def stop_reason(self) -> Optional[str]: + """Why this page ended the run, or None if it did not.""" + if self.load_failed: + return "page_load_timeout" + if self.blocked_by is not None: + return f"blocked_{self.blocked_by}" + return self.failure # Every `scheme://user:pass@` in a string, however many times it occurs. @@ -435,7 +449,7 @@ def _next_page_candidates(session, page_num: int) -> List[str]: hrefs or []) -def handle_datadome_if_present(session, args, html: str) -> bool: +def handle_datadome_if_present(session, args, html: str, budget) -> bool: """Buy a DataDome solve when — and only when — one can work. True if solved. Mirrors playwright_scraper.handle_datadome_if_present exactly: the @@ -476,6 +490,8 @@ def handle_datadome_if_present(session, args, html: str) -> bool: # A function BODY with an explicit return, not an arrow expression — # the dialect difference that keeps JavaScript out of page_flow. user_agent=driver.execute_script("return navigator.userAgent;")) + if not budget.spend(): + return False try: name, value = solve_datadome(challenge, args.twocaptcha_key, args.proxy, attempts=2) @@ -505,7 +521,7 @@ def handle_datadome_if_present(session, args, html: str) -> bool: return True -def handle_captcha_if_present(session, args) -> bool: +def handle_captcha_if_present(session, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Same detectors, same reconciliation and the same "detected is not @@ -543,6 +559,8 @@ def handle_captcha_if_present(session, args) -> bool: if not args.twocaptcha_key: logger.warning("No 2captcha API key, so this challenge cannot be solved.") return False + if not budget.spend(): + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -584,10 +602,9 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome block_retries = 0 if not page_flow.RETRY_ON_BLOCKED else ( args.proxy_block_retries if has_pool else page_flow.BLOCK_RETRIES_WITHOUT_POOL) - # Counted across the whole block-retry loop, not per attempt: a page that - # keeps coming back as a challenge would otherwise buy one solve per - # rotation, which is how a run quietly turns into a bill. - solves_bought = 0 + # One allowance for the whole block-retry loop and for BOTH solve call + # sites — see page_flow.SolveBudget. + budget = page_flow.SolveBudget() for block_attempt in range(block_retries + 1): logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) @@ -623,7 +640,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome break - if handle_captcha_if_present(session, args): + if handle_captcha_if_present(session, args, budget): time.sleep(1) html = d["content"]() or "" @@ -638,10 +655,8 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # The paid path, and the only state that reaches it. `should_solve` is # True for "captcha" and False for "blocked", which is where the # t=fe / t=bv distinction turns into a decision about money. - if (page_flow.should_solve(state) - and solves_bought < page_flow.SOLVES_PER_PAGE): - solves_bought += 1 - if handle_datadome_if_present(session, args, html): + if page_flow.should_solve(state) and budget.available(): + if handle_datadome_if_present(session, args, html, budget): html = d["content"]() or html state = page_flow.classify(html, url=d["current_url"](), frame_urls=_frame_urls(session)) @@ -674,12 +689,25 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # so a different exit is the only thing that plausibly changes the # outcome. if block_attempt < block_retries: - logger.warning("Page %d came back as %s from %s — retrying from " - "another exit (%d/%d).", page_num, state, - mask(pool.current), block_attempt + 1, block_retries) - pool.advance(f"{state} on page {page_num}") - session.relaunch() - d = _driver(session) + if has_pool: + logger.warning("Page %d came back as %s from %s — retrying " + "from another exit (%d/%d).", page_num, state, + mask(pool.current), block_attempt + 1, + block_retries) + pool.advance(f"{state} on page {page_num}") + session.relaunch() + d = _driver(session) + else: + # No pool, so nowhere else to go — a plain re-fetch through + # the same access path, as the Playwright engine does. Not a + # relaunch: over --cdp-endpoint a profile allows one live + # connection, and reconnecting risks profile_locked. + pause = args.retry_delay * (block_attempt + 1) + logger.warning("Page %d came back as %s — re-fetching through " + "the same access path in %.1fs (%d/%d).", + page_num, state, pause, block_attempt + 1, + block_retries) + time.sleep(pause) if load_failed: logger.error("Gave up loading %s after %d attempt(s).", url, args.retries) @@ -689,14 +717,15 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.state = state if state == "blocked": - # DataDome's `t=bv` says the challenge is unsolvable, so the generic - # challenge check below to find — it is the shop's own error page - # under a 403. Saying so plainly, and saying what actually clears it, - # is more use than a captcha hint that does not apply. + # DataDome's `t=bv` says the challenge is unsolvable, so this is + # reported here rather than left to the generic challenge check + # below: there is nothing on the page worth paying for. Saying so + # plainly, and saying what actually clears it, is more use than a + # captcha hint that does not apply. debug_html = f"{args.out}_page{page_num}_debug.html" write_private_text(debug_html, html) - logger.error("Etsy refused this request (DataDome under HTTP 403 " - "under HTTP 403) — saved to %s. There is no challenge on " + logger.error("Etsy refused this request (DataDome, HTTP 403) — " + "saved to %s. There is no challenge on " "that page to solve, so a 2Captcha key does not help; a " "residential exit does. This is exit 3, distinct from a " "genuinely empty result (exit 4).", debug_html) @@ -768,6 +797,16 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome final_url = d["current_url"]() or url products = _parse_for_mode(html, final_url, args) logger.info("Parsed %d row(s) from page %d.", len(products), page_num) + if (state == "content" and args.mode in ("listing", "shop") + and page_flow.parser_found_nothing(html, len(products), page_num)): + logger.error("Page %d was served and states it holds listings, but " + "the parser read none from it. That is this scraper " + "failing to read the page, not an empty catalogue — " + "re-run with --dump-html and check the tile markup.", + page_num) + outcome.failure = "parser_found_nothing" + outcome.final_url = d["current_url"]() + return outcome if args.mode == "shop" and page_num == 1: outcome.shop_facts = shop_metadata(html, d["current_url"]()) @@ -865,8 +904,7 @@ def scrape(args) -> int: outcomes.append(first) if not first.ok: - stop_reason = ("page_load_timeout" if first.load_failed - else f"blocked_{first.blocked_by}") + stop_reason = first.stop_reason blocked = first.blocked_by is not None elif args.mode in ("listing", "shop"): seen_keys.update(p.sku for p in first.products if p.sku is not None) @@ -905,8 +943,7 @@ def scrape(args) -> int: outcome = _fetch_one_page(session, args, pool, page_num, url) outcomes.append(outcome) if not outcome.ok: - stop_reason = ("page_load_timeout" if outcome.load_failed - else f"blocked_{outcome.blocked_by}") + stop_reason = outcome.stop_reason blocked = outcome.blocked_by is not None break diff --git a/smoke_test.py b/smoke_test.py index f865720..dc2f836 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -51,6 +51,7 @@ import tempfile import textwrap import threading +import time import builtins from contextlib import redirect_stdout from dataclasses import asdict as dataclasses_asdict, fields @@ -3307,10 +3308,13 @@ def test_datadome_policy_is_shared(): "should_pay_for(verdict, solve_mode)" in src) ok &= check("%s says why when it declines a solvable challenge" % name, "the solve is opt-in" in src) - ok &= check("%s caps purchases with SOLVES_PER_PAGE" % name, - "page_flow.SOLVES_PER_PAGE" in src) - ok &= check("%s counts purchases outside the retry loop" % name, - "solves_bought = 0" in src) + # The cap itself is asserted by BEHAVIOUR in + # test_engines_fetch_one_page_with_stubbed_drivers; this only pins + # that every engine holds one allowance object per page rather than + # a counter of its own beside one of the two call sites. + ok &= check("%s holds one SolveBudget per page" % name, + src.count("budget = page_flow.SolveBudget()") == 1 + and "solves_bought" not in src) ok &= check("%s takes the cookie domain from page_flow" % name, "page_flow.datadome_cookie_domain" in src) ok &= check("%s reads the browser's OWN user agent" % name, @@ -3397,7 +3401,7 @@ def default_exception_handler(self, context): def test_canary_separates_access_from_defect(): - group("the canary fails on defects and only WARNS on access conditions") + group("the canary is red unless it actually tested the site") ok = True wf_path = os.path.join(REPO_ROOT, ".github", "workflows", "canary.yml") wf = open(wf_path, encoding="utf-8").read() @@ -3419,20 +3423,13 @@ def test_canary_separates_access_from_defect(): script = "\n".join(line[10:] if line.startswith(" " * 10) else line for line in wf[start:end].splitlines()) - # WHY EACH CODE LANDS WHERE IT DOES: - # 0 got in and parsed -> pass, and the assertions then run - # 3 blocked before parsing -> ACCESS. Measured intermittent per - # profile on this site, so a daily red badge would be noise. - # 5 endpoint refused -> ACCESS, and the commonest cause is an - # EXPIRED SECRET. A credential is not forever; failing on it paints - # the badge red every day until someone notices. - # 6 partial -> ACCESS, usually a mid-run block. - # 1 crashed -> DEFECT. - # 2 bad arguments -> DEFECT (in the workflow itself). - # 4 served a page, ZERO rows -> DEFECT, and precisely the regression - # this canary exists to catch: the tile anchor moved. - expected = {0: "pass", 3: "warn", 5: "warn", 6: "warn", - 1: "fail", 2: "fail", 4: "fail", 99: "fail"} + # Only 0 — got in and parsed — passes. This used to let 3 (blocked), + # 5 (nothing obtained) and 6 (partial) through as WARNINGS on a green + # job, so the badge could stay green for weeks without the site being + # tested once. Each of those now fails, with a message naming why, so + # an access problem is still told apart from a parser regression. + expected = {0: "pass", 1: "fail", 2: "fail", 3: "fail", 4: "fail", + 5: "fail", 6: "fail", 99: "fail"} for code, want in sorted(expected.items()): body = script.replace('code="${{ steps.run.outputs.exit_code }}"', 'code="%d"' % code) @@ -3446,38 +3443,46 @@ def test_canary_separates_access_from_defect(): env=dict(os.environ, GITHUB_OUTPUT=out_file, GITHUB_STEP_SUMMARY=summary)) failed = done.returncode != 0 - warned = "::warning::" in done.stdout errored = "::error::" in done.stdout - wrote_summary = bool(open(summary, encoding="utf-8").read().strip()) tested = "tested=true" in open(out_file, encoding="utf-8").read() - if want == "pass": - got = not failed and not warned and not errored and tested - elif want == "warn": - # A warning must NOT read as a pass: it also has to say in the - # step summary that nothing was actually tested, and it must not - # claim `tested`. - got = not failed and warned and wrote_summary and not tested + got = not failed and not errored and tested else: - got = failed and errored + got = failed and errored and not tested ok &= check("exit %-2d is treated as %s" % (code, want), got) - ok &= check("the reason access is not a defect is written down", - "ACCESS CONDITIONS ARE NOT DEFECTS" in wf) - - # NO SCHEDULE, and the reason has to travel with the decision. A daily - # cron against a credential that does not survive a day gives either a - # permanently red badge or a permanently green one that tested nothing — - # and the green is worse, because it reads as "the parser still works". - # Restoring the cron is a legitimate change the day a long-lived - # credential exists; this check makes it a decision rather than a habit. - ok &= check("the canary has no cron schedule", - not re.search(r"^\s*-\s*cron:", wf, re.M)) + # No secret, no test — and that must be RED, naming the secret. Run the + # real step, as above. + try: + start = wf.index(" - name: Fail when the live run cannot happen") + start = wf.index(" run: |\n", start) + len(" run: |\n") + end = wf.index("\n\n", start) + except ValueError: + return check("the missing-secret step could be located", False) + missing = "\n".join(line[10:] for line in wf[start:end].splitlines()) + with tempfile.TemporaryDirectory() as td: + summary = os.path.join(td, "gh_summary") + done = subprocess.run(["bash", "-c", missing], capture_output=True, + text=True, env=dict(os.environ, + GITHUB_STEP_SUMMARY=summary)) + ok &= check("a missing ETSY_CDP_ENDPOINT fails the job and names it", + done.returncode != 0 and "::error::" in done.stdout + and "ETSY_CDP_ENDPOINT" in done.stdout) + ok &= check("that step runs exactly when the secret is absent", + "if: env.HAVE_CDP != 'true'" in wf) + ok &= check("no step in the canary emits a ::warning:: or ::notice:: " + "in place of a failure", + "::warning::" not in wf and "::notice::" not in wf) + + # Scheduled, so a site-side change is found without anyone remembering + # to look. No claim about how long an endpoint lives travels with it: a + # refused endpoint is red and says so. + ok &= check("the canary runs on a schedule", + bool(re.search(r"^\s*-\s*cron:", wf, re.M))) ok &= check("it is dispatchable by hand", "workflow_dispatch:" in wf) - ok &= check("and the reason the schedule is off is written down", - "does not survive a day" in wf) - ok &= check("the expired-secret case is named", - "expired" in wf.lower() and "refresh" in wf.lower()) + ok &= check("no unmeasured endpoint-lifetime claim", + "does not survive a day" not in wf + and "most likely expired" not in wf) return ok @@ -3501,6 +3506,15 @@ def test_ci_checks_is_actually_wired_up(): ok &= check("a workflow runs ci_checks.py", "ci_checks.py" in workflows) ok &= check("the secret check specifically is run", "--secret-check" in workflows or "--all" in workflows) + ok &= check("the help and sample checks are run from ci_checks.py", + "--help-check" in workflows and "--sample-check" in workflows) + # No workflow reimplements a check inline by importing this repo's own + # modules: nothing local runs a workflow, so such a copy drifts unseen. + inline = re.findall(r"^\s*(?:from|import)\s+(output_writer|product_parser|" + r"page_flow|playwright_scraper|puppeteer_scraper|" + r"selenium_scraper|diff_runs)\b", workflows, re.M) + ok &= check("no workflow imports a local module inline %s" % (inline or ""), + not inline) # AND IT PASSES ON THIS REPO. A check that is always red teaches everyone # to ignore checks; this one WAS red, on six documented placeholders. @@ -3548,6 +3562,51 @@ def test_ci_checks_is_actually_wired_up(): os.remove(planted) ok &= check("the probe file is cleaned up", not os.path.exists(planted)) + # Exclusions are relative to the REPOSITORY, and a scan of nothing is a + # failure. A copy of the tracked tree inside a directory called `venv` + # (and `.git`) used to scan 0 files and print "nothing credential-shaped". + import shutil + tracked = subprocess.run(["git", "ls-files"], cwd=REPO_ROOT, + capture_output=True, text=True).stdout.split() + if tracked: + with tempfile.TemporaryDirectory() as tmp: + clone = os.path.join(tmp, "venv", ".git", "etsy-scraper") + for f in tracked: + os.makedirs(os.path.dirname(os.path.join(clone, f)), exist_ok=True) + shutil.copy2(os.path.join(REPO_ROOT, f), os.path.join(clone, f)) + git_ok = subprocess.run(["git", "init", "-q", clone], + capture_output=True).returncode == 0 + ci = os.path.join(clone, ".github", "ci_checks.py") + + def scan(): + done = subprocess.run([sys.executable, ci, "--secret-check"], + cwd=clone, capture_output=True, text=True) + return done.returncode, done.stdout + done.stderr + + rc, out = scan() + m = re.search(r"(\d+) files scanned", out) + ok &= check("a clone under venv/.git/ ancestors still scans its " + "files (%s)" % (m.group(1) if m else out.strip()[-120:]), + rc == 0 and m is not None and int(m.group(1)) > 10) + key = "3f8a1c9e4b7d2065" + "af13ce88b409d752" + # A data file git would NOT ignore fails the scan... + with open(os.path.join(clone, "fixture.toml"), "w") as f: + f.write('key = "%s"\n' % key) + rc, out = scan() + ok &= check("a key in an unignored data file (.toml) fails the scan", + rc != 0 and "fixture.toml:1" in out) + os.remove(os.path.join(clone, "fixture.toml")) + # ...and a renamed .env / a dump that git ignores is still READ, + # and reported, without failing a run on a local key. + for name in (".env.bak", "scratch.json"): + with open(os.path.join(clone, name), "w") as f: + f.write("TWOCAPTCHA_KEY=%s\n" % key) + rc, out = scan() + ok &= check("an ignored .env.bak and scratch.json are scanned and " + "reported as warnings (git available: %s)" % git_ok, + ".env.bak:1" in out and "scratch.json:1" in out + and (rc == 0 if git_ok else rc != 0)) + # The pre-publication scan: the same rules over every blob that has EVER # existed. A later commit cannot remove what a published tag and a merged # PR's refs already hold, so this has to be runnable BEFORE the repo goes @@ -3667,6 +3726,60 @@ def test_no_capture_leaks(): "selenium_scraper.py") +PRIVATE_CITATION_RE = re.compile( + "|".join([re.escape("CLAUDE" + ".md"), re.escape("claude_" + "2scraper"), + re.escape("2scraper_" + "repo_rules"), "play" + "book", + chr(0xA7) + r"\s*\d"]), + re.IGNORECASE) + + +def check_no_private_citations(listed=None): + if listed is None: + listed = subprocess.run(["git", "ls-files"], cwd=REPO_ROOT, + capture_output=True, text=True) + if listed.returncode != 0 or not listed.stdout.strip(): + return True # not a git checkout: nothing to say about shipped files + files = [f for f in listed.stdout.split("\n") if f] + offenders, scanned = [], 0 + for f in files: + try: + text = open(os.path.join(REPO_ROOT, f), encoding="utf-8").read() + except (OSError, UnicodeDecodeError): + continue + scanned += 1 + for n, line in enumerate(text.splitlines(), 1): + if PRIVATE_CITATION_RE.search(line): + offenders.append("%s:%d" % (f, n)) + ok = check("the private-citation scan read the tracked files (%d)" % scanned, + scanned > 10) + ok &= check("no tracked file cites a private note or its sections %s" + % (offenders[:5] or ""), not offenders) + return ok + + +def test_gitignore_covers_dumps_and_keeps_shipped_files(): + group(".gitignore: dumps and captures ignored, shipped files kept") + ok = True + if subprocess.run(["git", "rev-parse"], cwd=REPO_ROOT, + capture_output=True).returncode != 0: + print(" (not a git checkout — nothing to ask git about)") + return True + + def ignored(path): + return subprocess.run(["git", "check-ignore", "-q", "--no-index", path], + cwd=REPO_ROOT).returncode == 0 + + for path in ("page.html", "out.page3", "canary_dump.json.page3", + "live/x.html", "x_debug.html", "etsy_products.json", + ".env", ".env.bak", ".env.local"): + ok &= check("%s is ignored" % path, ignored(path)) + for path in ("sample_output.json", "sample_output.csv", ".env.example", + ".github/workflows/tests.yml", ".github/ci_checks.py", + "tests/test_smoke.py", "smoke_test.py", "README.md"): + ok &= check("%s is NOT ignored" % path, not ignored(path)) + return ok + + def test_wording(): group("wording and removed flags") ok = True @@ -3698,6 +3811,13 @@ def test_wording(): offenders.append(f) ok &= check("no shipped file says %r" % phrase, not offenders) + # Private notes are not public references. A shipped file that points + # the reader at a maintainer's private instruction file, or at a numbered + # section of one, sends them somewhere they cannot follow. Every tracked + # file is scanned, this one included, so the tokens are assembled from + # pieces and the guard does not match itself. + ok &= check_no_private_citations(listed) + for flag in REMOVED_ENGINE_FLAGS: offenders = [] for f in ENGINE_FILES: @@ -3819,10 +3939,9 @@ def test_fingerprint_client_reads_env(): # THE DEFECT THIS PINS, found the first time --fingerprint was run live # here and confirmed present in five sibling repos: `--key` defaulted to # `os.environ.get("TWOCAPTCHA_KEY")` alone. So a key put in `.env` — - # which is exactly what §3, the README and .env.example instruct — worked + # which is exactly what the README and .env.example instruct — worked # for every engine and failed HERE with "No API key". A documented - # mechanism not applied on one path, which is the shape of half the - # defects §16 lists. + # mechanism not applied on one path. src = inspect.getsource(fpc.main) ok &= check("it loads .env itself, rather than hoping an engine did", "env_config.load_env()" in src) @@ -4137,7 +4256,7 @@ def reached(module, seen=None): def test_shared_calls_bind_against_the_real_signature(): - """§17's check #1: bind every call into a shared module against the + """Bind every call into a shared module against the callee's real signature. This exists because of a defect that no other check in this suite can @@ -4295,8 +4414,31 @@ def boom(*a, **k): ow.write_json(rows, fresh) ok &= check("an existing output keeps its mode across a rewrite", stat.S_IMODE(os.stat(jp).st_mode) == 0o640) - ok &= check("a new output is 0644, not the temp file's 0600", - stat.S_IMODE(os.stat(fresh).st_mode) == 0o644) + # A new file follows the umask exactly as open() would. 022 is the + # value every candidate rule agrees on, so 002 and 077 are the cases + # that tell a correct rule from a flat 0644. + for umask, want in ((0o022, 0o644), (0o002, 0o664), (0o077, 0o600)): + old_umask = os.umask(umask) + try: + got = {} + for label, path, call in ( + ("json", os.path.join(d, "m%o.json" % umask), + lambda p: ow.write_json(rows, p)), + ("csv", os.path.join(d, "m%o.csv" % umask), + lambda p: ow.write_csv(rows, p)), + ("sidecar", os.path.join(d, "m%o" % umask), + lambda p: ow.write_run_meta(p, {}))): + call(path) + real = path + ".meta.json" if label == "sidecar" else path + got[label] = stat.S_IMODE(os.stat(real).st_mode) + finally: + os.umask(old_umask) + ok &= check("umask %03o: new json/csv/sidecar are %03o (got %s)" + % (umask, want, {k: "%03o" % v for k, v in got.items()}), + set(got.values()) == {want}) + ok &= check("a new output is not the temp file's 0600 under umask 022", + stat.S_IMODE(os.stat(fresh).st_mode) == 0o644 + or os.umask(os.umask(0)) != 0o022) # CSV formulas: strings only, after the list join, JSON untouched. evil = Product(sku="2", url="u", title="=HYPERLINK(\"x\")", price=-5.0, @@ -4526,6 +4668,298 @@ def test_retries_is_validated_in_every_entry_point(): return check("--retries is validated in every entry point, so 0 is a usage error", not problems) + +# --------------------------------------------------------------------------- +# The three engines' page loop, driven with the DRIVER stubbed out +# --------------------------------------------------------------------------- +def _load_engine_with_stub_driver(name): + """Import `.py` under a private module name with fake driver modules. + + Deliberately not `import playwright_scraper`: that skips wherever the + driver is absent, which is the core-only venv and CI's offline job — so a + check written that way would never run in the two places that matter + most. The engine is read off disk and its driver imports are satisfied + by stand-ins; nothing here can reach a browser or the network. + """ + import importlib.util + import types + from unittest import mock + + def mod(modname, **attrs): + m = types.ModuleType(modname) + m.__path__ = [] + for k, v in attrs.items(): + setattr(m, k, v) + return m + + class _Err(Exception): + pass + + fakes = { + "playwright": mod("playwright"), + "playwright.sync_api": mod("playwright.sync_api", + sync_playwright=mock.MagicMock(), + Error=type("Error", (_Err,), {}), + TimeoutError=type("TimeoutError", (_Err,), {})), + "pyppeteer": mod("pyppeteer", launch=mock.MagicMock(), + connect=mock.MagicMock()), + "selenium": mod("selenium", webdriver=mock.MagicMock()), + "selenium.common": mod("selenium.common"), + "selenium.common.exceptions": mod( + "selenium.common.exceptions", + TimeoutException=type("TimeoutException", (_Err,), {}), + WebDriverException=type("WebDriverException", (_Err,), {}), + JavascriptException=type("JavascriptException", (_Err,), {})), + "selenium.webdriver": mod("selenium.webdriver"), + "selenium.webdriver.chrome": mod("selenium.webdriver.chrome"), + "selenium.webdriver.chrome.options": mod( + "selenium.webdriver.chrome.options", Options=mock.MagicMock()), + "selenium.webdriver.common": mod("selenium.webdriver.common"), + "selenium.webdriver.common.by": mod( + "selenium.webdriver.common.by", By=mock.MagicMock()), + "selenium.webdriver.support": mod("selenium.webdriver.support"), + "selenium.webdriver.support.ui": mod( + "selenium.webdriver.support.ui", WebDriverWait=mock.MagicMock()), + } + spec = importlib.util.spec_from_file_location( + "_stubbed_" + name, os.path.join(REPO_ROOT, name + ".py")) + engine = importlib.util.module_from_spec(spec) + with mock.patch.dict(sys.modules, fakes): + spec.loader.exec_module(engine) + return engine + + +class _StubResponse: + def __init__(self, status): + self.status = status + + +class _StubPage: + """Answers every driver call the three page loops make. `html` is what + each read returns, `status` what every navigation answers, `raises` an + exception every navigation throws instead.""" + + def __init__(self, html, status=200, raises=None): + self.html, self.status, self.raises = html, status, raises + self.url = "https://www.etsy.com/search?q=handmade+mug" + self.gotos = 0 + self.context = self + + # navigation, all three dialects + def goto(self, *a, **k): + self.gotos += 1 + if self.raises is not None: + raise self.raises + return _StubResponse(self.status) + + def get(self, url): # selenium + self.goto() + + def content(self): + return self.html + + @property + def page_source(self): + return self.html + + @property + def current_url(self): + return self.url + + def __getattr__(self, attr): + # Everything else (waits, evaluate, reload, screenshot, cookies, + # frames, querySelectorAll...) is a no-op returning an empty list, + # which reads as "no frames" and "no tiles". + return lambda *a, **k: [] + + +class _StubSession: + def __init__(self, page): + self.page = self.driver = page + self.bridge = self # pyppeteer: bridge.run(coro) -> value + self.relaunches = 0 + + def run(self, value): + return value + + def relaunch(self): + self.relaunches += 1 + + +def _drive_fetch_one_page(engine, name, page, solve_mode="when-blocked", + key=None, page_num=1, pool=None): + """Run `engine._fetch_one_page` once over `page` and report what it did.""" + import types + from unittest import mock + + session = _StubSession(page) + bought = {"recaptcha": 0, "datadome": 0} + + def fake_driver(sess): + return {"count": lambda sel: 0, "sleep": lambda ms: None, + "content": lambda: sess.page.html, + "current_url": lambda: sess.page.url} + + def solve_recaptcha(*a, **k): + bought["recaptcha"] += 1 + return "stub-token" + + def solve_datadome(*a, **k): + bought["datadome"] += 1 + return ("datadome", "stub-cookie") + + challenge = types.SimpleNamespace(kind="recaptcha-v2", source="stub", + sitekey="stub-sitekey", action=None) + patches = dict( + solve_recaptcha=solve_recaptcha, solve_datadome=solve_datadome, + detect_recaptcha_v3=lambda *a, **k: None, + detect_recaptcha_in_page=lambda *a, **k: None, + reconcile_detections=lambda *a, **k: challenge, + write_private_text=lambda *a, **k: None, + _frame_urls=lambda *a, **k: [], + time=types.SimpleNamespace(sleep=lambda s: None, time=time.time, + monotonic=time.monotonic)) + if name == "playwright_scraper": + patches.update(_content_when_settled=lambda p, *a, **k: p.html, + _content_or_none=lambda p: p.html) + elif name == "puppeteer_scraper": + patches.update(_driver=fake_driver, _content=lambda sess: sess.page.html) + else: + patches.update(_driver=fake_driver) + + argv = ["x", "--url", page.url, "--retries", "1", "--retry-delay", "0", + "--solve-captcha", solve_mode, "--out", "/nonexistent/etsy_stub"] + # No credential from the caller's environment reaches these args; the + # solver functions are stubbed regardless, so nothing is ever sent. + env = {k: v for k, v in os.environ.items() + if k != "TWOCAPTCHA_KEY" and not k.startswith("ETSY_")} + with mock.patch.object(sys, "argv", argv), \ + mock.patch.dict(os.environ, env, clear=True): + args = engine.parse_args() + args.twocaptcha_key = key + args.dump_html = None + error = None + outcome = None + import logging as _logging + _logging.disable(_logging.CRITICAL) # the loop's own log is not the subject + with mock.patch.multiple(engine, **patches): + try: + outcome = engine._fetch_one_page(session, args, pool, page_num, + page.url) + except Exception as e: # noqa: BLE001 — the crash IS the finding + error = "%s: %s" % (type(e).__name__, e) + finally: + _logging.disable(_logging.NOTSET) + return {"outcome": outcome, "error": error, "bought": bought, + "gotos": page.gotos, "relaunches": session.relaunches} + + +def test_engines_fetch_one_page_with_stubbed_drivers(): + group("engines: one page loop each, driver stubbed (no pool, budget, " + "404, parser failure)") + import page_flow as _pf + ok = True + attempts = _pf.BLOCK_RETRIES_WITHOUT_POOL + 1 + served_but_unreadable = ( + '' + '' + '
see /listing/ pages
') + not_found_page = ( + '' + '

Sorry, this page does not exist

' + 'mug' + '') + for name in ENGINES: + engine = _load_engine_with_stub_driver(name) + proxy_error = getattr(engine, { + "playwright_scraper": "PWError", + "puppeteer_scraper": "Exception", + "selenium_scraper": "WebDriverException", + }[name], Exception)("net::ERR_PROXY_CONNECTION_FAILED") + + # 1. The commonest Etsy outcome with no proxy pool — the ordinary + # case over --cdp-endpoint — must be a block (exit 3), not a crash. + r = _drive_fetch_one_page(engine, name, _StubPage(DD_BANNED_FIXTURE, 403)) + ok &= check("%s: t=bv with no pool does not raise (%s)" + % (name, r["error"]), r["error"] is None) + ok &= check("%s: t=bv with no pool is reported as datadome-bv" % name, + r["outcome"] is not None + and r["outcome"].blocked_by == "datadome-bv") + ok &= check("%s: t=bv with no pool re-fetches %d times and never " + "relaunches (one connection per CDP profile)" + % (name, attempts), + r["gotos"] == attempts and r["relaunches"] == 0) + + # 2. A dead exit with no pool: nothing to rotate to, so a load + # failure — not an AttributeError on pool.advance. + r = _drive_fetch_one_page(engine, name, _StubPage( + DD_BANNED_FIXTURE, raises=proxy_error)) + ok &= check("%s: a dead exit with no pool does not raise (%s)" + % (name, r["error"]), r["error"] is None) + ok &= check("%s: a dead exit with no pool is a load failure" % name, + r["outcome"] is not None and r["outcome"].load_failed + and r["outcome"].stop_reason == "page_load_timeout") + + # 3. The paid budget covers BOTH call sites over every retry. A page + # that stays a solvable challenge, with a reCAPTCHA reported on it + # too and --solve-captcha always: at most SOLVES_PER_PAGE purchases. + r = _drive_fetch_one_page(engine, name, + _StubPage(DD_SOLVABLE_FIXTURE, 403), + solve_mode="always", key="stub-key") + total = r["bought"]["recaptcha"] + r["bought"]["datadome"] + ok &= check("%s: %d attempt(s) bought %d solve(s) %s, limit %d" + % (name, r["gotos"], total, r["bought"], + _pf.SOLVES_PER_PAGE), + r["error"] is None and r["gotos"] == attempts + and 1 <= total <= _pf.SOLVES_PER_PAGE) + + # 4. A served page that states 201,606 listings and parses to none + # is the parser's failure, not an empty catalogue. + # Page 2 needs the site's own page count: `numberOfItems` alone does + # not say whether page 2 exists, and a page past the end is empty. + for page_num, html in ((1, served_but_unreadable), + (2, served_but_unreadable.replace( + "", + '' + ""))): + r = _drive_fetch_one_page(engine, name, _StubPage(html, 200), + page_num=page_num) + ok &= check("%s: page %d stating listings, none parsed -> " + "parser_found_nothing (%s)" + % (name, page_num, r["error"]), + r["outcome"] is not None + and r["outcome"].stop_reason == "parser_found_nothing" + and not r["outcome"].ok) + + # 5. HTTP 404 reaches the classifier where the driver reports one. + # Selenium cannot see a status at all, so it is exempt. + if name != "selenium_scraper": + r = _drive_fetch_one_page(engine, name, _StubPage(not_found_page, 404)) + ok &= check("%s: a 404 is not_found, one fetch, no rows (%s)" + % (name, r["error"]), + r["outcome"] is not None + and r["outcome"].stop_reason == "not_found" + and not r["outcome"].products and r["gotos"] == 1) + r = _drive_fetch_one_page(engine, name, _StubPage(not_found_page, 404), + page_num=3) + ok &= check("%s: a 404 past page 1 is the end of the listing" % name, + r["outcome"] is not None and r["outcome"].ok + and r["outcome"].state == "empty") + # And the run-level mapping: neither failure is "complete" or exit 4. + for reason, rows, want in (("parser_found_nothing", [], 5), + ("not_found", [], 5)): + with tempfile.TemporaryDirectory() as d: + code = finish_run(rows, os.path.join(d, "o"), "json", False, + blocked=False, stop_reason=reason, + pages_requested=1, pages_completed=0, + pages_failed=[1], mode="listing", + source="etsy.com", start_url="u", final_url="u") + ok &= check("finish_run: %s with no rows is exit %d (got %s)" + % (reason, want, code), code == want) + return ok + + def main() -> int: ok = True # Checks that could not run because an optional engine library is absent. @@ -4572,6 +5006,8 @@ def main() -> int: ok &= test_dockerfile_copies_what_it_runs() ok &= test_shared_calls_bind_against_the_real_signature() ok &= test_outputs_are_written_atomically_and_csv_is_formula_safe() + ok &= test_engines_fetch_one_page_with_stubbed_drivers() + ok &= test_gitignore_covers_dumps_and_keeps_shipped_files() ok &= test_sample_output() ok &= test_x_debug_header_is_redacted() ok &= test_scraper_api_sends_waitfor_as_an_object_and_reads_http_code()