From 06ceab64ea12a06874a002ae5e941f20ed9a9764 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:59:57 +0000 Subject: [PATCH] pyppeteer survives a failed load without a proxy; a late challenge can be solved - pyppeteer: a page that never loads with no --proxy ends as a load failure, not AttributeError/exit 1; rotation only for an unusable exit with a pool. - Solve allowance is claimed immediately before the paid request, so a detection that buys nothing no longer uses up the page's one solve. - Credential scan: skip list tested on the repo-relative path, zero files scanned is a failure, .env* backups/.json/.csv are read. - --pages/--concurrency >= 1, --delay/--retry-delay/--proxy-block-retries >= 0, refused by name in every engine. - .gitignore covers NAME.pageN dumps, root *.html and working dirs. - .env.example ships KOSE_URL commented out. - Canary: the Scraping Browser job fails with an error naming the missing secret instead of passing on a skip; exit 5 over CDP is red. - Comment fixes; no shipped file cites private notes, with a suite guard. Co-Authored-By: Claude Opus 5.5 --- .env.example | 4 +- .github/ci_checks.py | 36 ++- .github/workflows/canary.yml | 55 +++-- .github/workflows/tests.yml | 2 +- .gitignore | 23 +- CHANGELOG.md | 51 ++++- CONTRIBUTING.md | 10 +- README.md | 3 +- captcha_solver.py | 6 +- diff_runs.py | 8 +- env_config.py | 2 +- fingerprint_client.py | 2 +- http_scraper.py | 26 +-- output_writer.py | 54 ++--- page_flow.py | 77 +++++-- playwright_scraper.py | 125 ++++++----- product_parser.py | 102 +++++---- puppeteer_scraper.py | 119 +++++----- pyproject.toml | 2 +- scraper_api_client.py | 4 +- selenium_scraper.py | 108 +++++---- smoke_test.py | 423 +++++++++++++++++++++++++++++------ 22 files changed, 841 insertions(+), 401 deletions(-) diff --git a/.env.example b/.env.example index 5d1412f..33152c7 100644 --- a/.env.example +++ b/.env.example @@ -74,4 +74,6 @@ KOSE_CDP_ENDPOINT=ws://{login}-zone-scraping_browser-country-jp-pid-{profileId}: KOSE_PROXY=http://{user}:{password}@jp.proxy.2captcha.com:2334 # A default URL, so a run needs no --url. An explicit flag still wins. -KOSE_URL=https://maison.kose.co.jp/site/cosmedecorte/c/c15/ +# Shipped commented out: a copied .env should not decide what a run fetches, +# so --tags and --category keep working until you choose a default here. +# KOSE_URL=https://maison.kose.co.jp/site/cosmedecorte/c/c15/ diff --git a/.github/ci_checks.py b/.github/ci_checks.py index c383e1b..897fd1f 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -86,14 +86,26 @@ HEX32 = re.compile(r"\b[0-9a-f]{32}\b") # Contexts in which a 32-hex string is plainly not a key. Site-specific # entries belong here only once a committed fixture actually carries one — -# CLAUDE.md §16: a family-wide entry copied without checking is how dead +# A family-wide entry copied without checking is how dead # code spreads. Kept as a CONTEXT allowlist rather than by loosening the # pattern: a bare 32-hex string anywhere else still fails, which is the # point of the check. HEX32_ALLOWED = ("sha", "hash", "nonce", "example", "md5", "digest", "checksum") -SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example") +SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example", + ".json", ".csv", ".toml", ".cfg", ".ini") + + +def _is_scanned(path): + """A file of a scanned type, or any `.env*` file whatever its suffix. + + A working copy of `.env` renamed the way people rename them — `.env.bak`, + `.env.local`, `.env.save` — is untracked, ignored by git, and exactly + where a live key sits. Reading it here is what lets a local run of this + check say so before anything is committed. + """ + return path.suffix in SCANNED_SUFFIXES or path.name.startswith(".env") # Directory names that are never this repo's own source. @@ -112,7 +124,7 @@ def _is_inside_a_virtualenv(path): `.v` to fail, on a 32-hex string inside pip's vendored `packaging/_elffile.py`. - That is CLAUDE.md §21's failure exactly — a check that fails on a + That is the failure exactly — a check that fails on a legitimate tree is a check nobody can read, and it fails for everyone but the person who chose the blessed name. """ @@ -121,14 +133,18 @@ def _is_inside_a_virtualenv(path): break if (parent / "pyvenv.cfg").exists(): return True - return "site-packages" in path.parts + return "site-packages" in path.relative_to(REPO).parts def scanned_files(): for path in sorted(REPO.rglob("*")): - if not path.is_file() or path.suffix not in SCANNED_SUFFIXES: + if not path.is_file() or not _is_scanned(path): continue - if any(part in _SKIP_NAMES for part in path.parts): + # RELATIVE to the repository. `path.parts` is absolute, so a clone + # that merely lives under a directory called `build` or `dist` + # skipped every file and reported "0 files scanned, nothing + # credential-shaped" — a clean bill for a scan that read nothing. + if any(part in _SKIP_NAMES for part in path.relative_to(REPO).parts): continue if _is_inside_a_virtualenv(path): continue @@ -194,9 +210,9 @@ def sample_check(): failed.append("sample_output.csv header differs from output_writer.Product") if not failed: - discounted = sum(1 for r in rows if r.get("original_price")) + priced = sum(1 for r in rows if r.get("price") is not None) print(f"ok {len(rows)} rows, {len(expected)} columns, " - f"{discounted} discounted, schema matches") + f"{priced} priced, schema matches") return failed @@ -220,6 +236,10 @@ def secret_check(): failed.append(f"{rel}:{lineno} contains {match[:6]}… — a " f"32-char hex string, the shape of a 2captcha key") + if not scanned: + # Zero files is never a clean result: it means the scan did not run. + failed.append(f"no files were scanned under {REPO} — the scan read " + f"nothing, so it cannot vouch for anything") if not failed: print(f"ok {scanned} files scanned, nothing credential-shaped") return failed diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index c0b9d9c..a704078 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -10,16 +10,15 @@ # all, with no challenge of any kind. A GitHub runner is a datacentre address # too, so this workflow re-measures that claim every morning. # -# CLAUDE.md §11 says a canary that CANNOT pass must skip rather than fail, -# because a check that is always red teaches everyone to ignore checks. The -# other half of that rule applies here: **a canary that CAN pass without a -# credential must never be gated on one.** If Kosé ever puts its catalogue -# behind a key or a bot manager, this badge goes red the next morning and the -# README's claim is retested without anyone remembering to. +# **A canary that CAN pass without a credential must never be gated on one.** +# If Kosé ever puts its catalogue behind a key or a bot manager, this badge +# goes red the next morning and the README's claim is retested without anyone +# remembering to. # -# The last job is the exception and it is guarded: it exercises the PAID path -# (the Scraping Browser API over CDP), which cannot run without a secret, so -# it skips with a ::notice:: when none is set. +# The last job exercises the PAID path (the Scraping Browser API over CDP), +# which cannot run without a secret. It does not pass by skipping: with no +# secret it FAILS with an ::error:: naming the secret, because a green check +# that tested nothing reads exactly like one that tested something. # # gh secret set KOSE_CDP_ENDPOINT --repo 2scraper/kose-scraper # @@ -40,8 +39,8 @@ permissions: env: # Cosme Decorté's catalogue: 13 pages of 24 on 2026-09-18, so three pages - # really is three pages. §11: with one page pagination is never exercised, - # which is exactly how a silent single-page bug survived in a sibling repo. + # really is three pages. With one page pagination is never exercised, which + # is exactly how a silent single-page bug survived in a sibling repo. CANARY_URL: "https://maison.kose.co.jp/site/cosmedecorte/c/c15/" CANARY_PRODUCT: "https://maison.kose.co.jp/site/onebykose/g/gMUSC/" @@ -105,7 +104,7 @@ jobs: if not ok: failures.append(f"{label}: {detail}") - # §20: fail on a parser failure BY NAME. "0 products" would send + # Fail on a parser failure BY NAME. "0 products" would send # whoever reads this red badge to check the URL, when the thing that # moved is the parser. This stop_reason means the site served a page # that links to products and we turned it into nothing. @@ -147,7 +146,7 @@ jobs: want(f"{column} coverage", share >= 95.0, f"{share:.0f}% ({filled}/{len(rows)})") - # Values, not just coverage (§10). A column can be 100% populated + # Values, not just coverage. A column can be 100% populated # and entirely wrong. want("every currency is JPY", {r.get("currency") for r in rows if r.get("price") is not None} == {"JPY"}, @@ -164,7 +163,7 @@ jobs: all(r["sku"] and f"/g/g{r['sku']}/" in r["url"] for r in rows), "a row points at a product other than the one it describes") - # §18: page + position must be unique across a multi-page run, or + # page + position must be unique across a multi-page run, or # `position` is worthless — it restarts at 1 on every page. pairs = [(r.get("page"), r.get("position")) for r in rows] want("page+position unique", len(set(pairs)) == len(pairs), @@ -377,7 +376,7 @@ jobs: if-no-files-found: ignore scraping_browser: - name: Scraping Browser API (skips without a secret) + name: Scraping Browser API (needs the KOSE_CDP_ENDPOINT secret) runs-on: ubuntu-latest timeout-minutes: 20 env: @@ -385,17 +384,15 @@ jobs: # here and read in each step's `if`. HAVE_CDP: ${{ secrets.KOSE_CDP_ENDPOINT != '' }} steps: - - name: Explain the skip + - name: Require the secret if: env.HAVE_CDP != 'true' run: | - echo "::notice::Skipping the Scraping Browser run: no KOSE_CDP_ENDPOINT secret is set." - echo "This path is OPTIONAL on this site — the listing job above runs" - echo "the same scrape from a bare runner with no credentials at all," - echo "which is the README's central claim. To exercise the paid path:" + echo "::error::The KOSE_CDP_ENDPOINT secret is not set, so the Scraping Browser path was NOT tested." + echo "The listing jobs above test the site itself with no credentials." + echo "This job tests the paid path and fails rather than passing on a" + echo "skip. To make it run:" echo " gh secret set KOSE_CDP_ENDPOINT --repo 2scraper/kose-scraper" - echo "Note a Scraping Browser profile's credentials live about a day," - echo "so a stored secret goes stale between scheduled runs — prefer" - echo "dispatching this workflow manually when you want to test it." + exit 1 - uses: actions/checkout@v4 if: env.HAVE_CDP == 'true' @@ -415,7 +412,7 @@ jobs: env: # Through the ENVIRONMENT, never a command line: env_config.py reads # exactly this variable, and a credential in argv is readable by - # anything that can run `ps` (CLAUDE.md §3, §11). + # anything that can run `ps`. KOSE_CDP_ENDPOINT: ${{ secrets.KOSE_CDP_ENDPOINT }} run: | set +e @@ -432,11 +429,11 @@ jobs: import json, os, sys code = int(os.environ["EXIT_CODE"]) if code == 5: - print("::warning::Remote API error (exit 5). A Scraping Browser " - "profile allows ONE live connection and its credentials " - "live about a day, so a stale secret or a busy pid looks " - "exactly like this. Not a failure of the scraper.") - sys.exit(0) + print("::error::Exit 5: the content was never obtained over CDP. " + "A Scraping Browser profile allows ONE live connection, so " + "a busy pid looks like this; an expired profile answers " + "401. Either way nothing was tested.") + sys.exit(1) if code != 0: print(f"::error::The CDP run exited {code}.") sys.exit(1) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0240e82..02c6450 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,7 +51,7 @@ jobs: - name: Every CLI answers --help, and the sample output is real # BOTH of these were once INLINE reimplementations of checks that - # `.github/ci_checks.py` already implements — which is CLAUDE.md §17's + # `.github/ci_checks.py` already implements — which is the # "two sources of truth, one dead and one holed", and it bit on this # repo's very first CI run: the inline sample check still imported # `output_writer.Business`, a class this repo renamed, so the job died diff --git a/.gitignore b/.gitignore index f6c01d7..99b7aab 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,23 @@ live_results.zip *_debug.html *_debug.png *.html.page* +# `--dump-html NAME --pages N` writes NAME.pageN whatever NAME ends in, and a +# one-page dump named `dump.html` lands at the root. +*.page[0-9]* +/*.html + +# --- working directories by KIND, not by one chosen name ----------------- +live/ +out/ +output/ +run/ +runs/ +result/ +results/ +tmp/ +scratch/ +dump/ +dumps/ # --- captures kept for parser work --------------------------------------- # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py); @@ -49,7 +66,7 @@ build/ # --- environments -------------------------------------------------------- # `.venv*` rather than `.venv/`, because the README tells you to make ONE -# VENV PER ENGINE (§6: the three engines' pins are mutually unsatisfiable), +# VENV PER ENGINE (the three engines' pins are mutually unsatisfiable), # so `.venv-playwright/` and `.venv-selenium/` are the normal names here and # the bare `.venv/` pattern misses every one of them. .venv* @@ -77,7 +94,7 @@ secrets* # recoverable only because that repo was private at the time. On 2026-09-17 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC # repository, one `git add -A` from publication. Craigslist was fixed then -# and the fix never reached anyone else, which is §16's rule unapplied: when -# you fix one, check the siblings. +# and the fix never reached anyone else: when you fix one, check the +# siblings. .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 1186d6f..33010bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,55 @@ as a CLI toolkit can. A **patch** release means fixes; it does not promise that every flag and default is frozen. Where a patch changes behaviour an existing user would notice, the release notes lead with it. +## [Unreleased] + +> **Newly refused flag values.** `--pages 0`, `--concurrency 0` and any +> negative `--pages`, `--concurrency`, `--delay`, `--retry-delay` or +> `--proxy-block-retries` are now usage errors (exit 2) naming the flag, in +> every engine and in `scraper_api_client.py`. They used to be accepted: a +> zero page count fetched nothing and reported an empty catalogue, and a +> negative pause crashed the run after page 1. +> +> **pyppeteer, a page that never loads, no `--proxy`:** the run now ends as a +> load failure (exit 5, or 6 with rows) instead of a traceback and exit 1. +> With a pool, a repeated timeout no longer rotates the exit; only an +> unusable exit does, as in the other engines. + +### Fixed + +- **pyppeteer crashed with `AttributeError` when a page failed to load and + no proxy was configured.** The load-failure branch called + `pool.advance(...)` on a `None` pool. It now rotates only for an unusable + exit, and only when the pool has another exit to rotate to. +- **The per-page solve allowance was spent by a detection that bought + nothing.** The pre-classification captcha check used up the page's one + solve even when it found no challenge, so a challenge first seen after + classification was never offered to the solver and the page was reported + as blocked. The allowance is now claimed immediately before the paid + request, in all three browser engines. A purchase that fails still counts, + so the cap still holds at one per page across retries and rotations. +- **The credential scan read nothing from a clone under a directory named + `build` or `dist`.** The skip list was tested against the absolute path. It + now tests the path relative to the repository, a scan that reads zero + files fails, and `.env*` backups, `.json` and `.csv` files are read too. +- `.gitignore` now covers every `--dump-html NAME --pages N` output + (`NAME.pageN`), a root-level `*.html` dump and the usual working + directories (`live/`, `out/`, `tmp/`, ...). +- `.env.example` ships `KOSE_URL` commented out, so a copied `.env` does not + decide what a run fetches. +- The canary's Scraping Browser job no longer passes by skipping: with no + `KOSE_CDP_ENDPOINT` secret it fails with an error naming the secret, and an + exit 5 over CDP is a failure rather than a warning. +- Comments that named columns this schema does not have, stated a category + total as current, carried another vendor's wording or gave the wrong exit + code for `parser_found_nothing` (it is 5) were corrected; the CI sample + summary counts priced rows instead of a column that does not exist. +- Shipped files, including the README, `--help` output and the 0.1.1 notes + below, no longer cite private notes by name or section number; a suite + check keeps it that way. +- `smoke_test.py -v` no longer crashes on a check whose detail is not a + string. + ## [0.2.0] — 2026-10-13 Worked through a third-party audit. Its headline conclusion was wrong and @@ -190,7 +239,7 @@ as if they were this one. ## [0.1.1] — 2026-09-18 -A pass back over CLAUDE.md before calling the repo finished. Everything here +A pass back over the project's own rules before calling the repo finished. Everything here was found by checking a claim rather than by reading the code. ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 654417a..08c33fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,9 +96,11 @@ Then the rest of the presentation, in the order that matters: 1. `python3 smoke_test.py` green, and the canary dispatched at least once. **Both of this repo's real canary jobs run with NO secrets** and are expected to be green — that is not a convenience, it is what keeps the - README's central claim honest. Only the third job, which exercises the - Scraping Browser path, skips without a secret; dispatch it by hand once - and confirm the SKIP branch runs, not just the happy one. + README's central claim honest. Only the last job, which exercises the + Scraping Browser path, needs a secret (`KOSE_CDP_ENDPOINT`); without it + that job FAILS with an error naming the secret rather than passing on a + skip, because a green check that tested nothing is indistinguishable + from one that tested something. Note `workflow_dispatch` requires the workflow to exist on the DEFAULT branch — from a feature branch `gh workflow run` answers `HTTP 404: workflow canary.yml not found on the default branch`, which @@ -143,7 +145,7 @@ that breaks one will fail rather than silently regress: (種類を選ぶ, カートに入れる, 商品を購入する, 予約受付中) and two mean not (現在購入頂けません, 予約受付終了). A wording Kosé adds tomorrow must read as unknown rather than as a quiet "in stock". The column is verified in BOTH - directions — 19 of 171 tiles were unavailable — which §20 says is otherwise + directions — 19 of 171 tiles were unavailable — which is otherwise not verified at all. - **`tax_rate` is a column because Japan has two rates.** 10% standard, 8% diff --git a/README.md b/README.md index 521ff9f..176c8d6 100644 --- a/README.md +++ b/README.md @@ -339,8 +339,7 @@ into 264. Every fixture in the suite pins the value, not the coverage. ## What has and has not been verified live -Honest about the gap, because §13 of this family's playbook says a claim is -measured or absent: +Honest about the gap, because a claim here is either measured or absent: | path | status, 2026-10-13 | |---|---| diff --git a/captcha_solver.py b/captcha_solver.py index e57ecb3..e805dc8 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -614,8 +614,8 @@ def solve_recaptcha(challenge: CaptchaChallenge, twocaptcha_api_key: Optional[st api_version: str = "v2", min_score: float = 0.7) -> str: """Public entry point: solve `challenge` through 2captcha and return the token. - There used to be a `use_antidetect` branch here, behind a CLI flag of the - same name, that POSTed to a hardcoded local "solve-captcha" endpoint. + There used to be a second branch here, behind a CLI flag of its own, + that POSTed to a hardcoded local "solve-captcha" endpoint. It was removed before publishing: that endpoint was a placeholder for a product that does not exist under that name, so the flag could not work for anyone who set it, and the real "the browser solves it for you" path is @@ -715,4 +715,4 @@ class CaptchaUnsolvable(RuntimeError): # what the address has been doing — a narrow list is how a challenge gets # reported as an empty page months later. A solver for a challenge this site # has never been observed to serve is dead code; a DETECTOR for one is cheap -# insurance. See the family note in CLAUDE.md. +# insurance. diff --git a/diff_runs.py b/diff_runs.py index 8658912..fb987b7 100644 --- a/diff_runs.py +++ b/diff_runs.py @@ -17,7 +17,7 @@ the two rows were read a different way — a listing row and a `--mode product` row populate different columns — so the difference is about our two snapshots rather than about - the product, and `--fail-on-change` ignores it (§8) + the product, and `--fail-on-change` ignores it **Three things make two runs incomparable, and all three are refused rather than diffed.** Each would otherwise produce a diff whose every line is an @@ -35,7 +35,7 @@ different facets are two selections, not two observations. **Two guards the sibling repos have and this one deliberately does not**, and -the absences are measurements rather than omissions (CLAUDE.md §17: a policy +the absences are measurements rather than omissions (a policy nothing can enforce is the same defect as dead code): * no `sort` guard. Maison KOSÉ offers no ordering control on either @@ -94,7 +94,7 @@ # a product page. Diffing a listing run against a product run would report # each of these as a change on every row, and none of it would be about the # product. When the two rows disagree on `mode`, these are reported as -# `source_changed` rather than as changes — this family's rule (§8: a +# `source_changed` rather than as changes — this family's rule (a # difference that comes with a provenance difference says something about our # own two snapshots, not about the site). PROFILE_ONLY_FIELDS = ( @@ -275,7 +275,7 @@ def _check_comparable(args) -> bool: # control on either listing route — and no market either: it is one # Japanese storefront, so `source` is "maison.kose.co.jp" on both sides # and `currency` is JPY on every row. Neither guard would ever fire, and - # a guard that cannot fire is the dead code CLAUDE.md §17 is about. + # a guard that cannot fire is dead code that looks load-bearing. # # What DOES decide which products are in a file here is the ROUTE. A # category listing (/c/cNN/) holds one brand's catalogue in the site's diff --git a/env_config.py b/env_config.py index 98500aa..a2262e7 100644 --- a/env_config.py +++ b/env_config.py @@ -66,7 +66,7 @@ # A literal-only check reported both of those as CONFIGURED, so `cp # .env.example .env` followed by a run connected to cb.2captcha.com with the # string `{login}-zone-…` as its username and got a 401 — a confusing failure -# a long way from its cause, which is precisely what §3's rule exists to +# a long way from its cause, which is precisely what the rule exists to # prevent. Any `{…}` left in a value means the line was never filled in. _BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}") diff --git a/fingerprint_client.py b/fingerprint_client.py index e84ce1e..5dfb6c3 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -331,7 +331,7 @@ def main() -> int: # loader. It used to read `os.environ` alone, which meant a key put in # `.env` exactly as the README and .env.example instruct worked for every # engine and failed HERE with "No API key" — a documented mechanism not - # applied on one path, which is the shape of half the defects §16 lists. + # applied on one path, which is the shape of half the defects found in copied code. # # `.env` has to be LOADED before it can be read: `env_value` looks at # os.environ, and `load_env` is what fills that from the file. Calling it diff --git a/http_scraper.py b/http_scraper.py index 84373d1..84e4cb8 100644 --- a/http_scraper.py +++ b/http_scraper.py @@ -38,7 +38,7 @@ So the flags this engine cannot honour are REFUSED as usage errors rather than accepted and silently ignored: a setting that looks configurable and is -not is worse than one that is absent (§3). +not is worse than one that is absent. Usage: @@ -78,7 +78,7 @@ # An honest identity rather than a disguise. Measured above: this site serves # a project-named UA exactly as it serves Chrome, so there is nothing to gain -# by claiming to be a browser — and CLAUDE.md §24 records a sibling site that +# by claiming to be a browser — and the family's notes record a sibling site that # refuses precisely the CONTRADICTION between a claimed UA and the TLS # fingerprint underneath it. Saying what we are costs nothing here and cannot # backfire that way. @@ -124,7 +124,7 @@ def _mask(text: str) -> str: connection error, so a proxy URL with a password in it reaches the log the moment anything goes wrong. A masker that handles the first occurrence prints the password the other times and looks like it is - working (§8). Host and port are kept: which exit a run used is the point + working. Host and port are kept: which exit a run used is the point of the log and is not the secret. """ return _CREDENTIALS_RE.sub(r"\1***:***@", text or "") @@ -181,7 +181,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome except (requests.RequestException, ValueError) as e: # A dead or unauthenticated proxy is NOT a timeout, and the # two want opposite responses: a timeout deserves another try - # at the same exit, a dead proxy a different one (§8). + # at the same exit, a dead proxy a different one. kind = page_flow.classify_transport_error(e) load_failed = True if kind == "proxy": @@ -240,7 +240,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if args.dump_html: # On success too, not only on failure: a run can return the right # count with a column silently unpopulated, and then the exact bytes - # are the only way to tell a parsing bug from a bad response (§9). + # are the only way to tell a parsing bug from a bad response. path = (args.dump_html if args.pages == 1 else f"{args.dump_html}.page{page_num}") with open(path, "w", encoding="utf-8") as f: @@ -370,7 +370,7 @@ def scrape(args) -> int: fresh = [p for p in outcome.products if p.sku is not None and p.sku not in seen_keys] if not fresh: - # The data-based stop (§7 layer 3), and on this site it is + # The data-based stop, and on this site it is # not optional: past its last page a category listing serves # THAT PAGE AGAIN, byte-identical and HTTP 200, so a run that # waited for an empty page would never stop. @@ -380,7 +380,7 @@ def scrape(args) -> int: break seen_keys.update(p.sku for p in fresh) - # Merged in PAGE order, never arrival order (§8). + # Merged in PAGE order, never arrival order. all_rows: List = [] merged_seen: Set[str] = set() for oc in sorted(outcomes, key=lambda o: o.page_num): @@ -449,29 +449,29 @@ def parse_args(): p.add_argument("--category", default=None, help="A category id to build a URL from ('c15', 'c21'), or " "a label to tag output rows with.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.at_least_one, default=1, help="How many pages to read, starting at the one --url " "names (default 1).") - p.add_argument("--delay", type=float, default=1.0, + p.add_argument("--delay", type=page_flow.non_negative_seconds, default=1.0, help="Seconds between pages (default 1.0). Lower than the " "browser engines' default because a request here is " "one GET rather than a page load; on an ungated site " "politeness is the only reason for it.") p.add_argument("--retries", type=page_flow.attempts, default=3, help=page_flow.RETRIES_HELP) - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", type=page_flow.non_negative_seconds, default=2.0, help="Seconds before the first retry, doubling thereafter.") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="kose_products", help="Output file prefix") p.add_argument("--proxy", default=None, help="One proxy URL. Credentials ride in the URL and are " - "masked in every log line (§8).") + "masked in every log line.") p.add_argument("--proxy-file", default=None, help="A file of proxy URLs, one per line.") p.add_argument("--proxy-rotate", choices=list(ROTATE_MODES), default="per-run") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2) + p.add_argument("--proxy-block-retries", type=page_flow.non_negative_int, default=2) p.add_argument("--allow-empty", action="store_true", help="Write an empty result instead of leaving the " "previous good output in place.") @@ -489,7 +489,7 @@ def parse_args(): p.add_argument("--cdp-endpoint", default=None, help=argparse.SUPPRESS) p.add_argument("--solve-captcha", default=None, help=argparse.SUPPRESS) p.add_argument("--fingerprint", action="store_true", help=argparse.SUPPRESS) - p.add_argument("--concurrency", type=int, default=1, help=argparse.SUPPRESS) + p.add_argument("--concurrency", type=page_flow.at_least_one, default=1, help=argparse.SUPPRESS) args = p.parse_args() # What the USER typed, before the environment fills anything in. # diff --git a/output_writer.py b/output_writer.py index 4bbe898..a2384fa 100644 --- a/output_writer.py +++ b/output_writer.py @@ -26,7 +26,7 @@ Columns this site does NOT have, and the measurement behind each ---------------------------------------------------------------- -CLAUDE.md §9: a column null on every row of every run should not exist, and +A column null on every row of every run should not exist, and removing one needs the measurement written down so someone can put it back with a better one. Counted 2026-09-18 over 171 tiles in 8 listing captures and 4 product captures: @@ -35,7 +35,7 @@ is the load-bearing one.** 360 price nodes were surveyed and not one carries a struck-through, was-, or reference price: every node is a single amount in `1,234円（税込）` shape. So there is no discount chain, and - CLAUDE.md §4 is explicit that the tile-price overlay must be DELETED rather + the rule is explicit that the tile-price overlay must be DELETED rather than ported when that is true — there is no second view to reconcile against, and an overlay would be dead code that looks load-bearing. The EU Omnibus 30-day-low column that mediamarkt-scraper needs has no counterpart @@ -55,12 +55,12 @@ no ordering control on either listing route, so there is no ordering to record and nothing a run could have asked for. That is the opposite of bbb-scraper, where the ordering decides WHICH rows are in the file and - therefore had to be a column (CLAUDE.md §21). + therefore had to be a column. What IS kept, byte-identical and in order, is the family prefix — `source`, `scraped_at`, `url`, `sku`, `title` — so one column name works across the family and a consumer reading several of these repos reads the same first -five columns in the same order (§9). +five columns in the same order. """ import contextlib @@ -98,7 +98,7 @@ def utc_now() -> str: class Product: """One Maison KOSÉ product, from a listing tile or from a product page.""" - # ---- the family prefix, byte-identical and in order (§9) ------------ + # ---- the family prefix, byte-identical and in order ------------ source: str = SOURCE_DEFAULT scraped_at: str = field(default_factory=utc_now) # The absolute product URL. A tile publishes it ROOT-RELATIVE @@ -128,16 +128,16 @@ class Product: # Tax-INCLUSIVE, which is what this site prints and what its JSON-LD # publishes; `tax_rate` below says which rate is included. Structured and # displayed prices were checked against each other rather than assumed - # (§4) and agree on 4 of 4 product captures. + # and agree on 4 of 4 product captures. price: Optional[float] = None # "JPY" where a price was read, None where none was. Never defaulted: a - # row with no price has no currency either (§8). + # row with no price has no currency either. currency: Optional[str] = None # From the site's own call-to-action text on a listing and from # `offers.availability` on a product page, both as ALLOWLISTS so an - # unanticipated value reads as unknown rather than as a quiet True (§20). + # unanticipated value reads as unknown rather than as a quiet True. # - # Verified in BOTH directions, which §20 says is otherwise not verified at + # Verified in BOTH directions, which is otherwise not verified at # all: 19 of 171 measured tiles say 現在購入頂けません or 予約受付終了, and # `gJQBA`'s JSON-LD independently says `OutOfStock` where its tile does. in_stock: Optional[bool] = None @@ -149,7 +149,7 @@ class Product: category: Optional[str] = None # ---- provenance and ordering --------------------------------------- - # WHICH node the price was read from, per §8. There is no discount chain + # WHICH node the price was read from. There is no discount chain # on this site and therefore no DOM overlay to reconcile against, so this # records the source rather than a confirmation: # @@ -160,15 +160,15 @@ class Product: price_source: Optional[str] = None # `page` is 1-based over the run; `position` is the item's rank WITHIN its # page and restarts at 1 on each one. Neither is unique alone — the suite - # asserts the PAIR is unique across a multi-page run (§18). + # asserts the PAIR is unique across a multi-page run. page: Optional[int] = None position: Optional[int] = None # Which mode produced the row. The repo no longer implies it and # `diff_runs.py` refuses to compare two runs whose modes it cannot line - # up (§9). + # up. mode: Optional[str] = None - # ---- site-specific, at the end (§9) --------------------------------- + # ---- site-specific, at the end --------------------------------- # The consumption-tax rate the price INCLUDES, as a percentage. Japan runs # two: 10% standard, and 8% reduced on ingestibles — a beauty supplement # sits at 8% while the cream beside it is at 10%. The site states the @@ -188,7 +188,7 @@ class Product: badges: Optional[str] = None # Everything below is published on a PRODUCT page and not on a tile, and # is None on a listing row. They are the reason to open a detail page at - # all, which is what keeps them off §9's "null on every row" list: in + # all, which is what keeps them off the "null on every row" list: in # `--mode product` they are populated. subcategory: Optional[str] = None # `容量 40g` — the pack size, from the detail page's variation line. @@ -331,7 +331,7 @@ def _atomic(path: str, newline: Optional[str] = None): # hackernews-scraper, which fixed this first — the first version # written here used a flat `0o644 & ~umask`, which quietly narrows # 0664 to 0644 under umask 002 and clobbers a deliberately tightened - # file (§16: lift the sibling's fix, do not invent a second one). + # file (lift the sibling's fix, do not invent a second one). try: mode = os.stat(path).st_mode & 0o777 except OSError: @@ -369,9 +369,8 @@ def _csv_escape(v: Any): """Neutralise a formula-shaped cell. Returns (value, was_escaped). Only STRINGS are touched. Escaping a number would turn `-5` into text - and break every sum a consumer writes over the column, and this schema - has negative numbers that matter: `savings_amount` and `discount_pct` - are what anyone monitoring this site is actually summing. + and break every sum a consumer writes over the column — `price` and + `rating` are what anyone monitoring this site is actually summing. CSV only, on purpose. The JSON output keeps the site's bytes exactly as served, so the two files deliberately differ, and `csv_cells_escaped` @@ -529,7 +528,7 @@ def write_last_attempt(out_prefix: str, meta: dict) -> str: `finish_run` deliberately writes no `.meta.json` for a failed run: `save` leaves the previous good output in place, and a "failed" sidecar beside good data would contradict it and make `diff_runs.py` refuse a - comparison of data that is in fact fine (§9). + comparison of data that is in fact fine. The cost of that, until this existed, was that a failed run left the caller NOTHING machine-readable — the exit code was the whole story, so @@ -589,8 +588,8 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, Two warnings live here rather than in a column, and both are this site's own arithmetic. A category listing publishes its total page count, so a run can PLAN against it and "complete" means what the word ought to mean: - `/c/c15/` states 13 pages, and 12 × 24 + 16 = 304 products is the whole - category. A TAG listing publishes no counter at all, so `pages_available` + on 2026-09-18 `/c/c15/` stated 13 pages, and 12 × 24 + 16 = 304 products + was the whole category then (the count moves with the catalogue). A TAG listing publishes no counter at all, so `pages_available` is None there — which means UNKNOWN and never zero, because capping a tag run at zero pages would turn a working route into an empty file. @@ -621,7 +620,7 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, } if extra: # Merged rather than nested under a key, so a consumer reads - # `shop_rating` at the top level beside `products`. Run fields win a + # `pages_available` at the top level beside `products`. Run fields win a # name collision: a caller cannot accidentally overwrite `status`. meta.update({k: v for k, v in extra.items() if k not in meta}) return meta @@ -690,8 +689,8 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, # # It is a COMPLETE run, and here it is complete in the strong sense: the site # imposes no cap of its own, so the planned page count IS the whole category. -# `/c/c15/` states 13 pages and 12 × 24 + 16 = 304 products is all of Cosme -# Decorté (measured 2026-09-18). +# On 2026-09-18 `/c/c15/` stated 13 pages and 12 × 24 + 16 = 304 products +# was all of Cosme Decorté; the count moves as the catalogue changes. # # Planning against it is not merely an optimisation on this site, which is # the part worth remembering. Walking off the end of a category returns the @@ -708,10 +707,11 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, # Note "parser_found_nothing" is deliberately ABSENT. A page Maison KOSÉ # served that links to products and parsed to zero rows is OUR failure, not a # complete answer, and a run that ends that way must not report `complete` -# (§20). Its exit code stays EXIT_NO_PRODUCTS — the catalogue question really -# was answered — so only the status and the stop_reason carry the distinction. +# Its exit code is EXIT_API_ERROR (5) through finish_run's not-complete rule: +# with zero rows nothing was obtained that a consumer could use, and +# EXIT_NO_PRODUCTS (4) would claim an empty catalogue the page contradicts. # -# It CAN fire on this site, which §20 says to check before adding the signal: +# It CAN fire on this site, which is worth checking before adding the signal: # rows are built from `li.c-product__item` tiles while `product_link_count` # counts `/g/g…/` hrefs, and a grid page carries 28 of those against 24 # tiles. So a tile-markup change leaves the links intact and the rows at diff --git a/page_flow.py b/page_flow.py index 327b538..d605193 100644 --- a/page_flow.py +++ b/page_flow.py @@ -2,7 +2,7 @@ page_flow.py ------------ The retry / solve / blocked decision, as DATA rather than as three copies of -an if-chain (CLAUDE.md §1). +an if-chain. Maison KOSÉ answers a request four ways, and three of them want a different response: @@ -29,13 +29,13 @@ 12 consecutive page fetches, no delay between them -> 12 × HTTP 200 - zero captcha markers of any vendor across 13 captures (§18's count, in + zero captcha markers of any vendor across 13 captures (the count, in product_parser.BOT_CHALLENGE_MARKERS) So on this site the block path is a TRIPWIRE, not a description of anything observed. It is carried in full anyway, because the alternative to a tripwire is a run that reports an empty catalogue on the day Kosé turns something on — -and because CLAUDE.md §19's most expensive bug in this family was a SENTENCE +and because the most expensive bug in this family was a SENTENCE claiming a captcha could not be met. Nothing here claims a captcha is unsolvable on this site. There is no widget @@ -50,13 +50,13 @@ fetching until a page comes back empty" never terminates on this route, and `pages_to_plan` bounds a run by the site's own `N／Mページ` counter rather than by walking off the end. A tag listing DOES end with an empty page, so -both routes still need the data-based stop (§7 layer 3). +both routes still need the data-based stop. **No JavaScript crosses this boundary.** `wait_for_count` takes a callback each engine implements with its own `querySelectorAll`, never a string for the browser to evaluate: a site whose CSP omits `unsafe-eval` kills `wait_for_function` with an `EvalError` and takes the run down with exit 1 -(CLAUDE.md §18). maison.kose.co.jp's CSP does allow `unsafe-eval` — it is in +. maison.kose.co.jp's CSP does allow `unsafe-eval` — it is in the header, checked — so this costs nothing here and keeps the habit. """ @@ -69,7 +69,7 @@ # How many tiles mean "the grid has painted". Must be > 1: waiting for one -# match can resolve on something unrelated long before a grid is there (§5). +# match can resolve on something unrelated long before a grid is there. # # Two rather than four, and the number is measured rather than inherited. The # anchor below is a real product TILE, not a link, so it cannot resolve on a @@ -126,7 +126,7 @@ def wait_for_count(count: Callable[[str], int], selector: str, minimum: int, Returns the last count seen, so a caller can report "13 of 24 painted" rather than a bare timeout. The OPERATION is named and the driver's own primitive is passed in, so no JavaScript dialect crosses this boundary - (CLAUDE.md §1): Selenium's `execute_script` wants a function body with an +: Selenium's `execute_script` wants a function body with an explicit `return` while Playwright and pyppeteer want `() => expr`, and a shared module that spelled either would quietly acquire one driver's accent. @@ -171,7 +171,7 @@ def attempts(value: str) -> int: the HTTP path should replace it. The same command with `--retries 1`, same address and same minute, returned 199,558 bytes and 24 products. So this flag did not merely waste a run: it produced a confident, wrong - verdict about the whole project. CLAUDE.md §28 records the identical + verdict about the whole project. the family's notes record the identical sequence on a sibling repo, which is why this is a usage error now rather than a documented quirk. """ @@ -205,6 +205,47 @@ def extra_attempts(value: str) -> int: return n +def at_least_one(value: str) -> int: + """argparse type for a COUNT that must be positive: --pages, --concurrency. + + `--pages 0` or `--pages -1` used to be accepted and fetched nothing, so + the run reported "no products" for a site it never asked. + """ + try: + n = int(value) + except ValueError: + raise argparse.ArgumentTypeError(f"{value!r} is not an integer") + if n < 1: + raise argparse.ArgumentTypeError("must be at least 1") + return n + + +def non_negative_int(value: str) -> int: + """argparse type for a count where 0 means "none": --proxy-block-retries.""" + try: + n = int(value) + except ValueError: + raise argparse.ArgumentTypeError(f"{value!r} is not an integer") + if n < 0: + raise argparse.ArgumentTypeError("cannot be negative") + return n + + +def non_negative_seconds(value: str) -> float: + """argparse type for a pause in seconds: --delay, --retry-delay. + + A negative pause is not a shorter one: `time.sleep` raises ValueError on + it, mid-run, after the first page has already been fetched. + """ + try: + n = float(value) + except ValueError: + raise argparse.ArgumentTypeError(f"{value!r} is not a number") + if not n >= 0: # also refuses NaN + raise argparse.ArgumentTypeError("cannot be negative") + return n + + def classify(html: Optional[str], status: Optional[int] = None, url: str = "") -> str: """The page's state, as one word. @@ -213,7 +254,7 @@ def classify(html: Optional[str], status: Optional[int] = None, against every call site by the suite, because a sibling repo shipped two engines calling `classify(html, url=…)` against a `classify(html, status, url)` and both crashed on their FIRST fetch — invisible to import, - `--help` and `compileall` (CLAUDE.md §17). + `--help` and `compileall`. """ return classify_with_reason(html, status, url)[0] @@ -229,7 +270,7 @@ def classify_with_reason(html: Optional[str], status: Optional[int] = None, # A transport-level failure, which never reaches `classify` because no # response arrived. Chromium reports a dead proxy as a generic error rather # than as a timeout, and the two want OPPOSITE responses: a timeout deserves -# another try at the same exit, a dead proxy a different one (CLAUDE.md §8). +# another try at the same exit, a dead proxy a different one. _PROXY_FAILURE_RE = re.compile( r"ERR_PROXY_CONNECTION_FAILED|ERR_TUNNEL_CONNECTION_FAILED" r"|ERR_PROXY_AUTH_UNSUPPORTED|ERR_UNEXPECTED_PROXY_AUTH" @@ -273,9 +314,9 @@ def looks_like_a_parse_failure(state: str, rows: int, link_count: int) -> bool: That combination is this repo's bug, not the site's, and it deserves to say so by name — "0 products" sends the reader to check the URL when the - thing to check is the parser (CLAUDE.md §20). + thing to check is the parser. - It CAN fire here, which §20 says to check before adding the signal: rows + It CAN fire here, which is worth checking before adding the signal: rows come from `li.c-product__item` tiles while the link count comes from `/g/g…/` hrefs, and a grid page carries 48-55 of those against 24 tiles. A tile-markup change therefore leaves the links untouched and the rows at @@ -311,7 +352,7 @@ def looks_like_a_parse_failure(state: str, rows: int, link_count: int) -> bool: # A refusal with no challenge on it. There is nothing to solve — an edge # that declines is not offering a test — so the only move is a different # exit. `solve` is False on a state whose name says blocked, and that is - # the point: §8's "detected ≠ blocking ≠ paying". + # the point: the "detected ≠ blocking ≠ paying". "blocked": {"retry": True, "solve": False, "blocked": True, "parse": False}, # A challenge widget. This one IS a test and is the state that pays for a # solver. A fresh browser from a different exit clears some challenges @@ -319,7 +360,7 @@ def looks_like_a_parse_failure(state: str, rows: int, link_count: int) -> bool: "challenge": {"retry": True, "solve": True, "blocked": True, "parse": False}, # Not recognisably a Maison KOSÉ page: an interstitial, a proxy's error # page, or Chromium's own — which carries the site's hostname in its - # and would fool a title check (§18). A wait, not a spend. + # <title> and would fool a title check. A wait, not a spend. "unknown": {"retry": True, "solve": False, "blocked": False, "parse": False}, } @@ -344,7 +385,7 @@ def should_parse(state: str) -> bool: # engines rather than merely documented — setting it False really does stop # the retry loop. (A sibling repo carried this constant with a paragraph of # justification and no reader, which is the same defect as dead code that -# looks load-bearing: §17.) +# looks load-bearing.) RETRY_ON_BLOCKED = True # How many times to re-fetch a blocked page when there is no pool to rotate @@ -361,7 +402,7 @@ def pagination_is_addressable(url: str) -> bool: """Can page N of this listing be fetched without fetching page N-1? Yes on both of this site's listing routes, and it was CHECKED rather than - assumed (CLAUDE.md §7): page 1's own `rel=next` agrees with what the + assumed: page 1's own `rel=next` agrees with what the convention builds, on both. That is what lets `--concurrency` above 1 mean anything here. @@ -378,7 +419,7 @@ def pages_to_plan(pages_requested: int, pages_avail: Optional[int], One implementation, in the shared module, because three engines each doing this arithmetic is how three engines come to disagree about where a - run ends (§1). Returns a range so a caller cannot get the inclusive bound + run ends. Returns a range so a caller cannot get the inclusive bound wrong in its own loop. `start_page` is honoured rather than normalised away: a run pointed at @@ -409,6 +450,6 @@ def concurrency_limit(cdp_endpoint: Optional[str]) -> Optional[int]: The Scraping Browser API allows ONE live connection per profile, so workers sharing an endpoint collide with `profile_locked`. Several `pid`s - and one run each is the way to parallelise that path (CLAUDE.md §7). + and one run each is the way to parallelise that path. """ return 1 if cdp_endpoint else None diff --git a/playwright_scraper.py b/playwright_scraper.py index d12e4f6..6e970f0 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -21,7 +21,7 @@ What is different about this site --------------------------------- * **You need none of the paid products, and the README says so plainly - (§13).** Measured 2026-09-18 from a datacentre address: `curl`, +.** Measured 2026-09-18 from a datacentre address: `curl`, `python-requests` and a request with NO User-Agent header each got the same 90,310-byte listing; twelve pages back to back were twelve 200s; and thirteen captures carry zero captcha markers of any vendor. The captcha, @@ -44,7 +44,7 @@ * **Wrong pagination parameters fail SILENTLY.** `&pageno=2` and `&page=2` on a tag listing both return page 1 with HTTP 200. `page_url()` builds the convention the site publishes in its own `rel=next`, which is the check - CLAUDE.md §7 demands before trusting a constructed URL. + is required before trusting a constructed URL. * **`li.` is load-bearing in the tile selector.** A product page reuses the class `c-product__item` on `<div>`s for staff-review cards — 15 on one @@ -53,7 +53,7 @@ class `c-product__item` on `<div>`s for staff-review cards — 15 on one * **There is one market.** No `hreflang`, no `/en/` route, every price in JPY, so there is no `--locale` to pass and no ordering control on either listing route, so there is no `--sort`. Both flags are ABSENT rather than - inert (CLAUDE.md §3). + inert. Usage: @@ -106,7 +106,7 @@ class `c-product__item` on `<div>`s for staff-review cards — 15 on one # The browser locale the engines present. A constant rather than a flag: # maison.kose.co.jp is a single Japanese market — no hreflang set, no /en/ # route, every price in JPY — so a --locale would be a setting that changes -# nothing, which CLAUDE.md §3 calls out by name. Sending ja-JP is also the +# nothing, which is the failure the loader's warning exists for. Sending ja-JP is also the # honest value: claiming en-US while reading a Japanese storefront is itself # a mismatch between what the browser says and what it is doing. BROWSER_LOCALE = "ja-JP" @@ -199,7 +199,7 @@ def ok(self) -> bool: # tile on this site carries a `/img/goods/` thumbnail, 171 of 171 in the # captures and 72 of 72 on a live run. The first version of the parser read # the tile's FIRST `<img>` and got the "NEW" badge on every row that had one -# — 100% coverage of the wrong value, which is exactly the failure §10 means +# — 100% coverage of the wrong value, which is exactly the failure a value check exists for # by "assert VALUES on real fixtures, not coverage". # The share of rows that must carry a usable price before the run is worth @@ -231,7 +231,7 @@ def _driver(page): # driver's dialect. # # There is no scroll primitive here, and its absence is measured rather - # than forgotten (§8: missing content is one of three things, so check + # than forgotten (missing content is one of three things, so check # which before "fixing" it). Maison KOSÉ SERVER-RENDERS its grid: a plain # HTTP fetch with no JavaScript at all returns every `li.c-product__item` # tile of the page (24 on a full one). Everything the parser @@ -266,7 +266,7 @@ def _target_url(args) -> str: Maison KOSE server-renders its catalogue and has no JSON endpoint to prefer over the rendered page, which was CHECKED rather than assumed - (CLAUDE.md §21: ask what the front end calls before assuming a browser). + (ask what the front end calls before assuming a browser). Grepped across the captures for `/api/`, `/graphql`, `__NEXT_DATA__` and `__PRELOADED_STATE__`: zero hits on every listing and product page. @@ -274,7 +274,7 @@ def _target_url(args) -> str: it names one. An earlier version normalised `/c/c15_p10/` back to `/c/c15/`, which meant a run pointed at page 10 quietly returned page 1's products and reported success. That is this codebase's most common bug - class (§8), and the fix is simply to fetch what was asked for: §7's "page + class, and the fix is simply to fetch what was asked for: the "page 1 is fetched alone" is about the run's FIRST page deciding how far the rest can be addressed, not about the listing's first page. """ @@ -293,7 +293,7 @@ def _plan_page_urls(args, page_one_url: str, The convention is not guessed from the shape of page 1's URL: it is the one the site itself publishes in `<link rel="next">`, and `page_url` - reproduces it byte for byte on both routes. That is the check §7 demands + reproduces it byte for byte on both routes. That is the check required before trusting a constructed URL, and it matters here because getting it wrong is SILENT — `&pageno=2` and `&page=2` on a tag listing both return page 1 with HTTP 200. @@ -574,7 +574,7 @@ def _is_endpoint(url: str) -> bool: it is honest; silently dropping the branch would make the engines diverge in shape for no reason. - CLAUDE.md §21 says to ask what the front end calls before assuming a + the rule is to ask what the front end calls before assuming a browser is needed, and that was done here first rather than last: the captures were grepped for `/api/`, `/graphql`, `__NEXT_DATA__` and `__PRELOADED_STATE__` and every one came back ZERO. This is an ASP.NET @@ -608,7 +608,7 @@ def _snapshot(page, url: str) -> Optional[str]: return _content_when_settled(page) -def handle_captcha_if_present(page, args) -> bool: +def handle_captcha_if_present(page, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Runs after EVERY navigation, for ANY page — not scoped to one URL. The @@ -624,11 +624,11 @@ def handle_captcha_if_present(page, args) -> bool: That is a statement about what this site renders TODAY, not a claim that a captcha here could not be solved. Nothing in this repo makes that - claim: CLAUDE.md §19 records what it cost the family once, and the only + claim: the family's notes record what it cost the family once, and the only sentence anyone is entitled to is which task type would be built. That is a statement about what the SITE renders, not about what can be - solved (§19: never write that a captcha cannot be solved — the only + solved (never write that a captcha cannot be solved — the only sentence anyone is entitled to is "this page carries no widget"). This path exists because a bot manager can be switched on between deploys, and because the family's rule is that detection stays broad: @@ -671,6 +671,10 @@ def handle_captcha_if_present(page, args) -> bool: "solved — continuing with whatever the page already " "holds.") return False + # The allowance is claimed here, immediately before the only paid + # request, so a call that detected nothing has cost nothing. + if not budget.claim(): + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -693,14 +697,14 @@ def _parse_for_mode(html: str, url: str, args, page_num: int = 1): The two entry points are separate on purpose and it is not tidiness. A product page carries 15 `div.c-product__item` staff-review cards and an `awoo-product-list` recommendation block, so a listing parser turned on - one is exactly how a parser invents products (CLAUDE.md §4). This repo's + one is exactly how a parser invents products. This repo's tile selector is element-qualified (`li.c-product__item`) and therefore sees zero of them, which `smoke_test.py` pins in both directions. `page_num` is threaded through rather than defaulted, because `position` restarts at 1 on every page: without the page number beside it, a row from page 2 claims the same position as one from page 1 and the two are - indistinguishable in the output (§18). + indistinguishable in the output. """ if args.mode == "product": rows = parse_product_detail(html, url, mode=args.mode) @@ -716,33 +720,40 @@ def _parse_for_mode(html: str, url: str, args, page_num: int = 1): class _SolveBudget: - """Every paid solve on one page goes through here. One counter, checked - BEFORE each purchase rather than beside one of them. + """The per-page allowance for paid solves. One counter, claimed + IMMEDIATELY BEFORE the paid request is sent, and nowhere else. - This exists because the budget was enforced at ONE of the two call sites. `handle_captcha_if_present` runs twice per attempt — once before the page is classified, so a challenge is cleared before anything is judged, and - once after, for the state that says the page really is gated — and only - the second call was counted. The uncounted one repeats on every - block-retry, so the constant read like a limit and was advisory. - - Nothing about that is visible on a site where a challenge is rare, which - is exactly this one: no challenge of any vendor has ever been observed on - maison.kose.co.jp, so this has never cost anything here. It is carried - because the day it does cost something is the day nobody is watching — - CLAUDE.md §17's "a policy constant nothing reads", with a bill attached, - and §23 measured one page buying three solves on a sibling. - - The budget deliberately does NOT reset on rotation: a fresh exit is a - reason to re-fetch, not a fresh allowance to spend. + once after, for the state that says the page really is gated. Both calls + take this object, and both claim from it only at the point where a task + would really be submitted. + + Two earlier versions were each wrong in one direction. The first counted + only the second call site, so the first repeated on every block-retry and + one page could buy a solve per retry against a limit of one. The second + counted every CALL, purchase or not: the pre-classification call, which + on a normal page detects nothing and buys nothing, used up the whole + allowance, so a challenge first seen after classification (a widget + rendered late, or a page that only turns into a challenge after the + readiness wait) was never offered to the solver and the page was + reported as blocked. + + A claim that is then refused or fails at the solver still counts: it was + a request sent, and possibly billed, and not counting it would reopen + the overspend. Detecting nothing, finding the content already rendered + or having no API key claims nothing. + + The allowance deliberately does NOT reset on retry or rotation: a fresh + exit is a reason to re-fetch, not a fresh allowance to spend. """ def __init__(self, limit: int): self.limit = limit self.spent = 0 - def spend(self, solve) -> bool: - """Run `solve()` if the budget allows, and count it. False if not.""" + def claim(self) -> bool: + """Count one purchase if the allowance permits it. False if not.""" if self.spent >= self.limit: logger.info("Not attempting another solve on this page: %d of %d " "already bought. A challenge that survives a solved " @@ -751,7 +762,7 @@ def spend(self, solve) -> bool: self.spent, self.limit) return False self.spent += 1 - return bool(solve()) + return True def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome: @@ -853,7 +864,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Routed through the budget like every other purchase. This # call used to be unguarded, which is how one page could buy # a solve per block-retry against a limit of one. - if budget.spend(lambda: handle_captcha_if_present(session.page, args)): + if handle_captcha_if_present(session.page, args, budget): # A solve navigated the page. Give the destination a moment # before judging what came back. session.page.wait_for_timeout(1000) @@ -898,7 +909,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # the two states, and it is bounded by SOLVES_PER_PAGE so a rotation # loop cannot become a bill. if page_flow.should_solve(state): - if budget.spend(lambda: handle_captcha_if_present(session.page, args)): + if handle_captcha_if_present(session.page, args, budget): session.page.wait_for_timeout(1000) html = _snapshot(session.page, url) or html state = _classify(session.page, html, http_status) @@ -924,9 +935,9 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Blocked or challenged. A different exit is the one thing that # plausibly changes the outcome: the ADDRESS is what was scored, not - # the URL, so retrying it unchanged would only confirm it. Measured - # 2026-09-09 — the same URL that answers 403 from a datacentre exit - # answers 200 from a residential one. + # the URL, so retrying it unchanged would only confirm it. (On this + # site no refusal has been observed from any address, datacentre + # included; this branch is here for the day that changes.) if block_attempt < block_retries: if has_pool: logger.warning("Page %d came back as %s from %s — retrying " @@ -1004,7 +1015,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # This site server-renders its whole grid in the first response, so there # is nothing to wait for and nothing to scroll into view. Porting the # sibling repos' scroll loop here would be dead code that looks - # load-bearing (CLAUDE.md §4). + # load-bearing. # Dumping on success, not only on failure: a run can return the right # NUMBER of rows with a field silently unpopulated, and then the only way @@ -1069,7 +1080,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Said once per run rather than per page. `pages_available` is # None on a TAG listing, which publishes no counter at all, and # None means UNKNOWN: the run then walks until a page adds no new - # sku instead of planning (§7 layer 3). + # sku instead of planning. logger.info("Page 1: %s tile(s), %s product-shaped link(s), the " "site reports %s. Canonical: %s", listing.tiles_on_page, listing.product_links_on_page, @@ -1078,7 +1089,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome else "no page counter (a tag listing publishes none)", listing.canonical or "none") - # §20: tell a BROKEN PARSER apart from an EMPTY CATEGORY before anything + # Tell a BROKEN PARSER apart from an EMPTY CATEGORY before anything # downstream reports "0 products" and sends the reader to check the URL. if not products and page_flow.looks_like_a_parse_failure( state, len(products), product_link_count(html or "")): @@ -1258,8 +1269,8 @@ def scrape(args) -> int: # "page_cap_reached" means the run asked for more pages than the # category holds and was clamped to the site's own count. On this site # that is a COMPLETE result in the strong sense: this site imposes no cap - # of its own, so the clamp IS the whole category — /c/c15/ states 13 - # pages, and 12 x 24 + 16 = 304 is all of Cosme Decorte. + # of its own, so the clamp IS the whole category — on 2026-09-18 /c/c15/ + # stated 13 pages, 12 x 24 + 16 = 304 products. # # Only --mode product is single-page. Both listing routes paginate — # `_pN` on a category, `?p=N` on a tag listing — so neither may be @@ -1316,7 +1327,7 @@ def scrape(args) -> int: if target != args.url: # The only normalisation this site needs: a listing URL that # already names a page is reset to page 1, because page 1 is - # what decides how far the rest can be addressed (§7). + # what decides how far the rest can be addressed. logger.info("Normalised to page 1: %s", target) # Page 1 is always fetched on its own: its content is what decides @@ -1343,7 +1354,7 @@ def scrape(args) -> int: "not the id.", first.url) elif first.state == "parse_failed": # Served, linked to products, parsed to nothing: OUR bug, and - # it must not reach the sidecar as a complete run (§20). + # it must not reach the sidecar as a complete run. stop_reason = "parser_found_nothing" elif args.mode == "product": pass # one page is the whole run — but many rows @@ -1469,7 +1480,7 @@ def scrape(args) -> int: # # NOT "pages x rows-per-page" as a hard expectation, even though the # page size is the site's fixed 24: the LAST page of a listing is - # legitimately short — /c/c15/ ends 12 x 24 + 16 = 304 — and a threshold that fires on every healthy run teaches the + # legitimately short — /c/c15/ ended 12 x 24 + 16 on 2026-09-18 — and a threshold that fires on every healthy run teaches the # reader to ignore it. What is worth warning about is a page that came # back materially THIN against its siblings, which is what a truncated # response looks like. @@ -1509,7 +1520,7 @@ def scrape(args) -> int: # No `locale` here, and its absence is a measurement rather than an # omission: maison.kose.co.jp is one market. No `hreflang` set, no /en/ # route, every price in JPY — so a locale field would carry `ja-jp` on - # every run forever, which is CLAUDE.md §9's "a value that never varies + # every run forever, which is the "a value that never varies # is not a field". extra = None if args.mode != "product": @@ -1528,11 +1539,11 @@ def scrape(args) -> int: # is the finding rather than an omission. bbb-scraper needs both # because that site refuses to serve past page 15 however many rows # matched, so "complete" there can mean a 1.2% sample. This site - # imposes no such cap: /c/c15/ states 13 pages and 12 x 24 + 16 = 304 - # is the whole of Cosme Decorte, so `pages_available` already says + # imposes no such cap: on 2026-09-18 /c/c15/ stated 13 pages and + # 12 x 24 + 16 = 304 products, the whole of Cosme Decorte, so `pages_available` already says # everything those fields would — and a `reachable_max` of # pages x page_size would OVERSTATE a short last page: 13 x 24 = 312 - # against the 304 that exist. + # against the 304 that existed then. # Which engine ran, what version of its driver, and how long it took. # A sidecar that says only "complete" cannot answer "why is this run @@ -1596,7 +1607,7 @@ def parse_args(): "/site/cosmedecorte/c/c15/ and /site/xyz/c/c15/ are " "the same page. Rows otherwise carry the cNN the URL " "asked for.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.at_least_one, default=1, help="Number of listing pages to fetch. Applies to --mode " "listing; ignored in --mode product. A category " "page prints the site's own page counter, so a run " @@ -1606,8 +1617,8 @@ def parse_args(): "on this site, so a request is limited only by what " "the category holds, and the sidecar records both " "numbers.") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--delay", type=page_flow.non_negative_seconds, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.at_least_one, default=1, metavar="N", help="Fetch pages through N parallel workers (default 1 — " "unchanged sequential behaviour). Each worker runs its " "own browser and holds its own proxy exit, so N>1 " @@ -1615,7 +1626,7 @@ def parse_args(): "from one address. Ignored with --cdp-endpoint.") p.add_argument("--retries", type=page_flow.attempts, default=3, help=page_flow.RETRIES_HELP) - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", type=page_flow.non_negative_seconds, default=2.0, help="Seconds before the first page-load retry, doubling " "thereafter (default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") @@ -1634,7 +1645,7 @@ def parse_args(): p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup, so concurrent runs do not " "all begin on the first exit in the file.") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", type=page_flow.non_negative_int, default=2, help="When a page comes back refused or behind a captcha, " "retry it from this many OTHER exits before giving up " "(default 2). Needs a pool of more than one; ignored " @@ -1755,7 +1766,7 @@ def parse_args(): # Refused rather than attempted. This parser reads Maison KOSE's own # tile markup and its /c/cNN/ and /g/gSKU/ URL shapes; pointing it at # another site would not fail loudly, it would return zero rows and - # look like an empty result (§8). The REASON is given, because "is + # look like an empty result. The REASON is given, because "is # not a Kose site" about kose.co.jp — which plainly is one — sends # the reader hunting a typo they did not make. p.error(f"{args.url!r} {why}.") diff --git a/product_parser.py b/product_parser.py index dd4e494..f6fcdb8 100644 --- a/product_parser.py +++ b/product_parser.py @@ -2,7 +2,7 @@ product_parser.py ----------------- Everything this repo knows about Maison KOSÉ. If you are writing a site fact -anywhere else, it belongs here (CLAUDE.md §1). +anywhere else, it belongs here. Which site this is, and why it is not `kose.com` ------------------------------------------------ @@ -16,8 +16,7 @@ Kosé's actual storefront is `maison.kose.co.jp` (Maison KOSÉ), which is where `www.kose.co.jp` sends a visitor with its own `<meta http-equiv="refresh">`. That is the site this repo reads, and it is one market: Japanese, JPY, no -`hreflang` set and no `/en/` route (all three checked 2026-09-18). CLAUDE.md -§15's "run a SECOND country site" step is therefore answered by "there is +`hreflang` set and no `/en/` route (all three checked 2026-09-18). The "run a SECOND country site" step is therefore answered by "there is only one", and the diversity in the fixtures is across BRANDS and page KINDS instead — which is where this site's traps turned out to live. @@ -46,7 +45,7 @@ Two pagination conventions, and one of them repeats its last page ----------------------------------------------------------------- -Both listing routes publish `<link rel="next">`, which is CLAUDE.md §7's +Both listing routes publish `<link rel="next">`, which is the layer 1 and leads `NEXT_PAGE_SELECTOR`. Layer 2 is `page_url()`, and it is route-dependent — a category takes `_p{N}` glued onto the id, a tag listing takes `?p={N}`. Layer 3, the data-based stop, is not optional here, because @@ -63,20 +62,20 @@ page again, forever. A run that stopped on "no tiles" would never stop, and a run that trusted a page count would be fine only until the catalogue changed under it. `parse_listing` reports the site's own `N／Mページ` counter so the -engines can PLAN against it (§7 layer 2, with the site doing the arithmetic), -and the engines still stop on "this page added no new sku" (§7 layer 3). +engines can PLAN against it (with the site doing the arithmetic), +and the engines still stop on "this page added no new sku". Wrong parameters are silent rather than loud, which is the same trap from the other side: `&pageno=2` and `&page=2` on a tag listing both return page 1 with HTTP 200. Where the data is: no JSON-LD on a listing, one block on a product ------------------------------------------------------------------- -Counted before a line of parser was written, per §15 step 2: +Counted before a line of parser was written step 2: listing pages 0 `application/ld+json` blocks (all 8 captures) product pages 1 block, `@type: Product` (all 4 captures) -So CLAUDE.md §4's primary path does not exist on the route that matters most, +So the primary path does not exist on the route that matters most, and the listing parser is built on the site's own tile markup instead — which here is unusually good: `li.c-product__item` holding `p.c-product__brand`, `p.c-product__product-name`, `p.c-product__price` and @@ -88,12 +87,11 @@ A product DETAIL page reuses the class `c-product__item` on a `<div>` for staff-review cards linking to `/staff/skincaredetail/…`. Measured on `product_MUSC`: **15** of them. A class-only selector would invent fifteen -products per detail page, each with a `/staff/` URL and no sku — CLAUDE.md -§4's junk-link data theft wearing a new costume. The element-qualified +products per detail page, each with a `/staff/` URL and no sku — the junk-link data theft wearing a new costume. The element-qualified `li.c-product__item` sees zero, and `smoke_test.py` pins that. The same page also carries an `awoo-product-list` recommendation block -(`awoo-product-name` ×15), which is the "similar products carousel" §4 warns +(`awoo-product-name` ×15), which is the "similar products carousel" to watch for about. Both are excluded by construction rather than by a filter: the tile selector names the element, and the detail parser reads the page's own JSON-LD, never tiles. @@ -107,7 +105,7 @@ 1,234円（税込）※ the same, with the footnote marker 3,024円（税込/8%）※ the reduced rate -- 2 nodes, both on /c/c15_p10/ -So §4's tile-price overlay is **deleted rather than ported**, which is what +So the tile-price overlay is **deleted rather than ported**, which is what that section says to do when a site has no discount chain: there is no second view to reconcile against, and an overlay here would be dead code that looks load-bearing. `price_source` records WHICH NODE was read (`tile` / `jsonld` / @@ -115,7 +113,7 @@ with like. The structured and displayed prices AGREE, which was checked rather than -assumed (§4): JLCW 7700/7,700円, MUSC 5940/5,940円, SIRY 3300/3,300円. +assumed: JLCW 7700/7,700円, MUSC 5940/5,940円, SIRY 3300/3,300円. `※` is `Maison KOSÉ販売価格` — "the Maison KOSÉ selling price", the store's own figure as opposed to a maker's suggested one. It is kept as @@ -130,7 +128,7 @@ Stock is read from the site's own button text, as an allowlist --------------------------------------------------------------- -Measured across the same 171 tiles, and this is CLAUDE.md §20's rule — map +Measured across the same 171 tiles, and this is the rule — map availability as an ALLOWLIST so an unanticipated value reads as "not available" rather than as a quiet True: @@ -141,14 +139,14 @@ 現在購入頂けません 18 currently unavailable -> NOT available 予約受付終了 1 pre-orders closed -> NOT available -The column is verified in BOTH directions, which §20 says is otherwise not +The column is verified in BOTH directions, which is otherwise not verified at all: 19 of 171 tiles are unavailable, and the JSON-LD on `gJQBA` independently says `OutOfStock` where its tile says 現在購入頂けません. Detection: there is nothing to detect, and that is the measurement ------------------------------------------------------------------- Every candidate marker was counted on pages known to be good BEFORE any was -added (§18), across all 13 captures including the site's own 404 and an +added, across all 13 captures including the site's own 404 and an empty listing: recaptcha · grecaptcha · g-recaptcha · data-sitekey · api.js?render · @@ -161,10 +159,10 @@ `BOT_CHALLENGE_MARKERS` is a short list of shapes that would have to appear if this site ever turned a challenge on — it is a tripwire, not a description of something observed, and it is documented as such so nobody later reads it -as evidence that a challenge was met. Per §18 the scan only ever REFINES the +as evidence that a challenge was met. The scan only ever REFINES the reason for a page the policy already gave up on. -`cf-turnstile` is deliberately ABSENT from that list, per §19: 2Captcha's own +`cf-turnstile` is deliberately ABSENT from that list: 2Captcha's own Scraping Browser auto-solve extension injects it into every page it loads, so it fires on good pages and misses real ones. `challenges.cloudflare.com` is the one that works, and it is what this list carries. @@ -172,8 +170,8 @@ The positive structural signal is `/js/sys/`, which is the site's own script directory: **9 to 11 references on every served page**, including the 404 and the empty listing, and none on anything the site did not build. That is -CLAUDE.md §8's `assets.mmsrg.com` trick, which also answers Chromium's own -network-error page (§18) — it carries the site's hostname in its title and +the `assets.mmsrg.com` trick, which also answers Chromium's own +network-error page — it carries the site's hostname in its title and would fool a title check. """ @@ -200,7 +198,7 @@ # A host that is a real Kosé property but NOT this site, kept so the refusal # can say WHY rather than "unsupported host", which sends the reader hunting -# for a typo (CLAUDE.md §5). +# for a typo. _OTHER_KOSE_HOSTS = { "www.kose.co.jp": "the Kosé corporate site, which publishes no catalogue " "and redirects to maison.kose.co.jp", @@ -257,7 +255,7 @@ def _split(url: str): HOST was "site" and whose path had lost its first segment. Every category row then parsed to `sku=None` and was dropped: 24 tiles in, ZERO rows out, while `tile_count` went on reporting 24. That is this codebase's - most common bug class (§8: doing less than it says while reporting + most common bug class (doing less than it says while reporting success) and it survived the first run because the tag fixtures, whose hrefs are absolute, passed throughout. The suite now pins both spellings. """ @@ -360,7 +358,7 @@ def brand_slug_from_url(url: str) -> Optional[str]: # --------------------------------------------------------------------------- def page_url(url: str, page_num: int) -> str: - """Layer 2 of CLAUDE.md §7: page N's address, built from the convention. + """Layer 2 of pagination: page N's address, built from the convention. Route-dependent, because this site has two conventions and using the wrong one FAILS SILENTLY: `?p=2` paginates a tag listing while `&page=2` @@ -460,19 +458,19 @@ def total_pages(html: str) -> Optional[int]: # `pages_to_fetch` used to live here and was removed on 2026-09-18, because # how many pages a RUN should ask for is policy rather than site knowledge -# (§1) and `page_flow.pages_to_plan` is the one implementation the engines +# and `page_flow.pages_to_plan` is the one implementation the engines # call. What is site knowledge, and stays here, is that the site publishes a # counter at all — `total_pages` above — and what that counter means when it # is absent. # # Recorded rather than deleted silently: it was found by grepping every -# public name for a consumer outside its own module (§17's check #5), which +# public name for a consumer outside its own module (the check #5), which # turned it up as a function only its own test still called. A check that # exercises something nothing uses passes for the wrong reason. -# `<link rel="next">` is published by both listing routes and is CLAUDE.md -# §7's layer 1: a standards-based signal, ahead of any build artefact. +# `<link rel="next">` is published by both listing routes and is +# pagination's layer 1: a standards-based signal, ahead of any build artefact. NEXT_PAGE_SELECTOR = ("link[rel='next']", "a[rel='next']") @@ -504,7 +502,7 @@ def canonical_url(html: str, base_url: str = "") -> Optional[str]: # JPY is the only currency this site quotes and it has no subunit, so the # comma is unambiguously a thousands separator and there is no decimal form -# to disambiguate (CLAUDE.md §4's three grouping conventions collapse to one +# to disambiguate (the three grouping conventions collapse to one # here). The largest price measured is 264,000 円. _PRICE_RE = re.compile(r"([0-9][0-9,]*)\s*円") @@ -540,7 +538,7 @@ def parse_tax_rate(text: str) -> Optional[int]: `（税込/8%）` states the reduced rate explicitly; a bare `（税込）` means the standard 10%. Absent entirely -> None, never a defaulted number, because a price with no tax annotation is not a tax-inclusive price we - can vouch for (CLAUDE.md §8: never present a guess as a fact). + can vouch for (never present a guess as a fact). """ if not text: return None @@ -554,7 +552,7 @@ def parse_tax_rate(text: str) -> Optional[int]: # Stock # --------------------------------------------------------------------------- -# The site's own call-to-action text, as an ALLOWLIST (CLAUDE.md §20). Counts +# The site's own call-to-action text, as an ALLOWLIST. Counts # are over the 171 tiles in the 2026-09-18 captures and are in the module # docstring. Anything not named here reads as "not available" rather than as # a quiet True, so a wording the site adds tomorrow under-reports instead of @@ -608,18 +606,18 @@ def _ld_in_stock(offer: dict) -> Optional[bool]: # A TRIPWIRE, not a description of anything seen. Every entry was counted on # 13 captures known to be good -- 8 listings, 4 product pages, the site's own -# 404 and an empty listing -- and every one was ZERO (§18: count a candidate +# 404 and an empty listing -- and every one was ZERO (count a candidate # on a page you know is good before adding it). They are here so that a # challenge switched on next month is reported as a challenge rather than as # an empty catalogue. # # `cf-turnstile` is deliberately absent: 2Captcha's Scraping Browser # auto-solve extension injects it into every page it loads, so it fires on -# good pages and misses real ones (CLAUDE.md §19). `challenges.cloudflare.com` +# good pages and misses real ones. `challenges.cloudflare.com` # is the marker that works and is what stands in for Turnstile here. # # Because nothing in this set matches anything that extension injects, the -# `chrome-extension://` strip that §8 describes would be dead code in this +# `chrome-extension://` strip would be dead code in this # repo and is deliberately not carried. BOT_CHALLENGE_MARKERS = ( # NOT a bare `recaptcha`. That is what this set carried until @@ -637,15 +635,15 @@ def _ld_in_stock(offer: dict) -> Optional[bool]: # www.google.com/recaptcha 0 # # Nothing broke, because `detect_page_state` checks for rendered tiles - # BEFORE it looks for a marker (§17's ordering) — but a page with no + # BEFORE it looks for a marker (the ordering) — but a page with no # tiles, an exhausted tag facet say, would have been reported as a # CHALLENGE over CDP: exit 3 for a correct answer. # # This repo's own claim that no marker fires on a good page was true of # every capture it had and false of one it did not have, because all of - # them were fetched locally. That is CLAUDE.md §21's "the guard passed + # them were fetched locally. That is the "guard passed # for the wrong reason: a fixture fetched the way a real run does not - # fetch", and §24 prescribes exactly the narrowing below. + # fetch", and that calls for exactly the narrowing below. ("recaptcha/api.js", "google recaptcha"), ("recaptcha/api2/anchor", "google recaptcha"), ("recaptcha/api2/bframe", "google recaptcha"), @@ -661,7 +659,7 @@ def _ld_in_stock(offer: dict) -> Optional[bool]: # A refusal page is small. Unescaping the whole of a 100 KB listing on every # fetch buys nothing and risks a product name deep in a grid reading as a -# marker (CLAUDE.md §20). +# marker. _UNESCAPE_PREFIX = 20_000 @@ -675,7 +673,7 @@ def detect_bot_challenge(html: str, url: str = "") -> Optional[str]: Normalises HTML entities over a bounded prefix first, because an edge can entity-escape the punctuation in its own marker and a literal then - matches a browser's DOM and misses a raw HTTP response (CLAUDE.md §20). + matches a browser's DOM and misses a raw HTTP response. """ if not html: return None @@ -688,7 +686,7 @@ def detect_bot_challenge(html: str, url: str = "") -> Optional[str]: # The site's own script directory, referenced 9 to 11 times on every served # page including its 404 and an empty listing, and zero times on anything the -# site did not build. CLAUDE.md §8's positive-asset signal, which is also the +# site did not build. the positive-asset signal, which is also the # only thing that answers Chromium's own network-error page -- that page # carries the site's hostname in its <title> and no vendor marker at all. _ASSET_MARKERS = ("/js/sys/", "/freepage/maison-kose/") @@ -711,7 +709,7 @@ def product_link_count(html: str) -> int: site serve a catalogue page at all", which is a readiness question, and it is generous on purpose. The gap between the two is real and is why rows are built from tiles instead -- a grid page carries 28 such links - against 24 tiles, the extra four being carousel banners (§4). + against 24 tiles, the extra four being carousel banners. """ return len(_PRODUCT_HREF_RE.findall(html or "")) @@ -741,7 +739,7 @@ def _has_product_ld(html: str) -> bool: # `driver.get()` returns None and WebDriver exposes no HTTP status at all, so # Selenium cannot thread one however carefully the other two do. Rather than # leave one engine worse informed than its twins — which is exactly the -# drift §1 says a shared module exists to prevent — the parser reads the +# drift a shared module exists to prevent — the parser reads the # status out of the body where the site states one. # # Maison KOSE states it in the TITLE of its platform 404: the page is a bare @@ -766,10 +764,10 @@ def status_from_body(html: str) -> Optional[int]: def detect_page_state(html: str, status: Optional[int] = None, url: str = "") -> Tuple[str, Optional[str]]: - """`(state, detail)` -- the triage every engine shares (CLAUDE.md §1). + """`(state, detail)` -- the triage every engine shares. Signals are ordered by how much they PROVE, not by how cheap they are - (§17's classification-order trap): a rendered tile is an unambiguous + (the classification-order trap): a rendered tile is an unambiguous positive and is checked before any threshold, so a thin-but-real page cannot come back as "blocked" and send the reader hunting for a proxy problem. @@ -786,7 +784,7 @@ def detect_page_state(html: str, status: Optional[int] = None, # The second half is not tidiness. Without it a perfectly good product # page classifies as "empty", every `--mode product` run logs "page 1 # came back as empty" while returning a correct row, and the reader - # goes hunting for a block that is not there. §17: order the signals + # goes hunting for a block that is not there. Order the signals # by how much they PROVE, and a page publishing its own structured # product data proves it was served. if tile_count(html) > 0: @@ -908,7 +906,7 @@ def parse_products(html: str, base_url: str = "", *, page: int = 1, """Rows from a listing page, in the order the page publishes them. The tile is the scope, and it is the OUTERMOST ancestor covering exactly - one product (CLAUDE.md §4): `li.c-product__item` holds one `/g/g…/` id + one product: `li.c-product__item` holds one `/g/g…/` id behind two links -- the thumbnail and the title -- which is exactly the MediaMarkt shape that section warns about. Counting distinct ids rather than links is what lets that pass; here the site names the boundary for @@ -1008,7 +1006,7 @@ def ld_blocks(html: str) -> List[dict]: """Every parseable JSON-LD block on the page, flattened. A listing publishes none and a product page publishes exactly one, but - the shapes CLAUDE.md §4 lists are all legal and a site can start + the shapes listed are all legal and a site can start publishing any of them after a deploy: a list at the top level, a `@graph`, a block that does not parse at all. None of those should be an exception on a page whose products are readable. @@ -1035,7 +1033,7 @@ def _ld_offer(node: dict) -> dict: `.get("offers", {})` is not enough: an explicit `"offers": null` is legal schema.org and a default only applies to a MISSING key, so the naive form - raises `AttributeError` on a page that is otherwise fine (CLAUDE.md §4). + raises `AttributeError` on a page that is otherwise fine. """ offers = node.get("offers") if isinstance(offers, dict): @@ -1091,7 +1089,7 @@ def parse_product_detail(html: str, base_url: str = "", *, Reads the page's own JSON-LD and never a tile. That is not a style preference: a detail page carries 15 `div.c-product__item` staff-review cards and an `awoo-product-list` recommendation block, and reading tiles - here is exactly how a parser invents products (§4). + here is exactly how a parser invents products. """ soup = BeautifulSoup(html or "", "html.parser") scraped_at = utc_now() @@ -1111,8 +1109,8 @@ def parse_product_detail(html: str, base_url: str = "", *, # The DOM's own price node, read as a CROSS-CHECK rather than as an # overlay: this site has no discount chain, so there is nothing to - # reconcile (§4 says delete the overlay, do not port it). Where the two - # disagree the row keeps the structured figure -- a fact, per §4's + # reconcile (delete the overlay, do not port it). Where the two + # disagree the row keeps the structured figure -- a fact's # trustworthiness ladder -- and the disagreement is reported rather than # silently resolved. dom_node = soup.select_one(".p-product-detail__item__price") @@ -1141,7 +1139,7 @@ def parse_product_detail(html: str, base_url: str = "", *, # 20 g one; MLYE010 -> MLYE001, JETC001 -> JETC000, MTVZ003 -> MTVX001, # MWJC -> MWJA. Taking the canonical as the row's URL would have produced # a quarter of all rows whose link opens a different product while their - # title, sku and price describe this one -- CLAUDE.md §4's "every row + # title, sku and price describe this one -- the "every row # points at the wrong page while everything else looks right", which is # the shape of bug that survives every coverage check. # @@ -1186,7 +1184,7 @@ def parse_product_detail(html: str, base_url: str = "", *, def detail_price_agrees(html: str) -> Optional[bool]: """Does the page's JSON-LD price equal the one it prints? - Its own function because CLAUDE.md §4 says to CHECK this on every new + Its own function because the rule is to CHECK this on every new site rather than assume it, and because a check that is only ever run once during a capture pass is a check that stops being true. The canary asserts it. Measured 2026-09-18: agrees on 4 of 4 product captures. diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 3d2fde8..ed749ed 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -3,7 +3,7 @@ kose-scraper — pyppeteer edition ================================= The same scrape as `playwright_scraper.py`, driven by pyppeteer. Kept at -parity in behaviour rather than in priority (CLAUDE.md §6): verified on +parity in behaviour rather than in priority: verified on 2026-09-18 by running all three engines against the same two pages of `/c/c15/` — **48 of 48 rows identical** on sku, price, brand and stock. @@ -18,7 +18,7 @@ Its driver is imported at MODULE level on purpose. An engine that imports its driver inside the launch path imports cleanly with the library absent, so the offline suite never records a skip and the CI job that exists to fail on -unexpected skips cannot catch a broken import (CLAUDE.md §10). The suite +unexpected skips cannot catch a broken import. The suite asserts the module-level import with an `ast` walk, because this drifts back silently. """ @@ -72,7 +72,7 @@ # The browser locale the engines present. A constant rather than a flag: # maison.kose.co.jp is a single Japanese market — no hreflang set, no /en/ # route, every price in JPY — so a --locale would be a setting that changes -# nothing, which CLAUDE.md §3 calls out by name. Sending ja-JP is also the +# nothing, which is the failure the loader's warning exists for. Sending ja-JP is also the # honest value: claiming en-US while reading a Japanese storefront is itself # a mismatch between what the browser says and what it is doing. BROWSER_LOCALE = "ja-JP" @@ -95,7 +95,7 @@ # tile on this site carries a `/img/goods/` thumbnail, 171 of 171 in the # captures and 72 of 72 on a live run. The first version of the parser read # the tile's FIRST `<img>` and got the "NEW" badge on every row that had one -# — 100% coverage of the wrong value, which is exactly the failure §10 means +# — 100% coverage of the wrong value, which is exactly the failure a value check exists for # by "assert VALUES on real fixtures, not coverage". # The share of rows that must carry a usable price. Both `tile` and @@ -477,14 +477,14 @@ def _parse_for_mode(html: str, url: str, args, page_num: int = 1): The two entry points are separate on purpose and it is not tidiness. A product page carries 15 `div.c-product__item` staff-review cards and an `awoo-product-list` recommendation block, so a listing parser turned on - one is exactly how a parser invents products (CLAUDE.md §4). This repo's + one is exactly how a parser invents products. This repo's tile selector is element-qualified (`li.c-product__item`) and therefore sees zero of them, which `smoke_test.py` pins in both directions. `page_num` is threaded through rather than defaulted, because `position` restarts at 1 on every page: without the page number beside it, a row from page 2 claims the same position as one from page 1 and the two are - indistinguishable in the output (§18). + indistinguishable in the output. """ if args.mode == "product": rows = parse_product_detail(html, url, mode=args.mode) @@ -507,7 +507,7 @@ def _is_endpoint(url: str) -> bool: it is honest; silently dropping the branch would make the engines diverge in shape for no reason. - CLAUDE.md §21 says to ask what the front end calls before assuming a + the rule is to ask what the front end calls before assuming a browser is needed, and that was done here first rather than last: the captures were grepped for `/api/`, `/graphql`, `__NEXT_DATA__` and `__PRELOADED_STATE__` and every one came back ZERO. This is an ASP.NET @@ -536,7 +536,7 @@ def _snapshot(session, url: str): return _driver(session)["content"]() -def handle_captcha_if_present(session, args) -> bool: +def handle_captcha_if_present(session, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Same two families, same order, same "detected is not blocking" rule as @@ -568,6 +568,10 @@ def handle_captcha_if_present(session, args) -> bool: if not args.twocaptcha_key: logger.warning("No 2captcha API key, so this challenge cannot be solved.") return False + # The allowance is claimed here, immediately before the only paid + # request, so a call that detected nothing has cost nothing. + if not budget.claim(): + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -587,7 +591,7 @@ def _target_url(args) -> str: Maison KOSE server-renders its catalogue and has no JSON endpoint to prefer over the rendered page, which was CHECKED rather than assumed - (CLAUDE.md §21: ask what the front end calls before assuming a browser). + (ask what the front end calls before assuming a browser). Grepped across the captures for `/api/`, `/graphql`, `__NEXT_DATA__` and `__PRELOADED_STATE__`: zero hits on every listing and product page. @@ -595,7 +599,7 @@ def _target_url(args) -> str: it names one. An earlier version normalised `/c/c15_p10/` back to `/c/c15/`, which meant a run pointed at page 10 quietly returned page 1's products and reported success. That is this codebase's most common bug - class (§8), and the fix is simply to fetch what was asked for: §7's "page + class, and the fix is simply to fetch what was asked for: the "page 1 is fetched alone" is about the run's FIRST page deciding how far the rest can be addressed, not about the listing's first page. """ @@ -614,7 +618,7 @@ def _plan_page_urls(args, page_one_url: str, The convention is not guessed from the shape of page 1's URL: it is the one the site itself publishes in `<link rel="next">`, and `page_url` - reproduces it byte for byte on both routes. That is the check §7 demands + reproduces it byte for byte on both routes. That is the check required before trusting a constructed URL, and it matters here because getting it wrong is SILENT — `&pageno=2` and `&page=2` on a tag listing both return page 1 with HTTP 200. @@ -645,33 +649,40 @@ def page_url_start(url: str) -> int: class _SolveBudget: - """Every paid solve on one page goes through here. One counter, checked - BEFORE each purchase rather than beside one of them. + """The per-page allowance for paid solves. One counter, claimed + IMMEDIATELY BEFORE the paid request is sent, and nowhere else. - This exists because the budget was enforced at ONE of the two call sites. `handle_captcha_if_present` runs twice per attempt — once before the page is classified, so a challenge is cleared before anything is judged, and - once after, for the state that says the page really is gated — and only - the second call was counted. The uncounted one repeats on every - block-retry, so the constant read like a limit and was advisory. - - Nothing about that is visible on a site where a challenge is rare, which - is exactly this one: no challenge of any vendor has ever been observed on - maison.kose.co.jp, so this has never cost anything here. It is carried - because the day it does cost something is the day nobody is watching — - CLAUDE.md §17's "a policy constant nothing reads", with a bill attached, - and §23 measured one page buying three solves on a sibling. - - The budget deliberately does NOT reset on rotation: a fresh exit is a - reason to re-fetch, not a fresh allowance to spend. + once after, for the state that says the page really is gated. Both calls + take this object, and both claim from it only at the point where a task + would really be submitted. + + Two earlier versions were each wrong in one direction. The first counted + only the second call site, so the first repeated on every block-retry and + one page could buy a solve per retry against a limit of one. The second + counted every CALL, purchase or not: the pre-classification call, which + on a normal page detects nothing and buys nothing, used up the whole + allowance, so a challenge first seen after classification (a widget + rendered late, or a page that only turns into a challenge after the + readiness wait) was never offered to the solver and the page was + reported as blocked. + + A claim that is then refused or fails at the solver still counts: it was + a request sent, and possibly billed, and not counting it would reopen + the overspend. Detecting nothing, finding the content already rendered + or having no API key claims nothing. + + The allowance deliberately does NOT reset on retry or rotation: a fresh + exit is a reason to re-fetch, not a fresh allowance to spend. """ def __init__(self, limit: int): self.limit = limit self.spent = 0 - def spend(self, solve) -> bool: - """Run `solve()` if the budget allows, and count it. False if not.""" + def claim(self) -> bool: + """Count one purchase if the allowance permits it. False if not.""" if self.spent >= self.limit: logger.info("Not attempting another solve on this page: %d of %d " "already bought. A challenge that survives a solved " @@ -680,7 +691,7 @@ def spend(self, solve) -> bool: self.spent, self.limit) return False self.spent += 1 - return bool(solve()) + return True def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome: @@ -722,7 +733,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # cannot be 0: with 0 this line announced a fetch the loop then never # made, so the log itself corroborated the wrong conclusion. logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) - load_failed = False + load_failed, exit_failed = False, None for attempt in range(1, args.retries + 1): try: response = bridge.run(page.goto(url, {"waitUntil": "domcontentloaded", @@ -744,6 +755,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if any(marker in text for marker in _PROXY_ERROR_MARKERS): logger.warning("Exit %s is unusable (%s).", mask(pool.current) if pool else "(none)", text[:120]) + exit_failed = text[:120] break if attempt < args.retries: pause = args.retry_delay * (2 ** (attempt - 1)) @@ -752,8 +764,14 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome args.retries, text[:120], pause) time.sleep(pause) - if load_failed and block_attempt < block_retries: - pool.advance("unusable exit or repeated load failure") + # Rotate only for an unusable EXIT, and only when there is another + # exit to rotate to — the same rule as the Playwright and Selenium + # engines. With no --proxy, `pool` is None, and this branch used to + # call `pool.advance` on it after any page that failed to load + # `--retries` times: AttributeError, exit 1, instead of a load + # failure. A repeated timeout is not a reason to change exits. + if exit_failed and has_pool and block_attempt < block_retries: + pool.advance(f"unusable exit: {exit_failed}") session.relaunch() bridge, page = session.bridge, session.page d = _driver(session) @@ -765,7 +783,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Routed through the budget like every other purchase. This # call used to be unguarded, which is how one page could buy # a solve per block-retry against a limit of one. - if budget.spend(lambda: handle_captcha_if_present(session, args)): + if handle_captcha_if_present(session, args, budget): time.sleep(1) html = _snapshot(session, url) or "" @@ -792,17 +810,16 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome html = _snapshot(session, url) or html state = page_flow.classify(html, http_status, page.url) - # The paid path is reached only for state "captcha" — a rendered - # Managed Challenge, which IS a test. It is NOT reached for - # "blocked": that page carries no widget, so a solve there would be a - # charge for nothing. Mirrors the other two engines. + # Not reached for "blocked": a hard refusal carries no widget, so a + # solve there would be a charge for nothing. Mirrors the other two + # engines. # # The paid path is reached only for state "challenge", which no # capture of this site has ever produced. Wired up because a bot # manager can be switched on between deploys, and bounded by # SOLVES_PER_PAGE so a speculative path cannot become a bill. if page_flow.should_solve(state): - if budget.spend(lambda: handle_captcha_if_present(session, args)): + if handle_captcha_if_present(session, args, budget): time.sleep(1) html = _content(session) or html state = page_flow.classify(html, http_status, page.url) @@ -838,7 +855,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # not: `mask(pool.current)` on a None pool raised # AttributeError and the run exited 1 — a CRASH reported for # a page that was merely blocked, which destroys the - # blocked/empty/partial distinction §8 calls an invariant. + # blocked/empty/partial distinction the exit codes exist for. # Reproduced by forcing the blocked state with no --proxy. pause = args.retry_delay * (block_attempt + 1) logger.warning("Page %d came back as %s and there is no proxy " @@ -929,7 +946,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Said once per run rather than per page. `pages_available` is # None on a TAG listing, which publishes no counter at all, and # None means UNKNOWN: the run then walks until a page adds no new - # sku instead of planning (§7 layer 3). + # sku instead of planning. logger.info("Page 1: %s tile(s), %s product-shaped link(s), the " "site reports %s. Canonical: %s", listing.tiles_on_page, listing.product_links_on_page, @@ -938,7 +955,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome else "no page counter (a tag listing publishes none)", listing.canonical or "none") - # §20: tell a BROKEN PARSER apart from an EMPTY CATEGORY before anything + # Tell a BROKEN PARSER apart from an EMPTY CATEGORY before anything # downstream reports "0 products" and sends the reader to check the URL. if not products and page_flow.looks_like_a_parse_failure( state, len(products), product_link_count(html or "")): @@ -1062,7 +1079,7 @@ def scrape(args) -> int: if target != args.url: # The only normalisation this site needs: a listing URL that # already names a page is reset to page 1, because page 1 is what - # decides how far the rest can be addressed (§7). + # decides how far the rest can be addressed. logger.info("Normalised to page 1: %s", target) first = _fetch_one_page(session, args, pool, 1, target) @@ -1085,7 +1102,7 @@ def scrape(args) -> int: "404 means the PATH is wrong, not the id.", first.url) elif first.state == "parse_failed": # Served, linked to products, parsed to nothing: OUR bug, and it - # must not reach the sidecar as a complete run (§20). + # must not reach the sidecar as a complete run. stop_reason = "parser_found_nothing" elif args.mode != "product": seen_keys.update(p.sku for p in first.products if p.sku is not None) @@ -1191,7 +1208,7 @@ def scrape(args) -> int: "route": route_of(final_url or args.url), "tags": tags_from_url(final_url or args.url)} # No `capped_by_site` / `reachable_max`: this site imposes no page - # cap (measured — /c/c15/ states 13 pages, 12 x 24 + 16 = 304), so + # cap (measured 2026-09-18: /c/c15/ stated 13 pages, 12 x 24 + 16), so # `pages_available` already says everything they would, and # pages x page_size would OVERSTATE a short last page. @@ -1257,7 +1274,7 @@ def parse_args(): "/site/cosmedecorte/c/c15/ and /site/xyz/c/c15/ are " "the same page. Rows otherwise carry the cNN the URL " "asked for.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.at_least_one, default=1, help="Number of listing pages to fetch. Applies to --mode " "listing; ignored in --mode product. A category " "page prints the site's own page counter, so a run " @@ -1267,8 +1284,8 @@ def parse_args(): "on this site, so a request is limited only by what " "the category holds, and the sidecar records both " "numbers.") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--delay", type=page_flow.non_negative_seconds, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.at_least_one, default=1, metavar="N", help="Fetch pages through N parallel workers (default 1 — " "unchanged sequential behaviour). Each worker runs its " "own browser and holds its own proxy exit, so N>1 " @@ -1276,7 +1293,7 @@ def parse_args(): "from one address. Ignored with --cdp-endpoint.") p.add_argument("--retries", type=page_flow.attempts, default=3, help=page_flow.RETRIES_HELP) - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", type=page_flow.non_negative_seconds, default=2.0, help="Seconds before the first page-load retry, doubling " "thereafter (default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") @@ -1295,7 +1312,7 @@ def parse_args(): p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup, so concurrent runs do not " "all begin on the first exit in the file.") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", type=page_flow.non_negative_int, default=2, help="When a page comes back refused or behind a captcha, " "retry it from this many OTHER exits before giving up " "(default 2). Needs a pool of more than one; ignored " @@ -1421,7 +1438,7 @@ def parse_args(): # Refused rather than attempted. This parser reads Maison KOSE's own # tile markup and its /c/cNN/ and /g/gSKU/ URL shapes; pointing it at # another site would not fail loudly, it would return zero rows and - # look like an empty result (§8). The REASON is given, because "is + # look like an empty result. The REASON is given, because "is # not a Kose site" about kose.co.jp — which plainly is one — sends # the reader hunting a typo they did not make. p.error(f"{args.url!r} {why}.") diff --git a/pyproject.toml b/pyproject.toml index c2d3d39..39e7bd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ license-files = ["LICENSE"] # CONTRIBUTING.md for why) — tests/test_smoke.py wraps it as a single pytest # test so `pytest` works as an entry point without maintaining two copies of # the same checks. Deliberately no COUNT here: an inherited number in a -# comment nobody re-measures is CLAUDE.md §13's trap, and this one arrived +# comment nobody re-measures is the trap, and this one arrived # from a sibling repo already wrong. `python3 smoke_test.py` prints it. Scoped to tests/ so pytest doesn't also try, and fail, # to collect the engine scripts and product_parser.py as test modules. testpaths = ["tests"] diff --git a/scraper_api_client.py b/scraper_api_client.py index 56977aa..0f146a5 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -7,7 +7,7 @@ running. The same parser reads the HTML it returns, so the rows are the rows the browser engines produce. -Say it plainly rather than selling it (§13). **This site served an ordinary +Say it plainly rather than selling it. **This site served an ordinary datacentre address on every route measured** (2026-09-18, from netcup in Nuremberg): a plain `curl`, a `python-requests` UA and a request with no User-Agent header each got the same 90,310-byte listing, and twelve pages @@ -356,7 +356,7 @@ def parse_args(): "flag of the same name counts ATTEMPTS. 0 is " "meaningful here and means one attempt. Each attempt " "is a separate billable task, so this defaults to 1.") - p.add_argument("--retry-delay", type=int, default=10, + p.add_argument("--retry-delay", type=page_flow.non_negative_int, default=10, help="Seconds between retries (default 10)") p.add_argument("--dump-html", default=None, help="Also write the raw returned HTML to this path (always, even on success)") diff --git a/selenium_scraper.py b/selenium_scraper.py index 2438e5a..786dfd0 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -4,8 +4,7 @@ ================================ The same scrape as `playwright_scraper.py`, driven by Selenium. Kept at parity in behaviour rather than in priority: all three engines must agree on -exit codes, run status, and whether a run crashes or spends money (CLAUDE.md -§6). Verified on 2026-09-18 by running all three against the same two pages +exit codes, run status, and whether a run crashes or spends money. Verified on 2026-09-18 by running all three against the same two pages of `/c/c15/` — **48 of 48 rows identical** on sku, price, brand and stock. Read `playwright_scraper.py`'s docstring for what this site is and what is @@ -70,7 +69,7 @@ # The browser locale the engines present. A constant rather than a flag: # maison.kose.co.jp is a single Japanese market — no hreflang set, no /en/ # route, every price in JPY — so a --locale would be a setting that changes -# nothing, which CLAUDE.md §3 calls out by name. Sending ja-JP is also the +# nothing, which is the failure the loader's warning exists for. Sending ja-JP is also the # honest value: claiming en-US while reading a Japanese storefront is itself # a mismatch between what the browser says and what it is doing. BROWSER_LOCALE = "ja-JP" @@ -93,7 +92,7 @@ # tile on this site carries a `/img/goods/` thumbnail, 171 of 171 in the # captures and 72 of 72 on a live run. The first version of the parser read # the tile's FIRST `<img>` and got the "NEW" badge on every row that had one -# — 100% coverage of the wrong value, which is exactly the failure §10 means +# — 100% coverage of the wrong value, which is exactly the failure a value check exists for # by "assert VALUES on real fixtures, not coverage". # The share of rows that must carry a usable price. Both `tile` and @@ -370,14 +369,14 @@ def _parse_for_mode(html: str, url: str, args, page_num: int = 1): The two entry points are separate on purpose and it is not tidiness. A product page carries 15 `div.c-product__item` staff-review cards and an `awoo-product-list` recommendation block, so a listing parser turned on - one is exactly how a parser invents products (CLAUDE.md §4). This repo's + one is exactly how a parser invents products. This repo's tile selector is element-qualified (`li.c-product__item`) and therefore sees zero of them, which `smoke_test.py` pins in both directions. `page_num` is threaded through rather than defaulted, because `position` restarts at 1 on every page: without the page number beside it, a row from page 2 claims the same position as one from page 1 and the two are - indistinguishable in the output (§18). + indistinguishable in the output. """ if args.mode == "product": rows = parse_product_detail(html, url, mode=args.mode) @@ -400,7 +399,7 @@ def _is_endpoint(url: str) -> bool: it is honest; silently dropping the branch would make the engines diverge in shape for no reason. - CLAUDE.md §21 says to ask what the front end calls before assuming a + the rule is to ask what the front end calls before assuming a browser is needed, and that was done here first rather than last: the captures were grepped for `/api/`, `/graphql`, `__NEXT_DATA__` and `__PRELOADED_STATE__` and every one came back ZERO. This is an ASP.NET @@ -434,7 +433,7 @@ def _snapshot(session, url: str): return _driver(session)["content"]() -def handle_captcha_if_present(session, args) -> bool: +def handle_captcha_if_present(session, args, budget) -> bool: """Detect and solve a challenge. True if something was solved. Same detectors, same reconciliation and the same "detected is not @@ -472,6 +471,10 @@ def handle_captcha_if_present(session, args) -> bool: if not args.twocaptcha_key: logger.warning("No 2captcha API key, so this challenge cannot be solved.") return False + # The allowance is claimed here, immediately before the only paid + # request, so a call that detected nothing has cost nothing. + if not budget.claim(): + return False try: token = solve_recaptcha(challenge, args.twocaptcha_key, api_version=args.captcha_api, @@ -495,7 +498,7 @@ def _target_url(args) -> str: Maison KOSE server-renders its catalogue and has no JSON endpoint to prefer over the rendered page, which was CHECKED rather than assumed - (CLAUDE.md §21: ask what the front end calls before assuming a browser). + (ask what the front end calls before assuming a browser). Grepped across the captures for `/api/`, `/graphql`, `__NEXT_DATA__` and `__PRELOADED_STATE__`: zero hits on every listing and product page. @@ -503,7 +506,7 @@ def _target_url(args) -> str: it names one. An earlier version normalised `/c/c15_p10/` back to `/c/c15/`, which meant a run pointed at page 10 quietly returned page 1's products and reported success. That is this codebase's most common bug - class (§8), and the fix is simply to fetch what was asked for: §7's "page + class, and the fix is simply to fetch what was asked for: the "page 1 is fetched alone" is about the run's FIRST page deciding how far the rest can be addressed, not about the listing's first page. """ @@ -522,7 +525,7 @@ def _plan_page_urls(args, page_one_url: str, The convention is not guessed from the shape of page 1's URL: it is the one the site itself publishes in `<link rel="next">`, and `page_url` - reproduces it byte for byte on both routes. That is the check §7 demands + reproduces it byte for byte on both routes. That is the check required before trusting a constructed URL, and it matters here because getting it wrong is SILENT — `&pageno=2` and `&page=2` on a tag listing both return page 1 with HTTP 200. @@ -553,33 +556,40 @@ def page_url_start(url: str) -> int: class _SolveBudget: - """Every paid solve on one page goes through here. One counter, checked - BEFORE each purchase rather than beside one of them. + """The per-page allowance for paid solves. One counter, claimed + IMMEDIATELY BEFORE the paid request is sent, and nowhere else. - This exists because the budget was enforced at ONE of the two call sites. `handle_captcha_if_present` runs twice per attempt — once before the page is classified, so a challenge is cleared before anything is judged, and - once after, for the state that says the page really is gated — and only - the second call was counted. The uncounted one repeats on every - block-retry, so the constant read like a limit and was advisory. - - Nothing about that is visible on a site where a challenge is rare, which - is exactly this one: no challenge of any vendor has ever been observed on - maison.kose.co.jp, so this has never cost anything here. It is carried - because the day it does cost something is the day nobody is watching — - CLAUDE.md §17's "a policy constant nothing reads", with a bill attached, - and §23 measured one page buying three solves on a sibling. - - The budget deliberately does NOT reset on rotation: a fresh exit is a - reason to re-fetch, not a fresh allowance to spend. + once after, for the state that says the page really is gated. Both calls + take this object, and both claim from it only at the point where a task + would really be submitted. + + Two earlier versions were each wrong in one direction. The first counted + only the second call site, so the first repeated on every block-retry and + one page could buy a solve per retry against a limit of one. The second + counted every CALL, purchase or not: the pre-classification call, which + on a normal page detects nothing and buys nothing, used up the whole + allowance, so a challenge first seen after classification (a widget + rendered late, or a page that only turns into a challenge after the + readiness wait) was never offered to the solver and the page was + reported as blocked. + + A claim that is then refused or fails at the solver still counts: it was + a request sent, and possibly billed, and not counting it would reopen + the overspend. Detecting nothing, finding the content already rendered + or having no API key claims nothing. + + The allowance deliberately does NOT reset on retry or rotation: a fresh + exit is a reason to re-fetch, not a fresh allowance to spend. """ def __init__(self, limit: int): self.limit = limit self.spent = 0 - def spend(self, solve) -> bool: - """Run `solve()` if the budget allows, and count it. False if not.""" + def claim(self) -> bool: + """Count one purchase if the allowance permits it. False if not.""" if self.spent >= self.limit: logger.info("Not attempting another solve on this page: %d of %d " "already bought. A challenge that survives a solved " @@ -588,7 +598,7 @@ def spend(self, solve) -> bool: self.spent, self.limit) return False self.spent += 1 - return bool(solve()) + return True def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome: @@ -622,7 +632,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # oversight: `driver.get()` returns None and WebDriver exposes no HTTP # status at all, so this engine cannot thread one however carefully its # twins do. Rather than leave it worse informed than they are — the - # drift a shared module exists to prevent (§1) — the classifier falls + # drift a shared module exists to prevent — the classifier falls # back to `product_parser.status_from_body`, which reads the status this # site states in its own error page (`<title>404- …`). Measured # 2026-09-18: that is what makes all three engines agree on a 404 rather @@ -668,7 +678,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Routed through the budget like every other purchase. This # call used to be unguarded, which is how one page could buy # a solve per block-retry against a limit of one. - if budget.spend(lambda: handle_captcha_if_present(session, args)): + if handle_captcha_if_present(session, args, budget): time.sleep(1) html = _snapshot(session, url) or "" @@ -696,13 +706,13 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome html = _snapshot(session, url) or html state = page_flow.classify(html, None, d["current_url"]()) - # The paid path is reached only for state "captcha" — a rendered - # Managed Challenge, which IS a test. It is NOT reached for - # "blocked": the hard "You have been blocked" page carries no widget - # and no sitekey, so a solve there would be a charge for nothing. + # The paid path is reached only for state "challenge" — a rendered + # widget, which IS a test. It is NOT reached for "blocked": a hard + # refusal carries no widget and no sitekey, so a solve there would be + # a charge for nothing. # Bounded by SOLVES_PER_PAGE. Mirrors playwright_scraper. if page_flow.should_solve(state): - if budget.spend(lambda: handle_captcha_if_present(session, args)): + if handle_captcha_if_present(session, args, budget): time.sleep(1) html = d["content"]() or html state = page_flow.classify(html, url=d["current_url"]()) @@ -738,7 +748,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # not: `mask(pool.current)` on a None pool raised # AttributeError and the run exited 1 — a CRASH reported for # a page that was merely blocked, which destroys the - # blocked/empty/partial distinction §8 calls an invariant. + # blocked/empty/partial distinction the exit codes exist for. # Reproduced by forcing the blocked state with no --proxy. pause = args.retry_delay * (block_attempt + 1) logger.warning("Page %d came back as %s and there is no proxy " @@ -834,7 +844,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Said once per run rather than per page. `pages_available` is # None on a TAG listing, which publishes no counter at all, and # None means UNKNOWN: the run then walks until a page adds no new - # sku instead of planning (§7 layer 3). + # sku instead of planning. logger.info("Page 1: %s tile(s), %s product-shaped link(s), the " "site reports %s. Canonical: %s", listing.tiles_on_page, listing.product_links_on_page, @@ -843,7 +853,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome else "no page counter (a tag listing publishes none)", listing.canonical or "none") - # §20: tell a BROKEN PARSER apart from an EMPTY CATEGORY before anything + # Tell a BROKEN PARSER apart from an EMPTY CATEGORY before anything # downstream reports "0 products" and sends the reader to check the URL. if not products and page_flow.looks_like_a_parse_failure( state, len(products), product_link_count(html or "")): @@ -956,7 +966,7 @@ def scrape(args) -> int: if target != args.url: # The only normalisation this site needs: a listing URL that # already names a page is reset to page 1, because page 1 is what - # decides how far the rest can be addressed (§7). + # decides how far the rest can be addressed. logger.info("Normalised to page 1: %s", target) first = _fetch_one_page(session, args, pool, 1, target) @@ -979,7 +989,7 @@ def scrape(args) -> int: "404 means the PATH is wrong, not the id.", first.url) elif first.state == "parse_failed": # Served, linked to products, parsed to nothing: OUR bug, and it - # must not reach the sidecar as a complete run (§20). + # must not reach the sidecar as a complete run. stop_reason = "parser_found_nothing" elif args.mode != "product": seen_keys.update(p.sku for p in first.products if p.sku is not None) @@ -1088,7 +1098,7 @@ def scrape(args) -> int: "route": route_of(final_url or args.url), "tags": tags_from_url(final_url or args.url)} # No `capped_by_site` / `reachable_max`: this site imposes no page - # cap (measured — /c/c15/ states 13 pages, 12 x 24 + 16 = 304), so + # cap (measured 2026-09-18: /c/c15/ stated 13 pages, 12 x 24 + 16), so # `pages_available` already says everything they would, and # pages x page_size would OVERSTATE a short last page. @@ -1154,7 +1164,7 @@ def parse_args(): "/site/cosmedecorte/c/c15/ and /site/xyz/c/c15/ are " "the same page. Rows otherwise carry the cNN the URL " "asked for.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.at_least_one, default=1, help="Number of listing pages to fetch. Applies to --mode " "listing; ignored in --mode product. A category " "page prints the site's own page counter, so a run " @@ -1164,8 +1174,8 @@ def parse_args(): "on this site, so a request is limited only by what " "the category holds, and the sidecar records both " "numbers.") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--delay", type=page_flow.non_negative_seconds, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.at_least_one, default=1, metavar="N", help="Fetch pages through N parallel workers (default 1 — " "unchanged sequential behaviour). Each worker runs its " "own browser and holds its own proxy exit, so N>1 " @@ -1173,7 +1183,7 @@ def parse_args(): "from one address. Ignored with --cdp-endpoint.") p.add_argument("--retries", type=page_flow.attempts, default=3, help=page_flow.RETRIES_HELP) - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", type=page_flow.non_negative_seconds, default=2.0, help="Seconds before the first page-load retry, doubling " "thereafter (default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") @@ -1192,7 +1202,7 @@ def parse_args(): p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup, so concurrent runs do not " "all begin on the first exit in the file.") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", type=page_flow.non_negative_int, default=2, help="When a page comes back refused or behind a captcha, " "retry it from this many OTHER exits before giving up " "(default 2). Needs a pool of more than one; ignored " @@ -1313,7 +1323,7 @@ def parse_args(): # Refused rather than attempted. This parser reads Maison KOSE's own # tile markup and its /c/cNN/ and /g/gSKU/ URL shapes; pointing it at # another site would not fail loudly, it would return zero rows and - # look like an empty result (§8). The REASON is given, because "is + # look like an empty result. The REASON is given, because "is # not a Kose site" about kose.co.jp — which plainly is one — sends # the reader hunting a typo they did not make. p.error(f"{args.url!r} {why}.") diff --git a/smoke_test.py b/smoke_test.py index 2ad164e..b4b2278 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -19,7 +19,7 @@ ------------ Every one is cut VERBATIM from a real capture taken 2026-09-18 and trimmed to the parts the parser reads. Each was verified to parse to IDENTICAL values to -its untrimmed original before being committed (§15 step 3): the trimming +its untrimmed original before being committed: the trimming script asserted field-by-field equality on every row, not merely that the row count matched. @@ -50,7 +50,7 @@ product-shaped link, which is why the parse-failure threshold is two. An earlier version of this fixture was written by hand with a 404 title on it, and was - wrong about the one thing it existed to show (§21: a + wrong about the one thing it existed to show (a guard is only as good as the fixture it runs against). HTTP404_HTML the site's REAL 404, verbatim from @@ -62,13 +62,13 @@ `<title>maison.kose.co.jp` — the site's own hostname — and no vendor marker at all, which is why positive-asset detection is the only thing that - classifies it correctly (§18). + classifies it correctly. No personal data appears in any of them. Maison KOSÉ does publish staff opinion pieces under real first names, and the one staff card in PRODUCT_HTML is there because the PARSER must ignore it — so the author's name and the review text are replaced with obvious placeholders while the -markup the site generates around them is left untouched (§10). +markup the site generates around them is left untouched. `check_fixtures_carry_no_session_material` guards the next capture by PATTERN rather than by these literals. """ @@ -109,10 +109,10 @@ def check(name, condition, detail=""): if condition: PASSED += 1 if VERBOSE: - print(" ok %s%s" % (name, (" — " + detail) if detail else "")) + print(" ok %s%s" % (name, (" — " + str(detail)) if detail else "")) else: - FAILURES.append("%s%s" % (name, (" — " + detail) if detail else "")) - print(" FAIL %s%s" % (name, (" — " + detail) if detail else "")) + FAILURES.append("%s%s" % (name, (" — " + str(detail)) if detail else "")) + print(" FAIL %s%s" % (name, (" — " + str(detail)) if detail else "")) def equal(name, got, want): @@ -755,9 +755,9 @@ def skip(group, reason): "puppeteer_scraper": "pyppeteer", } -# The family's flag contract (CLAUDE.md §9), plus the five it omitted for +# The family's flag contract, plus the five it omitted for # months while nearly every repo shipped them. Re-derive rather than trusting -# this comment (§13): +# this comment: # # grep -ohE '"--[a-z0-9-]+"' */playwright_scraper.py | sort | uniq -c | sort -rn CONTRACT_FLAGS = { @@ -772,7 +772,7 @@ def skip(group, reason): # `--locale` is in 16 of the 18 repos counted and is deliberately ABSENT # here, which is a measurement rather than an oversight: maison.kose.co.jp # is one market — no hreflang set, no /en/ route, JPY on every row — so the -# flag would be a setting that changes nothing, which §3 calls out by name. +# flag would be a setting that changes nothing, which is exactly what to avoid. # # `--sort` is absent for the same reason: the site offers no ordering control # on either listing route. That is the opposite of bbb-scraper, where the @@ -788,7 +788,7 @@ def skip(group, reason): BANNED_FLAGS = ("--antidetect", "--country-code", "--country", "--locale", "--sort") # Assembled from pieces rather than written out, so that this file can be -# SCANNED for them too (§22): three repos in the family exempted +# SCANNED for them too: three repos in the family exempted # `smoke_test.py` wholesale, which made the file most likely to acquire a # stray phrase the one file nobody checked. BANNED_WORDING = ( @@ -804,7 +804,7 @@ def skip(group, reason): # --------------------------------------------------------------------------- -# The parser, against real captures. Assert VALUES, not coverage (§10). +# The parser, against real captures. Assert VALUES, not coverage. # --------------------------------------------------------------------------- def check_listing_parses_with_the_values_the_page_shows(): @@ -838,8 +838,8 @@ def check_the_image_is_the_product_and_not_the_badge(): The first version of this parser read it and got `/freepage/…/badge/new.png` on every tile that had one — 100% coverage of - the WRONG value, which is exactly what §10 means by asserting values - rather than coverage. Every row here must point at `/img/goods/`. + the WRONG value, which is why values are asserted rather + than coverage. Every row here must point at `/img/goods/`. """ rows = product_parser.parse_products(LISTING_HTML, LISTING_URL) bad = [r.sku for r in rows if r.image_url and "/badge/" in r.image_url] @@ -852,7 +852,7 @@ def check_the_image_is_the_product_and_not_the_badge(): def check_stock_is_an_allowlist_and_takes_both_values(): - """§20: a column you never saw take its other value is not verified.""" + """A column you never saw take its other value is not verified.""" rows = product_parser.parse_products(LISTING_HTML, LISTING_URL) equal("in_stock across the fixture", [r.in_stock for r in rows], [False, False, False, True]) @@ -900,7 +900,7 @@ def check_a_tag_listing_mixes_brands_and_is_read_per_tile(): def check_a_product_page_is_not_a_listing(): - """The `li.` in the tile selector is the whole guard (§4). + """The `li.` in the tile selector is the whole guard. A product page reuses `c-product__item` on `
`s for staff-review cards. A class-only selector would emit one row per card — a product @@ -936,7 +936,7 @@ def check_product_detail_reads_the_pages_own_jsonld(): def check_the_structured_and_displayed_prices_agree(): - """§4 says CHECK this on every site rather than assume it.""" + """CHECK this on every site rather than assume it.""" equal("they agree on the captured product", product_parser.detail_price_agrees(PRODUCT_HTML), True) broken = PRODUCT_HTML.replace("5,940円", "9,999円") @@ -1072,7 +1072,7 @@ def check_pages_are_planned_from_the_counter(): Against `page_flow.pages_to_plan`, which is the function the ENGINES call. An earlier version of this check exercised a `product_parser` helper that the engines had stopped using, so it passed while saying - nothing about what a run would do (§17's check #5 found it). + nothing about what a run would do (the check #5 found it). """ equal("50 asked, 13 available", list(page_flow.pages_to_plan(50, 13)), list(range(1, 14))) @@ -1151,7 +1151,7 @@ def check_price_parsing(): # --------------------------------------------------------------------------- def check_no_marker_fires_on_a_page_the_site_serves(): - """§18: count every candidate on a page you KNOW is good, first.""" + """Count every candidate on a page you KNOW is good, first.""" good = [("listing", LISTING_HTML), ("tags", TAGS_HTML), ("product", PRODUCT_HTML), ("empty", EMPTY_HTML), ("404", NOTFOUND_HTML)] @@ -1161,7 +1161,7 @@ def check_no_marker_fires_on_a_page_the_site_serves(): def check_cf_turnstile_is_not_carried_as_a_marker(): - """§19: 2Captcha's own Scraping Browser extension injects `cf-turnstile` + """2Captcha's own Scraping Browser extension injects `cf-turnstile` into every page it loads, so it fires on GOOD pages and misses real ones. `challenges.cloudflare.com` is the one that works.""" markers = [m for m, _ in product_parser.BOT_CHALLENGE_MARKERS] @@ -1187,7 +1187,7 @@ def check_a_marker_would_still_fire_if_the_site_turned_one_on(): def check_a_marker_survives_both_encodings(): - """§20: an edge can entity-escape the punctuation in its own marker, so a + """An edge can entity-escape the punctuation in its own marker, so a literal matches a browser's DOM and misses a raw HTTP response.""" escaped = ("Access Denied" "' '
    ' @@ -1248,11 +1248,11 @@ def check_the_404_carries_exactly_one_product_link(): # --------------------------------------------------------------------------- -# Structural checks the whole family shares (§10, §17, §22). +# Structural checks the whole family shares. # --------------------------------------------------------------------------- def check_shared_calls_bind_against_the_real_signature(): - """§17's check #1, and the one that earns its keep. + """the check #1, and the one that earns its keep. A sibling repo shipped `classify(html, url=…)` in two of three engines against a callee taking `status` second, and BOTH crashed on their first @@ -1339,7 +1339,7 @@ def check_shared_calls_bind_against_the_real_signature(): def check_credential_scan_is_one_implementation_invoked_from_both(): - """§17: two sources of truth, one dead and one holed. + """Two sources of truth, one dead and one holed. `.github/ci_checks.py` sat in three repos invoked by NOTHING, while tests.yml carried an inline grep doing a narrower version of the same job @@ -1363,7 +1363,7 @@ def check_credential_scan_is_one_implementation_invoked_from_both(): def check_ci_calls_the_shared_checks_rather_than_restating_them(): - """§17: one implementation, invoked from both — asserted, not assumed. + """One implementation, invoked from both — asserted, not assumed. This repo's FIRST CI run failed on exactly this. `tests.yml` carried INLINE reimplementations of the `--help` and sample-output checks that @@ -1374,7 +1374,7 @@ def check_ci_calls_the_shared_checks_rather_than_restating_them(): nothing in the repo could see the difference. The guard triggers on the whole `.github` directory being absent, never - on a file inside it being missing (§22): two suites in this family run + on a file inside it being missing: two suites in this family run INSIDE the Docker image, which deliberately COPYs no `.github/`, so a check that reads a workflow file is correct in the repo and red in the image. A check that quietly starts passing once its input disappears is @@ -1417,7 +1417,7 @@ def check_ci_calls_the_shared_checks_rather_than_restating_them(): def check_banned_wording(): - """§12: enforced by this test rather than by review.""" + """Enforced by this test rather than by review.""" for root, dirs, files in os.walk(HERE): dirs[:] = [d for d in dirs if d not in (".git", "__pycache__", ".pytest_cache", "node_modules")] @@ -1434,6 +1434,50 @@ def check_banned_wording(): check("banned-wording scan ran", True) +# The project's notes outside this repository are private: a pointer to them +# is a dead reference for every reader. Assembled from pieces so this file +# can be scanned too. +PRIVATE_CITATIONS = ( + "CLAUDE" + ".md", + "claude_" + "2scraper", + "2scraper_" + "repo_rules", + "play" + "book", + "template " + "\u00a7", + "\u00a7", # a bare section sign: a numbered private rule +) + + +def check_no_citations_of_private_notes(): + """Shipped files state each rule in their own words. + + They used to cite a private file by name and by section number — in the + README, in `--help` output, in docstrings and in release notes — which + reads as a pointer into a document no reader can open. + """ + names = (".gitignore", ".dockerignore", "Dockerfile", ".env.example") + scanned = 0 + for root, dirs, files in os.walk(HERE): + dirs[:] = [d for d in dirs + if d not in (".git", "__pycache__", ".pytest_cache", + "node_modules") + and not os.path.exists(os.path.join(root, d, "pyvenv.cfg"))] + for filename in files: + if not (filename.endswith((".py", ".md", ".yml", ".yaml", ".txt", + ".toml", ".cfg", ".html", ".example")) + or filename in names): + continue + path = os.path.join(root, filename) + text = open(path, encoding="utf-8", errors="replace").read() + scanned += 1 + for token in PRIVATE_CITATIONS: + if token.lower() in text.lower(): + line = next(i for i, l in enumerate(text.splitlines(), 1) + if token.lower() in l.lower()) + check("%s:%d cites private notes (%r)" % ( + os.path.relpath(path, HERE), line, token), False) + check("the private-citation scan read files", scanned > 10, str(scanned)) + + def check_worker_pools_start_on_different_exits(): engine = _import_engine("playwright_scraper") if engine is None: @@ -1447,7 +1491,7 @@ def check_worker_pools_start_on_different_exits(): def check_fingerprint_kwargs_are_ones_the_driver_accepts(): - """§10: an unknown key in new_context(**kwargs) is a TypeError at launch, + """An unknown key in new_context(**kwargs) is a TypeError at launch, on the PAID path, at runtime.""" engine = _import_engine("playwright_scraper") if engine is None: @@ -1472,7 +1516,7 @@ def check_fingerprint_kwargs_are_ones_the_driver_accepts(): def check_engines_do_not_evaluate_a_string_in_the_browser(): - """§18: a site whose CSP omits `unsafe-eval` kills wait_for_function with + """A site whose CSP omits `unsafe-eval` kills wait_for_function with an EvalError and takes the run down with exit 1. Maison KOSÉ has not been measured for that, and the cheap habit costs nothing where it would have been allowed.""" @@ -1490,7 +1534,7 @@ def check_engines_do_not_evaluate_a_string_in_the_browser(): def check_credentials_never_reach_a_log(): - """§8: an EXCEPTION MESSAGE is a log, and the masker must be GLOBAL. + """An EXCEPTION MESSAGE is a log, and the masker must be GLOBAL. A Playwright connection error repeats the endpoint five times (the message plus a four-line call log), so a masker handling only the first @@ -1575,7 +1619,7 @@ def check_a_run_that_finds_nothing_writes_nothing(): code = save([], prefix, "json", allow_empty=True) equal("--allow-empty WRITES the empty file...", json.load(open(prefix + ".json", encoding="utf-8")), []) - # ...and still reports exit 4. Pinned deliberately (§10: pin a known + # ...and still reports exit 4. Pinned deliberately (pin a known # behaviour rather than half-guarding it): "zero products" is true # whether or not the file was written, and a caller that wanted the # file still wants to know the result was empty. @@ -1583,7 +1627,7 @@ def check_a_run_that_finds_nothing_writes_nothing(): def check_concurrency_with_the_browser_stubbed(): - """§10: a live run cannot always reach this machinery. + """A live run cannot always reach this machinery. Page 1 is fetched alone and decides how many pages there are, so a blocked page 1 means the workers never start. Driven directly instead, @@ -1753,7 +1797,7 @@ def check_no_statement_is_unreachable(): def check_undefined_names_in_every_module(): - """§10: compileall proves a file PARSES, not that its names RESOLVE. + """Compileall proves a file PARSES, not that its names RESOLVE. A live run of a sibling repo's pyppeteer engine died with NameError on a line reached only while fetching, after an import had been removed — the @@ -1792,7 +1836,7 @@ def check_undefined_names_in_every_module(): def check_dockerfile_copies_everything_the_entrypoint_imports(): - """§10: all three repos in this family shipped an image that died with + """All three repos in this family shipped an image that died with ModuleNotFoundError on every invocation, --help included, because proxy_pool.py was missing from the COPY list. CI never built the image; this check needs no Docker.""" @@ -1840,7 +1884,7 @@ def check_env_example_documents_exactly_what_the_loader_reads(): def check_a_copied_env_example_reads_as_UNSET(): - """§17: `cp .env.example .env` followed by a run must not connect. + """`cp .env.example .env` followed by a run must not connect. The placeholder check was a literal set in a sibling repo, and the two credentialled URLs are documented the way the vendor documents them — @@ -1854,14 +1898,22 @@ def check_a_copied_env_example_reads_as_UNSET(): check(".env.example exists", False) return text = open(example, encoding="utf-8").read() - values = dict(re.findall(r"^([A-Z][A-Z0-9_]+)=(.*)$", text, re.M)) + # KOSE_URL is shipped COMMENTED OUT: a copied .env that set it made + # every run without --url fetch that address, and the --tags/--category + # builders then had to work around a URL nobody typed. It is still + # documented, so it is still read from the commented line here. + check("KOSE_URL is shipped commented out", + re.search(r"^#\s*KOSE_URL=", text, re.M) is not None + and re.search(r"^KOSE_URL=", text, re.M) is None) + values = dict(re.findall(r"^(?:#\s*(?=KOSE_URL=))?([A-Z][A-Z0-9_]+)=(.*)$", + text, re.M)) check("the example actually sets every variable", set(values) == set(env_config.ENV_KEYS), "example has %s, loader reads %s" % (sorted(values), sorted(env_config.ENV_KEYS))) # Every CREDENTIAL must read as unset. The default TARGET must not: it is # a real, usable URL, and blanking it would remove the one setting this - # file exists to make convenient (§17's check #3 says exactly this — the + # file exists to make convenient (the check #3 says exactly this — the # credentials unset, the non-credential default still usable). CREDENTIALS = {"TWOCAPTCHA_KEY", "KOSE_CDP_ENDPOINT", "KOSE_PROXY"} before = dict(os.environ) @@ -1920,7 +1972,7 @@ def check_engines_import_their_driver_at_module_level(): def check_engine_flag_sets(): - """§17's check #2: against the contract AND against each other, both ways. + """the check #2: against the contract AND against each other, both ways. A missing flag fails; so does closing a difference the README documents. """ @@ -1937,7 +1989,7 @@ def check_engine_flag_sets(): # and could not launch it on the development machine, so it needs a way # to point at another one. Its twins have no equivalent because they do # not ship a browser. Listed here so that closing the difference — or - # growing a second one — fails the build (§17). + # growing a second one — fails the build. DOCUMENTED_DIFFERENCES = {"puppeteer_scraper": {"--chromium-path"}} names = sorted(sets) for i in range(len(names) - 1): @@ -1953,7 +2005,7 @@ def check_engine_flag_sets(): def check_banned_and_removed_flags(): """Scoped to the ENGINES. - `--country` is absent here — the flag CLAUDE.md §10 bans outright, + `--country` is absent here — the flag this project bans outright on a scraper, because it could disagree with the URL. `--locale` is absent too: maison.kose.co.jp is one market (see EXTRA_FLAGS above). @@ -1972,7 +2024,7 @@ def check_banned_and_removed_flags(): '"%s"' % flag not in source) check("%s does not define the banned --country" % module, '"--country"' not in source, - "§10 bans it: it could disagree with the URL") + "banned: it could disagree with the URL") check("%s refuses --tags alongside --url" % module, "--url already names what to fetch" in source, "two ways to say what to fetch must not be merged in silence") @@ -2045,7 +2097,7 @@ def check_sample_output_matches_the_schema(): def check_no_test_mutates_the_working_tree(): - """§10: one suite used its own file as a fake chromedriver and chmod'd it + """One suite used its own file as a fake chromedriver and chmod'd it to 755, leaving a mode change in git status. Compares the tree against how it looked when the suite STARTED, not @@ -2063,7 +2115,7 @@ def check_no_test_mutates_the_working_tree(): def check_captcha_capability_claims_match_the_code(): - """§19: the most expensive bug this family can ship is a SENTENCE. + """The most expensive bug this family can ship is a SENTENCE. It fails in both directions and this family has shipped both: @@ -2104,7 +2156,7 @@ def check_captcha_capability_claims_match_the_code(): # rots the moment the site switches a bot manager on. # # So the claim has to be PAIRED with the thing that would retest it — the - # daily canary — rather than left as a sentence nobody re-measures (§13). + # daily canary — rather than left as a sentence nobody re-measures. claims_no_key = ("no, and it would be dishonest" in low or "do i need a 2captcha account" in low) if claims_no_key: @@ -2114,7 +2166,7 @@ def check_captcha_capability_claims_match_the_code(): "daily canary is what turns it back into a measurement") check("...and dates the measurement it rests on", re.search(r"20\d\d-\d\d-\d\d", readme) is not None, - "§13: a number describing a living thing needs its moment") + "a number describing a living thing needs its moment") # And in the other direction: if the README names a task type, the solver # had better build it. A capability claim citing no task type is a guess @@ -2141,7 +2193,7 @@ def check_captcha_capability_claims_match_the_code(): def check_policy_constants_have_a_consumer(): - """§17: a policy constant nothing reads is the same defect as dead code. + """A policy constant nothing reads is the same defect as dead code. `RETRY_ON_BLOCKED` carried a paragraph of measured justification in a sibling repo and NO engine consulted it, so setting it False changed @@ -2196,7 +2248,7 @@ def check_pagination_addressability_is_asked_per_url(): check("pagination: a category listing is addressable", page_flow.pagination_is_addressable(LISTING_URL)) check("pagination: a tag listing is addressable too — both routes were " - "CHECKED against the site's own rel=next before being trusted (§7)", + "CHECKED against the site's own rel=next before being trusted", page_flow.pagination_is_addressable(TAGS_URL)) check("pagination: a PRODUCT page is not — it is one page", not page_flow.pagination_is_addressable(PRODUCT_URL)) @@ -2216,10 +2268,10 @@ def check_page_and_position_are_unique_across_pages(): def check_a_broken_parser_is_not_reported_as_an_empty_category(): - """§20: a served page that links to N products and parses to zero is OUR + """A served page that links to N products and parses to zero is OUR bug, and saying "0 products" sends the reader to check the URL instead. - §20 also says to check the signal CAN fire before adding it. Here it + It is also worth checking the signal CAN fire before adding it. Here it genuinely can, and the margin is measured: rows come from `li.c-product__item` tiles while `product_link_count` counts `/g/g…/` hrefs, and a real grid page carries 48-55 of the latter against 24 of the @@ -2259,7 +2311,7 @@ def check_a_broken_parser_is_not_reported_as_an_empty_category(): def check_parser_found_nothing_is_not_a_complete_run(): - """The stop_reason must not read as success (§8: blocked != empty).""" + """The stop_reason must not read as success (blocked != empty).""" from output_writer import COMPLETE_STOP_REASONS, run_meta check("parse-failure: 'parser_found_nothing' is NOT a complete stop reason", "parser_found_nothing" not in COMPLETE_STOP_REASONS, @@ -2320,7 +2372,7 @@ def check_sidecar_shape(): def check_row_schema(): from output_writer import Product, ROW_CLASS_BY_MODE, UNIQUE_BY_SKU_MODES names = [f.name for f in fields(Product)] - equal("the family prefix is byte-identical and in order (§9)", + equal("the family prefix is byte-identical and in order", names[:5], ["source", "scraped_at", "url", "sku", "title"]) # The commerce columns this site DOES have. Maison KOSÉ is a shop, so the @@ -2330,7 +2382,7 @@ def check_row_schema(): check("the commerce column %r is present" % present, present in names) # And the ones measured ABSENT, each with its count in output_writer's - # docstring. §9 says removing a column needs the measurement written + # docstring. Removing a column needs the measurement written # down; this pins that they stay removed until someone re-measures. for gone in ("original_price", "discount_pct", "lowest_price_30d", "rating", "review_count", "ean", "gtin", "engraved"): @@ -2338,7 +2390,7 @@ def check_row_schema(): gone not in names, "if this is back, output_writer's measurement should be too") - # Site-specific columns go at the END of the row (§9), after the family's. + # Site-specific columns go at the END of the row, after the family's. equal("site-specific columns come last", names[-9:], ["mode", "tax_rate", "price_is_store_price", "badges", "subcategory", "volume", "colour_count", @@ -2353,7 +2405,7 @@ def check_row_schema(): def check_fixtures_carry_no_session_material(): - """§10: a real page dump carries the session that fetched it. + """A real page dump carries the session that fetched it. A sibling repo committed three `sessionId` values, their CSRF tokens and a real customer's display name, profile permalink and review text. None @@ -2363,7 +2415,7 @@ def check_fixtures_carry_no_session_material(): Maison KOSÉ is the easy case: it publishes products, not people, and no capture here carries a name, a review or a login. So this guards the SHAPE rather than any literal, which is the half that keeps working when - the next capture is taken (§10: guard with PATTERNS, not the old + the next capture is taken (guard with PATTERNS, not the old values). """ fixtures = { @@ -2463,10 +2515,10 @@ def check_the_credential_scan_skips_a_virtualenv_whatever_it_is_called(): environment called `.v`: pip's vendored `packaging/_elffile.py` carries a 32-hex string, so the credential scan failed on a perfectly clean clone. A name-based skip list covers the names its author happened to think of - and fails for everyone else — CLAUDE.md §21's "a check that fails on its + and fails for everyone else — the "a check that fails on its own repository is a check nobody can read", one step out. - The control matters as much as the check (§22): the fixture below is + The control matters as much as the check: the fixture below is built so that WITHOUT the fix it really would fail, which is asserted rather than assumed. """ @@ -2507,6 +2559,52 @@ def check_the_credential_scan_skips_a_virtualenv_whatever_it_is_called(): ci_checks.REPO = before +def check_the_credential_scan_reads_a_clone_under_any_parent(): + """The skip list is applied to the path RELATIVE to the repository. + + It used to test the absolute path, so a clone that merely lived under a + directory named `build` or `dist` — both in the skip list — scanned no + files at all and printed "0 files scanned, nothing credential-shaped". + A scan that reads nothing must fail, not pass; and a renamed `.env` + backup, which is where a live key actually sits, must be read. + """ + sys.path.insert(0, os.path.join(HERE, ".github")) + try: + import ci_checks + except ImportError as e: + skip("ci_checks", str(e)) + return + import pathlib + key = "f0e1d2c3" * 4 # assembled, so this file does not carry the shape + before = ci_checks.REPO + try: + for parent in ("build", "dist", "plain"): + with tempfile.TemporaryDirectory() as tmp: + root = os.path.join(tmp, parent, "kose-scraper") + os.makedirs(root) + open(os.path.join(root, "planted.py"), "w").write( + "K = '%s'\n" % key) + ci_checks.REPO = pathlib.Path(root) + found = ci_checks.secret_check() + check("a key planted in a clone under %s/ is caught" % parent, + any("planted.py" in f for f in found), str(found)) + with tempfile.TemporaryDirectory() as tmp: + root = os.path.join(tmp, "kose-scraper") + os.makedirs(root) + open(os.path.join(root, ".env.bak"), "w").write( + "TWOCAPTCHA_KEY=%s\n" % key) + ci_checks.REPO = pathlib.Path(root) + found = ci_checks.secret_check() + check("a key in an untracked .env.bak is caught", + any(".env.bak" in f for f in found), str(found)) + with tempfile.TemporaryDirectory() as tmp: + ci_checks.REPO = pathlib.Path(tmp) + check("a scan that read no files fails instead of passing", + bool(ci_checks.secret_check())) + finally: + ci_checks.REPO = before + + def check_a_404_is_terminal_and_not_a_retry(): """A discarded HTTP status turns a 404 into a retryable nothing. @@ -2533,7 +2631,7 @@ def check_a_404_is_terminal_and_not_a_retry(): equal("a served-but-empty page is still EMPTY, not not_found", page_flow.classify(EMPTY_HTML, 200), "empty") - # §20: check the signal CAN fire before adding it. It can, on a URL this + # Check the signal CAN fire before adding it. It can, on a URL this # scraper ACCEPTS: the tag route with no `tags=` parameter is a supported # address and answers HTTP 404 (measured 2026-09-18). A user who pastes # the bare facet URL lands exactly here, so the branch is reachable @@ -2590,7 +2688,7 @@ def check_the_engines_really_thread_a_status(): # grepping for the variable name anywhere in the file. The first # version of this check did the latter and passed for the wrong # reason: reverting the fix left `http_status` declared and unused, - # and the check went on saying green (§22 — verify that your control + # and the check went on saying green (verify that your control # actually failed). tree = ast.parse(source) calls = [] @@ -2726,7 +2824,7 @@ class _StubDriver: version of this stub spoke only Playwright's (`goto`, `content`), so the Selenium control failed with `'_StubPage' object has no attribute 'get'` — red both with the fix and without it, which is a broken check wearing - a working control's clothes (§22: check WHICH check caught the fault). + a working control's clothes (check WHICH check caught the fault). """ url = LISTING_URL current_url = LISTING_URL @@ -2774,6 +2872,13 @@ def evaluate(self, *a, **k): def setDefaultNavigationTimeout(self, ms): pass + # After a solve: Playwright and pyppeteer reload, Selenium refreshes. + def reload(self, *a, **k): + return _StubResponse() + + def refresh(self): + return None + class _StubResponse: status = 200 @@ -2855,17 +2960,60 @@ def check_retries_zero_is_a_usage_error(): "a plain type=int lets 0 through and the loop then runs never") +def check_counts_and_pauses_refuse_negative_values(): + """--pages/--concurrency must be >= 1; pauses and block retries >= 0. + + A plain `type=int` let `--pages 0` through to a run that fetched nothing + and reported an empty catalogue, and a negative `--delay` reached + `time.sleep` after page 1 had been fetched and crashed the run there. + """ + for good, fn, want in (("2", page_flow.at_least_one, 2), + ("0", page_flow.non_negative_int, 0), + ("0", page_flow.non_negative_seconds, 0.0), + ("1.5", page_flow.non_negative_seconds, 1.5)): + equal("%s(%r) is accepted" % (fn.__name__, good), fn(good), want) + for bad, fn in (("0", page_flow.at_least_one), ("-1", page_flow.at_least_one), + ("-1", page_flow.non_negative_int), + ("-0.5", page_flow.non_negative_seconds), + ("nan", page_flow.non_negative_seconds)): + raised = False + try: + fn(bad) + except argparse.ArgumentTypeError: + raised = True + check("%s(%r) is refused as a usage error" % (fn.__name__, bad), raised) + + wanted = { + "--pages": "page_flow.at_least_one", + "--concurrency": "page_flow.at_least_one", + "--delay": "page_flow.non_negative_seconds", + "--retry-delay": "page_flow.non_negative_seconds", + "--proxy-block-retries": "page_flow.non_negative_int", + } + for module in ENGINES + ("http_scraper",): + src = open(os.path.join(HERE, module + ".py"), encoding="utf-8").read() + for flag, typ in wanted.items(): + if '"%s"' % flag not in src: + continue + check("%s: %s goes through %s" % (module, flag, typ), + '"%s", type=%s' % (flag, typ) in src, + "a plain type=int/float lets a negative through") + src = open(os.path.join(HERE, "scraper_api_client.py"), encoding="utf-8").read() + check("scraper_api_client: --retry-delay refuses a negative", + '"--retry-delay", type=page_flow.non_negative_int' in src) + + def check_a_blocked_page_without_a_proxy_does_not_crash(): """Selenium and pyppeteer raised AttributeError on a blocked page. `mask(pool.current)` with no `--proxy` is `None.current`. Without a pool `BLOCK_RETRIES_WITHOUT_POOL` is 1, so the branch runs on the FIRST block — and the run exited 1, a CRASH, for a page that was merely blocked. - That destroys the blocked/empty/partial distinction §8 calls an - invariant, and the primary engine had the guard all along: it is the - "all three must agree" rule (§6) broken in the two quiet engines. + That destroys the blocked/empty/partial distinction the exit codes + exist for, and the primary engine had the guard all along: it is the + "all three must agree" rule broken in the two quiet engines. - Asserted as BEHAVIOUR rather than by reading the source (§20): the first + Asserted as BEHAVIOUR rather than by reading the source: the first version of this check walked the AST for an unguarded `pool.` attribute and reported every engine broken, including the one that was correct. This drives the real loop with no pool and a forced block, and asks only @@ -2910,13 +3058,142 @@ def check_a_blocked_page_without_a_proxy_does_not_crash(): +# The exception each engine's load loop treats as a navigation timeout. +_LOAD_TIMEOUT_TYPE = { + "playwright_scraper": "PWTimeout", + "selenium_scraper": "TimeoutException", + "puppeteer_scraper": None, # catches Exception: pyppeteer raises many +} + + +def check_a_page_that_never_loads_without_a_proxy_does_not_crash(): + """pyppeteer raised AttributeError when a page failed to load, no pool. + + The blocked branch had been fixed, but the LOAD-FAILURE branch still + called `pool.advance(...)` with no `--proxy`, where `pool` is None: + a page whose navigation timed out `--retries` times ended the run with a + traceback and exit 1 instead of a load failure (exit 5, or 6 with rows). + The other two engines rotate only for an unusable exit and only when a + pool has another exit in it. + + Driven as behaviour, with a navigation that always times out and, + separately, one that fails with Chromium's dead-proxy error text. + """ + for module in ENGINES: + engine = _import_engine(module) + if engine is None: + continue + type_name = _LOAD_TIMEOUT_TYPE[module] + exc_type = getattr(engine, type_name) if type_name else Exception + for label, text in (("a timeout", "Navigation timeout of 60000 ms"), + ("a dead proxy", "net::ERR_PROXY_CONNECTION_FAILED")): + session = _StubSession() + gotos = [] + + def fail(*a, **k): + gotos.append(1) + raise exc_type(text) + session.page.goto = fail + session.page.get = fail + args = _stub_args() + args.retries = 2 + raised, outcome = None, None + try: + outcome = engine._fetch_one_page( + session, args, None, 1, LISTING_URL) # pool=None + except Exception as e: # noqa: BLE001 + raised = e + check("%s: %s with no --proxy does not raise" % (module, label), + raised is None, "%s: %s" % (type(raised).__name__, raised) + if raised else "") + check("%s: %s was actually attempted" % (module, label), + bool(gotos), "goto was never called") + if outcome is not None: + check("%s: ...and is reported as a load failure" % (module,) + + " (%s)" % label, outcome.load_failed) + check("%s: ...and no relaunch without a pool (%s)" % (module, label), + session.relaunched == 0, str(session.relaunched)) + + +class _StubChallenge: + kind, source, sitekey, action = "recaptcha-v2", "stub", "stub-key", None + + +def _drive_budget(engine, detections, solver_raises=False): + """Run one page with a forced challenge and no pool; return purchases. + + `detections` is the sequence of answers the detector gives, one per + call to `handle_captcha_if_present`: None for "nothing here", True for + "a challenge". Nothing reaches the network: the solver is a stub. + """ + answers = list(detections) + purchases = [] + + def reconcile(*a, **k): + hit = answers.pop(0) if answers else True + return _StubChallenge() if hit else None + + def solve(*a, **k): + purchases.append(1) + if solver_raises: + raise RuntimeError("stub solver failure") + return "stub-token" + + names = ("detect_recaptcha_v3", "detect_recaptcha_in_page", + "reconcile_detections", "solve_recaptcha") + saved = {n: getattr(engine, n) for n in names} + saved_pf = (page_flow.classify, page_flow.classify_with_reason) + try: + engine.detect_recaptcha_v3 = lambda *a, **k: None + engine.detect_recaptcha_in_page = lambda *a, **k: None + engine.reconcile_detections = reconcile + engine.solve_recaptcha = solve + page_flow.classify = lambda html, status=None, url="": "challenge" + page_flow.classify_with_reason = ( + lambda html, status=None, url="": ("challenge", "stub")) + args = _stub_args() + args.twocaptcha_key = "stub-not-a-key" + engine._fetch_one_page(_StubSession(), args, None, 1, LISTING_URL) + finally: + for n, v in saved.items(): + setattr(engine, n, v) + page_flow.classify, page_flow.classify_with_reason = saved_pf + return len(purchases) + + +def check_the_solve_allowance_is_spent_only_on_a_purchase(): + """A detection that buys nothing must not use up the page's allowance. + + The allowance used to be counted per CALL to the detector. The + pre-classification call runs on every attempt and, on a normal page, + detects nothing — yet it consumed the one solve a page is allowed, so a + challenge first seen after classification was never offered to the + solver and the page came out blocked. + + Both directions are pinned, because this is the paid path: a late + challenge gets exactly one purchase, a challenge seen on every call + still gets exactly one, and a purchase that FAILS still counts. + """ + for module in ENGINES: + engine = _import_engine(module) + if engine is None: + continue + limit = page_flow.SOLVES_PER_PAGE + equal("%s: a challenge seen only after classification is solved" + % module, _drive_budget(engine, [None]), limit) + equal("%s: a challenge seen on every call is bought once per page" + % module, _drive_budget(engine, []), limit) + equal("%s: a failed purchase still counts against the allowance" + % module, _drive_budget(engine, [], solver_raises=True), limit) + + def check_every_paid_solve_goes_through_one_budget(): """`handle_captcha_if_present` runs TWICE per attempt; one was uncounted. Measured on this repo with the solver stubbed and a forced challenge: **3 invocations against `SOLVES_PER_PAGE = 1`** before the fix, 1 after. The uncounted call repeats on every block-retry, so the constant read - like a limit and was advisory (§17, §23). + like a limit and was advisory. Asserted by PARSING rather than by grepping, because a grep for `SOLVES_PER_PAGE` matches the constant, the comment that explains it and @@ -3098,7 +3375,7 @@ def check_the_http_engine_is_the_same_scrape(): What it must NOT do is accept a flag it cannot honour: a setting that looks configurable and does nothing is worse than one that is absent - (§3). Those are refused as usage errors. +. Those are refused as usage errors. """ path = os.path.join(HERE, "http_scraper.py") if not os.path.exists(path): @@ -3146,7 +3423,7 @@ def check_the_http_engine_is_the_same_scrape(): def check_a_failed_run_still_leaves_something_machine_readable(): """`.last_attempt.json`, beside a sidecar that may describe good data. - §9 is right that a FAILED run must write no `.meta.json`: `save` + It is right that a FAILED run must write no `.meta.json`: `save` leaves the previous good output in place, and a "failed" sidecar beside it would contradict it. The gap that left — a failed run told a caller nothing but an exit code — was raised by a third-party audit on @@ -3187,7 +3464,7 @@ def check_pyproject_matches_the_files_on_disk(): """`pyproject.toml` and `requirements*.txt` are kept in sync BY HAND. CI installs only `requirements.txt`, so a drift between the two goes - unnoticed there (§6) — which is why it is checked here instead. Raised + unnoticed there — which is why it is checked here instead. Raised by a third-party audit on 2026-10-13 as "one authoritative definition, or a check that they have not diverged"; this is the second, because the two files are read by different tools and neither can be generated from @@ -3314,12 +3591,12 @@ def check_no_marker_fires_on_a_page_fetched_over_CDP(): This repo carried a bare `recaptcha` marker until that page existed, and it FIRED on it. Nothing broke, because tiles are checked before markers - (§17) — but a page with no tiles, an exhausted tag facet say, would have + — but a page with no tiles, an exhausted tag facet say, would have been reported as a challenge over CDP: exit 3 for a correct answer. The claim "no marker fires on a good page" had been true of every capture this repo had, and every one of them was fetched LOCALLY. That - is §21's guard passing for the wrong reason, and §24 says to commit a + is the guard passing for the wrong reason, and the rule is to commit a CDP-fetched fixture for exactly this. """ check("the fixture really carries the extension's injections", @@ -3353,7 +3630,7 @@ def check_no_marker_fires_on_a_page_fetched_over_CDP(): equal("a REAL reCAPTCHA loader is still detected", page_flow.classify(real, 200), "challenge") - # §24: the marker set must score zero WITHOUT any extension strip, or + # The marker set must score zero WITHOUT any extension strip, or # the strip is load-bearing and the next marker added inherits the hole. check("this repo carries no chrome-extension strip to lean on", "chrome-extension" not in open( @@ -3393,7 +3670,7 @@ def check_a_url_from_the_environment_does_not_block_the_builders(): # captures the same filled-in value and nothing changes. Compared # against the CALL, not the string: the first version of this check # matched the word inside its own explanatory comment and reported - # every engine broken (§22 — a check that matches a comment rather + # every engine broken (a check that matches a comment rather # than the code). check("%s: captured before env_config.apply(args) runs" % module, src.index("typed_url = args.url")