diff --git a/.env.example b/.env.example
index 28c9dff..1796af9 100644
--- a/.env.example
+++ b/.env.example
@@ -37,9 +37,10 @@
TWOCAPTCHA_KEY=your_2captcha_api_key_here
# --- The Scraping Browser API (a managed browser over CDP) ---------------
-# The endpoint SHAPE, not a working endpoint: a profile's credentials live
-# about a day, so anything pasted here is stale by tomorrow and produces an
-# auth error a long way from its cause. Get fresh ones from the dashboard.
+# The endpoint SHAPE, not a working endpoint. A profile's credentials do
+# expire (how long they last varies; do not plan around a fixed figure), and
+# a stale endpoint produces an auth error (HTTP 401) a long way from its
+# cause. Get fresh ones from the dashboard.
#
# country-xx picks the exit country. `country-jp` for a Japanese exit.
# pid-... is a profile with persistent cookies. ONE live connection
diff --git a/.github/ci_checks.py b/.github/ci_checks.py
index c6adccb..de0fe26 100644
--- a/.github/ci_checks.py
+++ b/.github/ci_checks.py
@@ -300,7 +300,7 @@ def tracked_files():
# into pip's vendored code and flags a 32-hex string in `_elffile.py` as
# key-shaped. Correct about the string, wrong about the file — and it is the
# FIRST thing a new user sees from `python3 smoke_test.py`. A guard people
-# have to argue with is one they learn to suppress (CLAUDE.md §22).
+# have to argue with is one they learn to suppress.
#
# Deliberately NOT a narrowing of what gets scanned: an untracked file is
# still read, because a key pasted into a scratch file beside the scripts is
diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml
index ea5008a..93c38bc 100644
--- a/.github/workflows/canary.yml
+++ b/.github/workflows/canary.yml
@@ -20,10 +20,11 @@ name: canary
# kind: a keyword search returned 45 products a page, and a three-page run
# returned 135 rows with 100% price coverage and status `complete`.
#
-# The family's rule is that a canary which CANNOT pass must skip rather than
-# fail, because a check that is always red teaches everyone to ignore checks.
-# The other half of that rule applies here: a canary that CAN pass without a
-# credential must never be gated on one.
+# A canary that CAN pass without a credential must never be gated on one,
+# and a run that did not actually test the site must never end green: a
+# block (exit 3), a partial run (exit 6), content never obtained (exit 5) or
+# zero rows all fail the job. A green badge here means a real scrape of
+# rakuten.co.jp parsed real rows today, and nothing less.
#
# What a runner cannot test, and why there is no second job for it:
#
@@ -99,40 +100,28 @@ jobs:
- name: Interpret the exit code
id: verdict
- # ACCESS CONDITIONS ARE NOT DEFECTS. Failing the job on any non-zero
- # exit would make the canary red for things that are nobody's bug —
- # so 3 (blocked) and 6 (partial) are WARNINGS, while a run that got
- # in and found the wrong thing stays red.
- #
- # Note that on THIS site a blocked canary is genuinely surprising
- # rather than routine, because the listing routes are not gated. If
- # exit 3 starts appearing every day, that is news: either Rakuten has
- # begun refusing GitHub's address ranges, or something in this repo
- # has started sending a client whose claimed identity and TLS
- # fingerprint disagree — which is the one thing Akamai was measured
- # to refuse here.
+ # EVERY NON-ZERO EXIT FAILS THE JOB. This canary needs no secret and
+ # the listing routes were measured ungated from datacentre
+ # addresses, GitHub's runners included, so a run that did not get in
+ # is news rather than an access condition to wave through. The
+ # messages differ only in where to look first.
run: |
set -e
code="${{ steps.run.outputs.exit_code }}"
- tested=true
case "$code" in
0)
echo "OK — products parsed from a live run." ;;
3)
- tested=false
- echo "::warning::Blocked before parsing (exit 3) — reported as an ACCESS condition rather than a code defect, so this run is not failed. On this site that is NOT routine: the listing routes were measured ungated from a datacentre address. Check canary_run_page1_debug.html — if it is 43 bytes of 'Reference #', Akamai refused the client, and the first thing to suspect is a user agent or fingerprint that disagrees with the TLS handshake underneath it, not the address."
- echo "### Canary did NOT test anything (exit 3, blocked)" >> "$GITHUB_STEP_SUMMARY"
- echo "The site did not serve the request, so the parser was not exercised. Unusual here — see TROUBLESHOOTING.md, first section." >> "$GITHUB_STEP_SUMMARY" ;;
+ echo "::error::Blocked before parsing (exit 3). The listing routes were measured ungated from datacentre addresses, so this is news: either Rakuten has begun refusing GitHub's address ranges, or this repo has started sending a client whose claimed identity and TLS fingerprint disagree. Check canary_run_page1_debug.html — 43 bytes of 'Reference #' is Akamai refusing the client; a branded page under HTTP 403 is a route refusal."
+ exit 1 ;;
6)
- tested=false
- echo "::warning::Partial run (exit 6) — some pages were gathered and the run then stopped early. canary_run.meta.json names which pages failed. If the sidecar says stop_reason=end_of_listing this is not partial at all and the code is wrong; if it repeats every day, suspect the ?p= convention."
- echo "### Canary tested only PART of the run (exit 6)" >> "$GITHUB_STEP_SUMMARY"
- echo "See \`pages_failed\` in the sidecar artefact." >> "$GITHUB_STEP_SUMMARY" ;;
+ echo "::error::Partial run (exit 6): some pages were gathered and the run then stopped early, so the run did not test what it was asked to. canary_run.meta.json names which pages failed. If stop_reason is end_of_listing this is not partial at all and the code is wrong; a stop_reason of blocked_* or a throttle that never cleared means the site refused part of the run."
+ exit 1 ;;
4)
echo "::error::Ran fine but parsed 0 rows (exit 4) from a genre listing that holds 45 products per page. Either window.__INITIAL_STATE__.state.data.ichibaSearch has moved, or the item-URL pattern the DOM fallback anchors on has. This is the regression this canary exists to catch — the HTML dump in the artefacts tells you which."
exit 1 ;;
5)
- echo "::error::Remote API error (exit 5) from a run that uses no remote API. That should be impossible here; read the log."
+ echo "::error::The content was never obtained (exit 5): a navigation timeout, a transport failure or a page that parsed to nothing although the site stated rows. This run uses no remote API and no credential, so diagnose the transport and the dump; the log names the stop_reason."
exit 1 ;;
2)
echo "::error::Bad usage (exit 2) — the workflow's own arguments are wrong, not the site."
@@ -141,7 +130,7 @@ jobs:
echo "::error::The scraper crashed (exit $code)."
exit 1 ;;
esac
- echo "tested=$tested" >> "$GITHUB_OUTPUT"
+ echo "tested=true" >> "$GITHUB_OUTPUT"
- name: Check what the run actually produced
if: steps.verdict.outputs.tested == 'true'
@@ -197,7 +186,7 @@ jobs:
"this site is a mall; shop_code came from the payload on "
"every measured row")
- # THE ARITHMETIC BUG §18 NAMES: position restarts per page, so the
+ # THE ARITHMETIC BUG: position restarts per page, so the
# pair is what has to be unique.
pairs = [(r.get("page"), r.get("position")) for r in rows]
dupes = [p for p, n in collections.Counter(pairs).items() if n > 1]
@@ -218,7 +207,7 @@ jobs:
and not r.get("review_count") for r in rows))
# NO NEGATIVE DISCOUNTS. Costs one line and catches the regression
- # forever (§4).
+ # forever.
want("a row has an original_price at or below its price",
not any(r.get("original_price") is not None
and r.get("price") is not None
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index dde5f39..74737ab 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -83,52 +83,6 @@ jobs:
sys.exit(1 if failed else 0)
EOF
- - name: Committed sample output is real
- # The README links to these two files, so a push without them ships
- # broken links. And a sample invented by hand drifts from the schema and
- # teaches readers the wrong field names, so these are cut from a real
- # run rather than written by hand.
- run: |
- python - <<'EOF'
- import csv, json, sys
- from pathlib import Path
-
- MARKERS = ("sample-product-", "example brand", "sample product",
- "product description text", "lorem ipsum")
-
- fail = []
- for name in ("sample_output.json", "sample_output.csv"):
- if not Path(name).is_file():
- fail.append(f"{name} is missing — regenerate it from a real run")
-
- if not fail:
- rows = json.loads(Path("sample_output.json").read_text())
- if not rows:
- fail.append("sample_output.json is empty")
- blob = json.dumps(rows).lower()
- hit = [m for m in MARKERS if m in blob]
- if hit:
- fail.append(f"sample_output.json looks fabricated: {hit}")
-
- # Column set must match what the writers actually produce.
- from output_writer import Product
- from dataclasses import asdict
- expected = list(asdict(Product()).keys())
- for i, row in enumerate(rows):
- if list(row.keys()) != expected:
- fail.append(f"row {i} columns drifted from output_writer.Product")
- break
-
- with open("sample_output.csv", newline="", encoding="utf-8") as f:
- header = next(csv.reader(f))
- if header != expected:
- fail.append("sample_output.csv header drifted from output_writer.Product")
-
- for line in fail:
- print("FAILED:", line)
- sys.exit(1 if fail else 0)
- EOF
-
- name: Config reporter runs with no .env present
run: python env_config.py
@@ -154,10 +108,12 @@ jobs:
# endpoint, a planted 32-hex key and a planted http proxy URL.
run: python3 .github/ci_checks.py --secret-check
- - name: The repo's own --help and sample checks agree with this workflow
- # ci_checks.py implements all three checks. The two above are also
- # done as workflow steps for the clearer failure output; running them
- # here as well is what keeps the two from drifting apart again.
+ - name: The repo's own --help and sample checks
+ # One implementation, in ci_checks.py. The sample check used to be
+ # duplicated here as an inline heredoc importing the row class; it is
+ # now only run from the script, so the two cannot drift apart. The
+ # committed samples are cut from a real run, and this fails if they
+ # go missing, look fabricated, or drift from output_writer.Product.
run: python3 .github/ci_checks.py --help-check --sample-check
engine-smoke:
diff --git a/.gitignore b/.gitignore
index 2cd9ded..6559086 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,8 +40,9 @@ run/
runs/
results/
scratch/
-*.page[0-9]
-*.page[0-9][0-9]
+# --dump-html X --pages N writes X.pageN, and --pages goes to 150, so the
+# pattern covers any number of digits rather than stopping at 99.
+*.page[0-9]*
live_results.zip
# --- debug artifacts the engines drop next to themselves -----------------
@@ -99,7 +100,7 @@ secrets*
# recoverable only because that repo was private at the time. On 2026-09-17
# a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC
# repository, one `git add -A` from publication. Craigslist was fixed then
-# and the fix never reached anyone else, which is §16's rule unapplied: when
+# and the fix never reached anyone else, which is the rule unapplied: when
# you fix one, check the siblings.
.env*
!.env.example
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e7cc7ba..d6f4102 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,32 @@ their output or their bill.
## [Unreleased]
+> **Behaviour change: a 403 refusal is now a block, not a throttle.** No
+> engine used to pass the HTTP status to the page classifier, and Rakuten
+> renders its 403 refusal and its 503 rate-limit page with the same body, so
+> every 403 was read as a throttle: waited out three times (about 51 s) at
+> the same exit and then reported as a failed page. Playwright and pyppeteer
+> now carry the navigation's status into the classifier, so a 403 is
+> handled as a block (re-fetch or rotate, then exit 3) and a 503 is still
+> waited out. Selenium has no status to give and keeps the recoverable
+> reading.
+>
+> **Behaviour change: output files honour your umask.** `.json`, `.csv` and
+> `.meta.json` were written mode 0600 since the atomic writer arrived in
+> 0.2.1. A new file now gets `0666` minus the umask (0644 under the usual
+> 022), and rewriting an existing file keeps that file's mode.
+>
+> **Behaviour change: more numeric flags are checked at the command line.**
+> `--pages 0`, `--concurrency 0`, a negative `--delay`, `--retry-delay` or
+> `--proxy-block-retries` are now usage errors (exit 2) naming the flag, in
+> all three engines; `scraper_api_client.py` rejects a negative `--retries`
+> or `--retry-delay` and a `--timeout` below 1. `--pages 0` used to fetch
+> page 1 anyway and record `pages_requested: 0`.
+>
+> **Behaviour change: the canary is red whenever it did not test the site.**
+> A blocked (exit 3) or partial (exit 6) run used to end the job green with
+> a warning. Every non-zero exit now fails it.
+>
> **Behaviour change:** `--retries 0` is now a usage error (exit 2). It used
> to be accepted and made the page-load loop unreachable, so nothing was
> fetched and the engine reported the browser's own blank page as a result:
@@ -25,9 +51,41 @@ their output or their bill.
`range(1, retries + 1)`, so 0 never runs it and the engine judged
`about:blank`. Three third-party audits of this family ran exactly that and
reported a working scraper as broken. `--retries` now goes through
- `page_flow.attempts` in every entry point; a check reads the source of each
- one and calls the validator, so a new engine declaring the flag with a
- plain int fails the suite.
+ `page_flow.attempts` in all three browser engines; a check reads the
+ source of each one and calls the validator, so a new engine declaring the
+ flag with a plain int fails the suite. (`scraper_api_client.py --retries`
+ counts EXTRA attempts, so 0 stays legal there.)
+- **pyppeteer crashed with `AttributeError` when a page kept failing to load
+ and no proxy was configured.** Its load-failure branch called
+ `pool.advance()` without checking for a pool. It now rotates only for an
+ unusable exit and only when there is another exit to go to, as the
+ Playwright and Selenium engines already did.
+- **A failing captcha solver could be asked twice for one page.** The first
+ of the two solve call sites counted only successful solves against the
+ per-page allowance. Both now count every attempted purchase.
+- **The static reCAPTCHA detector missed a v2 widget** in a saved page that
+ carried only the `g-recaptcha` container or the `api2/anchor` frame,
+ without the literal `grecaptcha`, so a `--dump-html` capture of one was
+ reported as having no captcha. No captcha has been observed on Rakuten;
+ this only affects diagnosis.
+- **Help text and comments described another site's refusal** (an HTTP/2
+ stream reset, a headless-only "Access Denied" page). The `--solve-captcha`
+ help in all three engines and the exit-code comments now describe
+ Rakuten's own: a 43-byte Akamai deny under HTTP 200, a branded page under
+ HTTP 403, and the same page under HTTP 503 as a rate limit.
+- **The canary's exit-5 message** said a remote API error was impossible;
+ exit 5 means the content was never obtained, which a transport failure can
+ cause on any run.
+- `.gitignore` covered `--dump-html` page dumps only up to `.page99`;
+ `--pages` goes to 150. `requirements-selenium.txt` mentioned a
+ `--chromedriver` flag that does not exist. `.env.example` no longer gives
+ a fixed lifetime for Scraping Browser credentials.
+- Comments and docstrings no longer cite the numbered sections of internal
+ notes the public cannot read; each rule is stated in this repository's own
+ words, and the suite fails if such a citation comes back.
+- The CI sample-output check runs from one implementation
+ (`ci_checks.py --sample-check`) instead of also as an inline copy, and the
+ wording scan now covers `smoke_test.py` itself.
## [0.2.1] — 2026-09-30
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3a3db7f..651f6fa 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -105,10 +105,10 @@ Then the rest of the presentation, in the order that matters:
runs daily on a schedule, which is deliberate: the README's central claim
is that you need no key, no proxy and no account to read this site's
listings, and a scheduled, ungated, real three-page run is that claim
- under test every morning. The family's rule that a canary which cannot
- pass must SKIP has a second half — a canary that CAN pass without a
- credential must never be gated on one, or the badge goes green every day
- while testing nothing.
+ under test every morning. A canary that CAN pass without a credential
+ must never be gated on one, and any run that did not really test the
+ site (blocked, partial, nothing obtained, zero rows) fails the job, so a
+ green badge always means real rows were parsed that day.
Note that `workflow_dispatch` needs the workflow to exist on the DEFAULT
branch: from a feature branch `gh workflow run` answers
diff --git a/README.md b/README.md
index e80dcf5..ee1d379 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,8 @@ Two more things follow from that table, and both are easy to get wrong:
The canary in this repository is that claim under test: a real three-page
scrape, daily, from a bare GitHub runner, with **no secrets**. If Rakuten ever
puts the listing routes behind something, the badge above goes red the next
-morning.
+morning: a blocked, partial or empty run fails the job rather than passing
+with a warning.
## Install
diff --git a/captcha_solver.py b/captcha_solver.py
index 535dfbf..b83cbbb 100644
--- a/captcha_solver.py
+++ b/captcha_solver.py
@@ -201,7 +201,37 @@ def detect_recaptcha_v3(html: str, page_url: str) -> Optional[CaptchaChallenge]:
)
if "grecaptcha" not in html:
- return None
+ # A rendered v2 widget can reach a saved page without the literal
+ # `grecaptcha` anywhere in it: only the hyphenated container class
+ # `g-recaptcha` with its `data-sitekey`, or the widget's own
+ # `api2/anchor` iframe carrying the key as `k=`. Bailing here
+ # misdiagnosed such a `--dump-html` capture as "no captcha".
+ # A challenge frame present settles the variant as v2: v3 renders
+ # no challenge frame at all.
+ anchor = re.search(
+ r"recaptcha/(?:api2|enterprise)/anchor\?[^\"'>]*?[?&]k=([\w-]{20,})",
+ html)
+ container = re.search(
+ r"<[^>]*class=[\"'][^\"']*\bg-recaptcha\b[^>]*>", html, re.IGNORECASE)
+ key = None
+ size = None
+ if container:
+ km = re.search(r"data-sitekey=[\"']([\w-]{20,})[\"']",
+ container.group(0))
+ sm = re.search(r"data-size=[\"'](\w+)[\"']", container.group(0))
+ key = km.group(1) if km else None
+ size = sm.group(1) if sm else None
+ if key is None and anchor:
+ key = anchor.group(1)
+ sm = re.search(r"[?&]size=(\w+)",
+ html[anchor.start():anchor.start() + 600])
+ size = size or (sm.group(1) if sm else None)
+ if key is None:
+ return None
+ kind = ("recaptcha_v2_invisible" if size == "invisible"
+ else "recaptcha_v2")
+ return CaptchaChallenge(kind=kind, sitekey=key, page_url=page_url,
+ size=size)
exec_match = re.search(
r"grecaptcha\.execute\(\s*['\"]([\w-]{20,})['\"]\s*,\s*\{\s*action:\s*['\"]([\w_]+)['\"]",
@@ -723,14 +753,15 @@ class CaptchaUnsolvable(RuntimeError):
# a JPEG of distorted text and a GET form). Roughly 190 lines of it, and none
# of it is ported here, because this site has no such page.
#
-# What this site does instead is refuse a HEADLESS browser. Measured
-# 2026-09-10 from five different addresses, four of them residential: Akamai
-# answers with HTTP 403 and a 394-byte "Access Denied" page carrying a
-# reference id — no form, no image, no widget, nothing for a solver to
-# answer. And the trigger is the CLIENT rather than the address: the very
-# same addresses were served HTTP 200 and the full catalogue by a browser
-# with a real window. So the response to a block here is `--headful` or
-# `--cdp-endpoint`, not a solve and not a better proxy, and
+# What this site does instead is refuse the CLIENT. Measured 2026-09-21
+# from one datacentre address: Akamai answers a client whose claimed
+# User-Agent contradicts its TLS fingerprint with a 43-byte HTTP-200 body
+# holding only a reference id, and the ranking route answers a headless
+# browser with a branded HTTP-403 page — no form, no image, no widget,
+# nothing for a solver to answer on either. The same address was served the
+# full catalogue by plain curl and by a consistent browser. So the response
+# to a block here is a consistent client (or `--headful` on the ranking
+# route), not a solve and not a better proxy, and
# product_parser.detect_page_state reports it as "blocked" rather than
# "challenge" precisely so no solve is attempted and nothing is charged.
#
diff --git a/diff_runs.py b/diff_runs.py
index 28faea5..2c8bd7b 100644
--- a/diff_runs.py
+++ b/diff_runs.py
@@ -160,8 +160,7 @@ def diff_products(old: List[dict], new: List[dict],
# those sponsored entries never become rows at all: they carry a
# click-tracking redirect instead of a product URL and `parse_products`
# drops them. So there is no placement column for a lifecycle bucket to
- # key on, and porting one would be dead code that looks load-bearing
- # (§4).
+ # key on, and porting one would be dead code that looks load-bearing.
changed, source_changed, within_tolerance = [], [], []
for sku in old_by_sku.keys() & new_by_sku.keys():
before, after = old_by_sku[sku], new_by_sku[sku]
diff --git a/env_config.py b/env_config.py
index 879fdf0..c0dccf0 100644
--- a/env_config.py
+++ b/env_config.py
@@ -67,7 +67,7 @@
# A literal-only check reported both of those as CONFIGURED, so `cp
# .env.example .env` followed by a run connected to cb.2captcha.com with the
# string `{login}-zone-…` as its username and got a 401 — a confusing failure
-# a long way from its cause, which is precisely what §3's rule exists to
+# a long way from its cause, which is precisely what the rule exists to
# prevent. Any `{…}` left in a value means the line was never filled in.
_BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}")
@@ -204,7 +204,7 @@ def _dests_given_on_argv(dests, argv=None):
headful path and its Scraping Browser path are both first-class, and
switching between them is routine rather than exceptional.
- Scanning argv is the smallest fix that keeps §3's precedence intact --
+ Scanning argv is the smallest fix that keeps the precedence intact --
explicit flag, then exported variable, then `.env`, then default -- with
"explicit" now meaning what the caller typed rather than what happens to
be truthy.
diff --git a/fingerprint_client.py b/fingerprint_client.py
index 677d2b2..93d0d8a 100644
--- a/fingerprint_client.py
+++ b/fingerprint_client.py
@@ -248,7 +248,7 @@ def playwright_context_kwargs(fp: dict) -> dict:
# 2026-09-11: a fingerprint stating `deviceScaleFactor: 1.25` produced a
# browser reporting `window.devicePixelRatio === 1`, so the identity
# contradicted itself on an axis any fingerprinter reads for free. Same
- # shape as the three defects §16 lists -- a key the API returns that
+ # shape as the three wrong-key defects once found in this client -- a key the API returns that
# nothing applies -- and found the same way, by comparing the live
# browser against the fingerprint rather than by reading the code.
scale = (fp.get("screen") or {}).get("deviceScaleFactor")
@@ -349,7 +349,7 @@ def main() -> int:
# loader. It used to read `os.environ` alone, which meant a key put in
# `.env` exactly as README and .env.example instruct worked for every
# engine and failed here with "No API key" — a documented mechanism not
- # applied on one path, which is the shape of half the defects §16 lists.
+ # applied on one path, which is the shape of half the defects once found in this client.
# `.env` has to be LOADED before it can be read: `env_value` looks at
# os.environ, which `load_env` is what fills from the file. Calling it
# here rather than relying on an engine having called it is the whole
diff --git a/make_fixtures.py b/make_fixtures.py
index ab68dcf..bf9ca0f 100644
--- a/make_fixtures.py
+++ b/make_fixtures.py
@@ -162,7 +162,7 @@ def _scrub_obj(node):
# a public URL. Not a credential and not secret — and it
# trips every scanner that looks for a 32-hex string,
# including this repo's own CI grep, which would greet a new
- # user with a FAILED line on their first command (§23).
+ # user with a FAILED line on their first command.
#
# Scrubbed rather than allowlisted, because an allowlist
# that forgives 32-hex inside a rakuten.co.jp URL is a hole
@@ -245,7 +245,7 @@ def _unreviewed(hit):
text, re.S):
ld += block
# THE SCRAPING BROWSER'S OWN EXTENSION TAGS, kept verbatim when the
- # capture has them. This is the fixture §21 asks for: a guard is only as
+ # capture has them. This is the fixture the guard needs: a guard is only as
# good as the fixture it runs against, and the one that matters is the
# page fetched the way a real run fetches it.
#
diff --git a/output_writer.py b/output_writer.py
index 1550523..8f60908 100644
--- a/output_writer.py
+++ b/output_writer.py
@@ -119,7 +119,7 @@ class Product:
# ブランド. Populated on 240 of 405 measured rows (59%).
#
# `hit["brand"]` — the field that looks like the source — is null on
- # 405 of 405 and would make this a dead column (§9). Deliberately NOT
+ # 405 of 405 and would make this a dead column. Deliberately NOT
# derived by splitting the title: a Rakuten product name opens with
# campaign text (`【9/24まで P10倍！】`) far more often than with a maker,
# so a split would be a guess wearing a fact's clothes.
@@ -137,7 +137,7 @@ class Product:
# 405 rows), so the value comes from the page's own structured data:
# `offers.priceCurrency` in the listing page's JSON-LD, or
# `` on a detail page. Null rather than
- # defaulted when a page states nothing (§4).
+ # defaulted when a page states nothing.
currency: Optional[str] = None
# The was-price, and ONLY where Rakuten's own
# `doublePrice.referencePriceVerified` flag is true — Japan's
@@ -335,6 +335,18 @@ def _atomic(path: str, newline: Optional[str] = None):
yield handle
handle.flush()
os.fsync(handle.fileno())
+ # NamedTemporaryFile creates the file 0600 and os.replace keeps that
+ # mode, so without this every output would be owner-only. A file that
+ # already exists keeps its own mode; a new one gets what a plain
+ # open(path, "w") would have given it, i.e. 0666 minus the umask.
+ # Never a flat 0644: under umask 002 that would narrow 0664.
+ try:
+ mode = os.stat(path).st_mode & 0o777
+ except OSError:
+ umask = os.umask(0)
+ os.umask(umask)
+ mode = 0o666 & ~umask
+ os.chmod(handle.name, mode)
os.replace(handle.name, path)
except BaseException:
try:
@@ -436,27 +448,20 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> int:
# search genuinely matched nothing" from "something stood between us and the
# content". See product_parser.detect_bot_challenge.
#
-# On this site this code specifically does NOT cover the three ways to get a
-# real page with no products on it: a `/p/` discovery hub, which
-# answers 200 with banners and carousels and no grid; a search whose query
-# matches nothing ("Oops, produk nggak ditemukan"); and one page past the
-# end of a category listing. All three are EXIT_NO_PRODUCTS — the request
-# was served exactly as asked and simply has no products on it. Reporting
-# any of them as blocked would send a user hunting for a proxy problem that
-# does not exist.
+# On this site this code specifically does NOT cover a real page with no
+# products on it: a search whose query matches nothing, or a genre with no
+# items. Those are EXIT_NO_PRODUCTS — the request was served exactly as
+# asked and simply has no products on it. Reporting either as blocked would
+# send a user hunting for a proxy problem that does not exist.
#
-# What EXIT_BLOCKED means here is unusually literal: this site refuses a
-# address it has scored NOTHING at all. No status code, no interstitial, no
-# vendor marker — the HTTP/2 stream is reset and the run sees a connection
-# error rather than a page.
+# What EXIT_BLOCKED means on Rakuten is one of two refusals, both carrying
+# Akamai's reference id: a tiny HTTP-200 deny (about 43 bytes, given to a
+# client whose claimed User-Agent contradicts its TLS fingerprint), or a
+# branded refusal page under HTTP 403. The same branded body under HTTP 503
+# is a THROTTLE, not a block: it is waited out at the same exit, and only
+# counts here if it never clears.
EXIT_BLOCKED = 3
-# Exit code for a run that gathered SOME rows and then stopped early — a
-# page-load timeout, a 503 throttle, or a challenge on page 3 of 10. The
-# output file is still written (throwing away three good pages would be
-# worse), but it is not a complete picture, and a consumer that cannot tell
-# the difference will read the pages that were never fetched as products that
-# disappeared from the catalogue. See write_run_meta.
# A REMOTE service failed — the Scraping Browser refusing the connection
# (`profile_locked` is the common one: a profile allows a single live
# connection), or the Scraper API answering an error. Distinct from 1 (a
@@ -467,6 +472,12 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> int:
# drifts.
EXIT_API_ERROR = 5
+# Exit code for a run that gathered SOME rows and then stopped early — a
+# page-load timeout, a 503 throttle that never cleared, or a refusal on page
+# 3 of 10. The output file is still written (throwing away three good pages
+# would be worse), but it is not a complete picture, and a consumer that
+# cannot tell the difference will read the pages that were never fetched as
+# products that disappeared from the catalogue. See write_run_meta.
EXIT_PARTIAL = 6
diff --git a/page_flow.py b/page_flow.py
index c66605b..3167764 100644
--- a/page_flow.py
+++ b/page_flow.py
@@ -2,7 +2,7 @@
Rakuten answers a request six ways and five of them want a different
response, which is why this module exists rather than the same triage being
-written three times inside three engines and drifting apart (§1):
+written three times inside three engines and drifting apart:
content the listing payload or the item island is in the response
empty served exactly as asked and `numFound: 0` — the site's own
@@ -35,7 +35,7 @@
content() -> Optional[str] current HTML, None if unavailable
sleep(ms) -> None wait
-No JavaScript crosses that boundary in either direction (§1): Selenium's
+No JavaScript crosses that boundary in either direction: Selenium's
`execute_script` takes a function BODY with an explicit `return` where
Playwright and pyppeteer take `() => expr`, so the shared module names the
OPERATION and the engine spells it in its own driver's dialect.
@@ -80,7 +80,7 @@
# Readiness
# ---------------------------------------------------------------------------
# A price node WITH SOMETHING IN IT rather than a card count, for the reason
-# §8 gives: a selector matching the wrong element is worse than one matching
+# the rule is: a selector matching the wrong element is worse than one matching
# nothing, and a tile's shell can exist while its amount is still blank.
#
# What a timed-out wait costs here is small and worth stating plainly,
@@ -93,7 +93,7 @@
READY_SELECTOR_LISTING = '[class*="price--"]'
READY_SELECTOR_PRODUCT = '[class*="price"], #priceCalculationConfig'
-# Above 1, per §5: waiting for a single match resolves on an unrelated node
+# Above 1: waiting for a single match resolves on an unrelated node
# long before a grid paints. Below the smallest real page — a Rakuten
# listing page is 45 products on 11 of 11 captures, and the LAST page of a
# capped query is 45 too (page 150 came back `start: 6705`, exactly
@@ -118,7 +118,7 @@ def min_matches(mode: str = "listing", expected: Optional[int] = None) -> int:
`expected` is the page's own hit count where the caller knows it,
because this site states it: a query with 5 total hits renders 5 tiles,
and waiting for 8 of them would time out on a page that is fully
- painted. Never above the floor and never below 2 (§5).
+ painted. Never above the floor and never below 2.
"""
floor = _MIN.get(mode, MIN_CARD_MATCHES)
if expected is None:
@@ -158,7 +158,7 @@ def wait_for_count(count: Callable[[str], int], sleep: Callable[[int], None],
`EvalError: Evaluating a string as JavaScript violates the following
Content Security Policy directive` on a site whose CSP has no
`unsafe-eval`, and took the run down with exit 1 on that site's most
- obvious URL (§18). Counting elements goes through the protocol, works
+ obvious URL. Counting elements goes through the protocol, works
under any CSP, and spells the same in all three drivers.
"""
waited = 0
@@ -187,7 +187,7 @@ def classify(html: Optional[str], status: Optional[int] = None,
Getting that wrong is not a style question: a sibling repo shipped two
of three engines calling this as `classify(html, url=...)`, both crashed
on their first fetch, and nothing short of a live run or a
- signature-binding check saw it (§17).
+ signature-binding check saw it.
On this site the status is more load-bearing than on any sibling: it is
the only thing separating a 503 throttle from a 403 block, because
@@ -199,7 +199,7 @@ def classify(html: Optional[str], status: Optional[int] = None,
# The retry/solve/blocked decision as DATA rather than as three copies of an
-# if-chain in three engines (§1).
+# if-chain in three engines.
#
# parse is there anything on this page worth writing down?
# retry would fetching it again plausibly help?
@@ -215,7 +215,7 @@ def classify(html: Optional[str], status: Optional[int] = None,
# nothing, and nothing downstream could tell them from results.
"empty": {"parse": False, "retry": False, "solve": False, "blocked": False},
# Served, and still painting. Wants the readiness wait, not another
- # fetch: refetching a shell just buys another shell (§18).
+ # fetch: refetching a shell just buys another shell.
"shell": {"parse": True, "retry": False, "solve": False, "blocked": False},
# Retryable, NOT blocked, and not paid for. Rakuten's 503 is a rate
# limit: it arrived while a probe was fetching with no delay between
@@ -272,7 +272,7 @@ def is_unpainted(state: str, html: Optional[str]) -> bool:
# answer and the advice below says what is.
#
# The engines read these constants rather than computing their own budget —
-# a policy constant nothing consults is the same defect as dead code (§17),
+# a policy constant nothing consults is the same defect as dead code,
# and the suite asserts each one has a consumer.
RETRY_ON_BLOCKED = True
BLOCK_RETRIES_WITHOUT_POOL = 1
@@ -292,7 +292,7 @@ def is_unpainted(state: str, html: Optional[str]) -> bool:
# through `solve_budget`, because a sibling repo discovered that this
# constant had read like a limit in every repo in the family while every
# engine called the solver twice per attempt and counted once — one page
-# bought three Turnstile solves (§23).
+# bought three Turnstile solves.
SOLVES_PER_PAGE = 1
@@ -357,7 +357,7 @@ def block_advice(html: Optional[str], headless: bool,
# ---------------------------------------------------------------------------
# Pagination
# ---------------------------------------------------------------------------
-# Three layers, weakest last (§7). On this site layer 1 is MISSING, which is
+# Three layers, weakest last. On this site layer 1 is MISSING, which is
# worth stating because the family's template leads with it:
#
# 1. A standards-based `` — ABSENT. Rakuten publishes
@@ -399,8 +399,8 @@ def requested_page_number(url: str, loop_index: int = 1) -> int:
complete run with nothing in it.
Caught by running `--url '...?p=2' --pages 1`, which is an obvious thing
- for a user to type and which no test and no ordinary run had tried
- (§15). Inside the loop the two numbers agree, because the URL is built
+ for a user to type and which no test and no ordinary run had tried.
+ Inside the loop the two numbers agree, because the URL is built
by `page_url` from that same counter — which is exactly why the bug was
invisible.
"""
@@ -426,7 +426,7 @@ def served_the_page_asked_for(html: Optional[str],
payload states `pagination.start`: asked for page 151 and handed
`start: 0` is the server saying outright that it served page 1.
- That is §23's trap on a second site and in a second spelling — there the
+ That is the trap on a second site and in a second spelling — there the
server's own page number hid in an Apollo cache key, here it is a plain
integer — and the general rule is the transferable part: before trusting
`?p=N`, fetch one page PAST the site's own last and read what comes
@@ -475,7 +475,7 @@ def _as_hrefs(next_href: Union[str, Iterable[str], None]) -> List[str]:
string is what took a sibling repo's FIRST live run down —
`TypeError: Cannot mix str and non-str arguments`, from `urljoin` being
handed a list — and nothing short of running it saw that, because the
- offline checks called it the way its author was thinking (§17).
+ offline checks called it the way its author was thinking.
"""
if next_href is None:
return []
@@ -567,7 +567,7 @@ def pages_at_cap(html: Optional[str]) -> bool:
numbers itself: one measured query reported `numFound: 3,053,682`
against a `subset` of 6,750. A 150-page run of that query is complete —
it fetched everything Rakuten will serve — and it is a 0.2% sample.
- Only saying so lets a consumer tell the two apart (§21), and it gives
+ Only saying so lets a consumer tell the two apart, and it gives
`diff_runs.py` the third meaning of `removed`: not delisted, not
un-fetched, but outside this run's slice of a capped result set.
"""
@@ -599,7 +599,7 @@ def concurrency_limit(url: str) -> Optional[int]:
def concurrency_refusal(url: str) -> Optional[str]:
"""Why concurrency above 1 is refused for this URL, or None.
- Refused WITH the reason (§18): a single product page and the mall's
+ Refused WITH the reason: a single product page and the mall's
front page have no page 2 at all, so there is nothing for a second
worker to fetch, and silently running one worker would look like the
flag did something.
@@ -669,3 +669,42 @@ def attempts(value) -> int:
f"With 0 the load loop never runs, nothing is fetched, and the "
f"engine reports the browser's blank page as a result.")
return number
+
+
+def at_least_one(value) -> int:
+ """argparse `type=` for counts that must be 1 or more (--pages,
+ --concurrency). `--pages 0` used to fetch page 1 anyway and record
+ `pages_requested: 0` in the sidecar, and `--concurrency 0` meant nothing
+ at all; both are usage errors now, named by argparse."""
+ try:
+ number = int(value)
+ except (TypeError, ValueError):
+ raise _argparse.ArgumentTypeError(f"must be an integer, not {value!r}")
+ if number < 1:
+ raise _argparse.ArgumentTypeError(f"must be at least 1, not {number}")
+ return number
+
+
+def non_negative_int(value) -> int:
+ """argparse `type=` for a count that may be 0 but not negative
+ (--proxy-block-retries)."""
+ try:
+ number = int(value)
+ except (TypeError, ValueError):
+ raise _argparse.ArgumentTypeError(f"must be an integer, not {value!r}")
+ if number < 0:
+ raise _argparse.ArgumentTypeError(f"must be 0 or more, not {number}")
+ return number
+
+
+def non_negative_seconds(value) -> float:
+ """argparse `type=` for a delay in seconds (--delay, --retry-delay). A
+ negative sleep raises ValueError deep inside a run instead of at the
+ command line."""
+ try:
+ number = float(value)
+ except (TypeError, ValueError):
+ raise _argparse.ArgumentTypeError(f"must be a number, not {value!r}")
+ if not number >= 0: # also refuses nan
+ raise _argparse.ArgumentTypeError(f"must be 0 or more, not {value}")
+ return number
diff --git a/playwright_scraper.py b/playwright_scraper.py
index ebb9780..5794411 100644
--- a/playwright_scraper.py
+++ b/playwright_scraper.py
@@ -190,7 +190,7 @@ class PageOutcome:
# Rakuten's own arithmetic about the query: total_results,
# pages_available, reachable_max, capped_by_site, pages_beyond_cap. In
# the sidecar because on this site "complete" and "exhaustive" are wildly
- # different words and a status alone would be lying by omission (§21).
+ # different words and a status alone would be lying by omission.
cap: Optional[dict] = None
# Whether the SERVER served the page we asked for, read off
# `pagination.start` rather than trusted from the request. False means
@@ -297,7 +297,7 @@ def _advertised_next_hrefs(page, page_num: int) -> List[str]:
Rakuten DOES advertise its own page links — nine of them on page 1 of a
150-page query, absolute, spelling `?p=2` through `?p=9` — and it
publishes **no `` at all**, on 3 of 3 captures. Only
- `rel="canonical"`. So §7's most-durable layer simply does not exist on
+ `rel="canonical"`. So the most-durable layer simply does not exist on
this site and the selector reaches for a build artefact because there is
nothing better to reach for; you cannot order signals by durability when
the site declines to publish the durable one.
@@ -413,7 +413,7 @@ def _same_url(a: str, b: str) -> bool:
reproduced inside one engine.
On this site the comparison has to strip a long tracking tail: a listing
- anchor arrives with `?extParam=…keyword=kopi&search_id=…&src=search` and
+ anchor arrives with a tracking query string (`?l2-id=…`, `scid=…` and similar) and
a detail page's own canonical arrives with a UTM triple, so two views of
one page never match unless both sides are cleaned.
"""
@@ -573,12 +573,10 @@ def _connect_remote(pw, args):
# challenges inside the browser: https://2captcha.com/scraper/browser-api/api
# Tried first when --cdp-endpoint is set; this script's own detect+solve
# logic still runs as a fallback if the endpoint does not support it.
- # Note it does NOT cover this site's refusal, which is not a challenge:
- # Akamai answers a headless browser with a 394-byte "Access Denied" that
- # has nothing to solve on it, and a real window rather than a better
- # address is the answer. No challenge has ever been observed here; this is
- # wired
- # up because one can appear between deploys.
+ # Note it does NOT cover this site's refusals, which are not challenges:
+ # Akamai's 43-byte deny and the branded 403 page have nothing to solve on
+ # them. No challenge has ever been observed here; this is wired up
+ # because one can appear between deploys.
try:
cdp_session = context.new_cdp_session(page)
cdp_session.send("Captcha.setAutoSolve", {"autoSolve": True, "options": [{"type": "*"}]})
@@ -659,9 +657,13 @@ def _content_when_settled(page, attempts: int = 4, pause_ms: int = 700):
return None
-def handle_captcha_if_present(page, args) -> bool:
+def handle_captcha_if_present(page, args, purchases=None) -> bool:
"""Detect and solve a challenge. True if something was solved.
+ `purchases`, when given, gets one entry appended immediately BEFORE the
+ solver is asked for a token, so a caller can count attempted purchases
+ (successful or not) against the page's solve allowance.
+
Runs after EVERY navigation, for ANY page — not scoped to one URL. The
static-HTML and runtime reCAPTCHA detectors are run and reconciled
against each other rather than short-circuited, because they can disagree
@@ -671,7 +673,7 @@ def handle_captcha_if_present(page, args) -> bool:
honest sentences about it. **This repo implements reCAPTCHA v2/v3;** if
Rakuten ever renders an enterprise reCAPTCHA or a Turnstile, 2Captcha
solves both and this client would need the matching task type added —
- that is a TODO here, not a limit of the product (§19).
+ that is a TODO here, not a limit of the product.
What is measured is narrower: no captcha of any kind is configured
anywhere on rakuten.co.jp. Zero reCAPTCHA, hCaptcha, Turnstile,
@@ -724,6 +726,8 @@ def handle_captcha_if_present(page, args) -> bool:
"solved — continuing with whatever the page already "
"holds.")
return False
+ if purchases is not None:
+ purchases.append(challenge.kind)
try:
token = solve_recaptcha(challenge, args.twocaptcha_key,
api_version=args.captcha_api,
@@ -758,7 +762,7 @@ def _parse_for_mode(html: str, url: str, args, page_num: int = 1) -> List:
restarts at 1 on every page: without the page number beside it, a row
from page 2 claims the same position as one from page 1 and the two are
indistinguishable in the output. A sibling repo's first live run wrote
- 120 rows all labelled page 1 for exactly that reason (§18).
+ 120 rows all labelled page 1 for exactly that reason.
"""
if args.mode == "product":
return parse_product_page(html, url, page=page_num,
@@ -828,9 +832,17 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
# Retry a navigation timeout rather than ending the run on it. One
# network flap on page 12 of 50 should not break the loop.
load_failed, exit_failed = False, None
+ # The HTTP status of the navigation, carried into the classifier.
+ # On this site it is the only thing that separates a 403 refusal
+ # from a 503 throttle: Rakuten renders both with the same branded
+ # body, and without the status the classifier has to read that body
+ # as the recoverable one (a throttle).
+ status = None
for attempt in range(1, args.retries + 1):
try:
- session.page.goto(url, wait_until="domcontentloaded", timeout=60000)
+ resp = session.page.goto(url, wait_until="domcontentloaded",
+ timeout=60000)
+ status = resp.status if resp is not None else None
load_failed = False
break
except (PWTimeout, PWError) as e:
@@ -868,17 +880,26 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
# counted only at the second call site, so `SOLVES_PER_PAGE = 1`
# read like an enforced cap in every repo of this family while one
# page could buy two solves — measured at three on a site where a
- # challenge rendered on every fetch (§23). Invisible wherever a
+ # challenge rendered on every fetch. Invisible wherever a
# challenge is rare, which is everywhere until it is not.
if page_flow.solve_budget(solves_bought):
- if handle_captcha_if_present(session.page, args):
- solves_bought += 1
+ # Counted when a purchase is ATTEMPTED, not when it succeeds: a
+ # solver that raised was still asked to work, and counting only
+ # successes let one page reach the solver twice. A page with no
+ # captcha on it spends nothing here.
+ purchases = []
+ solved = handle_captcha_if_present(session.page, args,
+ purchases=purchases)
+ solves_bought += len(purchases)
+ if solved:
# A solve navigated the page. Give the destination a moment
- # before judging what came back.
+ # before judging what came back, and forget the status of the
+ # navigation it replaced.
+ status = None
session.page.wait_for_timeout(1000)
html = _content_when_settled(session.page) or ""
- state = _classify(session.page, html)
+ state = _classify(session.page, html, status)
# "Not painted yet" is not a fault, and telling it apart from one is
# what the first live search run of this engine got wrong. A CATEGORY
@@ -913,13 +934,13 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
logger.info("The grid still had not painted after %.0fs "
"(%d match(es)).", wait_timeout / 1000, found)
html = _content_when_settled(session.page) or html
- state = _classify(session.page, html)
+ state = _classify(session.page, html, status)
# No interstitial-settling step here, and its absence is measured
# rather than an omission. This site has no interstitial to settle: a
- # refused request gets a 394-byte "Access Denied" with no markup at
- # all, so there is nothing to wait out and nothing to reclassify. See
- # page_flow's "There is no block page".
+ # refused request gets a 43-byte deny or a branded 403 page, so there
+ # is nothing to wait out and nothing to reclassify. A 503 throttle is
+ # handled below by waiting at the same exit.
#
# The paid path is reached only for state "challenge", which NO
# capture of this site has ever produced. It is wired up because a
@@ -930,9 +951,12 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
and page_flow.solve_budget(solves_bought)):
solves_bought += 1
if handle_captcha_if_present(session.page, args):
+ # The solve reloaded the page; the status of the navigation
+ # before it no longer describes what is on screen.
+ status = None
session.page.wait_for_timeout(1000)
html = _content_when_settled(session.page) or html
- state = _classify(session.page, html)
+ state = _classify(session.page, html, status)
# The VERIFIED outcome, and the only one worth reporting: a
# "ready" task result is not evidence the token works. This
# line is what says whether the money bought anything.
@@ -1020,8 +1044,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
# solve, and that a proxy is probably not the answer either. Akamai
# refuses this site with a 43-byte body whose whole content is
# `Reference #18.…` — no widget, nothing for any solver at any
- # price — and it arrives under **HTTP 200**, so the status code is
- # not the signal.
+ # price — and it arrives under **HTTP 200**. A route Rakuten refuses
+ # outright answers instead with its branded page under HTTP 403, and
+ # that status is what tells it apart from the same page under 503,
+ # which is a throttle and never reaches this branch.
#
# The dump is written even when it is empty, because "43 bytes" is
# itself the diagnosis here and a reader who finds no file at all
@@ -1115,7 +1141,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
# so this runs only for a state the policy has already given up on and
# only REFINES the reason.
#
- # The marker set itself is chosen the same way (§18): every candidate
+ # The marker set itself is chosen the same way: every candidate
# was counted on pages Rakuten plainly served before any of it was
# trusted. `cf-turnstile` is excluded because the Scraping Browser's own
# auto-solve extension injects it into every page it loads, and a bare
@@ -1152,7 +1178,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
# with 45 real products, and `/category/{id}/?p=2` answers 200 with page
# 1 without even redirecting. A run that trusted its own request would
# re-collect page 1 for as long as it was asked to and report a
- # complete, entirely duplicate file (§23).
+ # complete, entirely duplicate file.
if args.mode == "listing":
# Compared against the page the fetched URL actually ASKS for, not
# against the loop's counter — a run started on a URL that already
@@ -1202,7 +1228,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
# rather than guessed. Both numbers are recorded, because on this site
# they are wildly different and only having both makes the run honest:
# one measured query reported `numFound` 3,053,682 against a `subset` of
- # 6,750, so a full run of it is complete AND a 0.2% sample (§21).
+ # 6,750, so a full run of it is complete AND a 0.2% sample.
#
# `numFound` is also a LIVING number and is recorded as the site's
# answer at this moment rather than asserted against — three fetches of
@@ -1265,8 +1291,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome
# payload carries the price and the rendered tile prints that same
# number formatted, so `price_source` is "state" on every listing row
# and a confirmation threshold would describe nothing. Porting a
- # sibling repo's overlay would be dead code that looks load-bearing
- # (§4).
+ # sibling repo's overlay would be dead code that looks load-bearing.
#
# What IS worth reporting is the share of rows Rakuten publishes a
# was-price for, because on the listing route the honest answer is
@@ -1656,7 +1681,7 @@ def scrape(args) -> int:
# the query, and on this site that is not optional decoration: a
# `status: complete` run of a query the site caps at 6,750 of 3,053,682
# matches is complete and is a 0.2% sample, and a sidecar that said only
- # "complete" would be lying by omission (§21). It also gives
+ # "complete" would be lying by omission. It also gives
# `diff_runs.py` the third meaning of `removed` — not delisted, not
# un-fetched, but outside this run's slice of a capped result set.
extra = None
@@ -1730,7 +1755,7 @@ def parse_args():
"the request — the browsed URL is in the run's "
"sidecar. Pass this to override it with a label of "
"your own.")
- p.add_argument("--pages", type=int, default=1,
+ p.add_argument("--pages", type=page_flow.at_least_one, default=1,
help="Number of listing pages to crawl (45 products "
"each). Applies to --mode listing; a product page is "
"one page. Rakuten caps EVERY query at 6,750 results "
@@ -1738,8 +1763,8 @@ def parse_args():
"for page 151 gets page 1 again with HTTP 200, so a "
"run stops at the cap and the sidecar records both "
"numbers.")
- p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds")
- p.add_argument("--concurrency", type=int, default=1, metavar="N",
+ p.add_argument("--delay", type=page_flow.non_negative_seconds, default=2.0, help="Delay between pages, seconds")
+ p.add_argument("--concurrency", type=page_flow.at_least_one, default=1, metavar="N",
help="Fetch pages through N parallel workers (default 1 — "
"unchanged sequential behaviour). Each worker runs its "
"own browser and holds its own proxy exit, so N>1 "
@@ -1751,7 +1776,7 @@ def parse_args():
"that comes back EMPTY is not retried — see "
"page_flow.STATE_POLICY — because an empty hub "
"category is a correct answer, not a fault.")
- p.add_argument("--retry-delay", type=float, default=2.0,
+ p.add_argument("--retry-delay", type=page_flow.non_negative_seconds, default=2.0,
help="Seconds before the first page-load retry, doubling "
"thereafter (default 2.0)")
p.add_argument("--format", choices=["json", "csv", "both"], default="both")
@@ -1781,7 +1806,7 @@ def parse_args():
p.add_argument("--proxy-shuffle", action="store_true",
help="Shuffle the pool at startup, so concurrent runs do not "
"all begin on the first exit in the file.")
- p.add_argument("--proxy-block-retries", type=int,
+ p.add_argument("--proxy-block-retries", type=page_flow.non_negative_int,
default=page_flow.BLOCK_RETRIES_WITH_POOL,
help="When a page comes back refused, retry it from this "
"many OTHER exits before giving up (default %d, from "
@@ -1834,10 +1859,12 @@ def parse_args():
help="when-blocked (default): only pay to solve a "
"reCAPTCHA if the content is not already readable. "
"always: solve whenever one is detected. Neither "
- "setting touches this site's refusal, which is not a "
- "page at all — the HTTP/2 stream is reset and nothing "
- "arrives — so no solve helps there and none is "
- "attempted or billed. In fact NO challenge has ever "
+ "setting touches Rakuten's refusals (a 43-byte "
+ "Akamai deny under HTTP 200, a branded page under "
+ "HTTP 403, or the same page under HTTP 503 as a rate "
+ "limit): none carries a widget, so no solve helps "
+ "and none is attempted or billed. In fact NO "
+ "challenge has ever "
"been observed on this site; the path is wired up "
"because a bot manager can be switched on between "
"deploys.")
@@ -1857,7 +1884,7 @@ def parse_args():
"well as failure. Useful when the row count is right but "
"a column comes back empty — see TROUBLESHOOTING.md.")
# HEADLESS is the default, and here that is measured in BOTH directions
- # rather than inherited — §19's rule, because a sibling site answered 0
+ # rather than inherited — the rule, because a sibling site answered 0
# of 4 headless against 4 of 4 headful and the family default had been
# headless simply because headless is what a scraper does.
#
@@ -1907,7 +1934,7 @@ def parse_args():
# Rakuten Group property like ranking.rakuten.co.jp or Rakuten Books
# gets a different sentence from a typo, because "is not a Rakuten
# site" would be FALSE for those and would send the reader hunting
- # for a misspelling that is not there (§5).
+ # for a misspelling that is not there.
p.error(unsupported_reason(args.url)
or f"{args.url!r} is not a Rakuten Ichiba URL this scraper "
f"reads.")
diff --git a/product_parser.py b/product_parser.py
index 298f167..1461027 100644
--- a/product_parser.py
+++ b/product_parser.py
@@ -2,7 +2,7 @@
This module IS the site. Everything that knows what Rakuten publishes, how
it spells a product id, how it paginates and how it refuses lives here; the
-engines carry a dozen named constants and nothing else (§1).
+engines carry a dozen named constants and nothing else.
Where the data actually is
--------------------------
@@ -29,7 +29,7 @@
below).
A DETAIL page publishes a different `@type` again — its only JSON-LD is a
-`BreadcrumbList` (§20, proven here on a second site). The listing parser
+`BreadcrumbList` (proven here on a second site). The listing parser
finds nothing at all on a detail page, which is asserted in the suite rather
than left to be discovered.
@@ -82,7 +82,7 @@
HOSTS = (SEARCH_HOST, WWW_HOST, ITEM_HOST, "rakuten.co.jp")
# Hosts that belong to Rakuten Group but are NOT Rakuten Ichiba. Refused with
-# the reason rather than with a generic "unsupported host" — §5: telling a
+# the reason rather than with a generic "unsupported host" — telling a
# reader their URL "is not a Rakuten site" when it plainly is sends them
# hunting for a typo.
OTHER_RAKUTEN_HOSTS = {
@@ -124,19 +124,19 @@
#
# The listing payload states NO currency: zero occurrences of `priceCurrency`,
# `currency` or the string `JPY` across 405 rows. So it is not a fact read
-# off the row (§4 rung 1), and defaulting it would be exactly the "guess
-# presented as a fact" §8 forbids.
+# off the row, and defaulting it would be exactly the "guess
+# presented as a fact" this project never writes.
#
# What the site does state, on the same page: `"priceCurrency": "JPY"` in the
# listing page's own JSON-LD carousel block, and `` on every detail page — a written
-# ISO code, §4 rung 2. `currency_from_page` reads those. This constant is the
+# ISO code, the second-best source. `currency_from_page` reads those. This constant is the
# allowlist of what a read is allowed to return, not a default, and a row
# whose page states nothing gets `None`.
CURRENCY = "JPY"
CURRENCY_CODES = frozenset(("JPY",))
-# The positive structural signal (§8): a page Rakuten actually served is
+# The positive structural signal: a page Rakuten actually served is
# built out of Rakuten's own asset hosts, and an interstitial is not.
# Measured across six served pages and three refusals:
#
@@ -153,7 +153,7 @@
_SEARCH_PATH_RE = re.compile(r"^/search/mall/(?P[^/]+)/?")
SELECTORS = {
- # Anchored on the URL pattern, never a class (§4). Rakuten's own class
+ # Anchored on the URL pattern, never a class. Rakuten's own class
# names on the search grid are build hashes.
"item_link": 'a[href*="item.rakuten.co.jp/"]',
"tile_price": '[class*="price--"]',
@@ -172,7 +172,7 @@
# 150 is exactly the last: it comes back `start: 6705`, which is 6705 + 45 =
# 6750. Page 151 does not error — see `served_offset`.
#
-# This is §21's "complete and exhaustive are different words" with Rakuten's
+# This is the "complete and exhaustive are different words" with Rakuten's
# own arithmetic: one measured query reported `numFound: 3,053,682` against a
# `subset` of 6,750, so a full 150-page run is a complete run and a 0.2%
# sample. The sidecar records both numbers rather than only the status.
@@ -203,7 +203,7 @@
# site's rendered markup: `￥1,234` (fullwidth yen), `¥1,234` (ASCII yen) and
# `1,234円`. There are no decimal subunits in JPY, so a comma is always a
# thousands separator here and the "whichever comes last is the decimal
-# point" rule §4 describes does not apply — a bare `1,234` is 1234, never
+# point" rule does not apply — a bare `1,234` is 1234, never
# 1.234. Used only by the DOM fallback; the payload paths read integers.
_AMOUNT = (r"\d{1,3}(?:[,%s]\d{3})+|\d+" % _GROUP_SPACES)
_PRICE_RE = re.compile(
@@ -238,7 +238,7 @@
# needs neither arrives under HTTP 200. Neither signal is sufficient alone
# and each covers the other's blind spot.
#
-# Bounded to a prefix (§20): every refusal body is under 2 KB entire, so
+# Bounded to a prefix: every refusal body is under 2 KB entire, so
# matching in the first 8 KB covers all of them and cannot let a product
# title deep in a 981 KB grid read as a marker.
_MARKER_PREFIX_BYTES = 8192
@@ -246,7 +246,7 @@
# Whitespace-tolerant on purpose. The bytes Akamai actually sends are
# `Reference #18.536a645f.1789976769.70a9bb4f` — with TWO ASCII spaces. A
# literal `"Reference #"` marker matches none of the three refusals, which is
-# §20's "a marker must survive both spellings of the same page" arriving in a
+# the "a marker must survive both spellings of the same page" arriving in a
# third costume.
_AKAMAI_REF_RE = re.compile(
r"Reference\s*#\s*\d+\.[0-9a-f]+\.\d+\.[0-9a-f]+", re.I)
@@ -258,10 +258,10 @@
# Deliberately not carried, each for a measured reason:
#
-# cf-turnstile §8/§19 — the Scraping Browser's auto-solve extension
+# cf-turnstile — the Scraping Browser's auto-solve extension
# injects it into every page it loads, so it fires on good
# pages. Measured useless on two sibling sites.
-# akamai / akam §18 — a marker that matches every page is worse than no
+# akamai / akam — a marker that matches every page is worse than no
# marker. Rakuten is fronted by Akamai, and although its
# served markup happens to reference no Akamai host (0 of 6
# captures), a CDN name is a fact about the site's
@@ -269,7 +269,7 @@
# Reference # the literal spelling, for the two-space reason above.
# No captcha is configured anywhere on this site, and that is a measurement
-# rather than an assumption — §18's "did we meet one" is the wrong question,
+# rather than an assumption — the "did we meet one" is the wrong question,
# so the right one was asked: *is one configured, and would we recognise it
# if it appeared?*
#
@@ -282,7 +282,7 @@
# So this set exists to RECOGNISE an escalation, not because one was seen —
# and none of these strings appears on a served page, so none of them can
# fire on a good one. What the narrow, honest word "unsolvable" applies to
-# (§19) is that 43-byte deny page: it carries no widget, so there is nothing
+# is that 43-byte deny page: it carries no widget, so there is nothing
# on it for any solver at any price. That is a property of that page and says
# nothing about what 2Captcha can solve.
BOT_CHALLENGE_MARKERS = (
@@ -384,7 +384,7 @@ def site_host(url: str) -> Optional[str]:
def unsupported_reason(url: str) -> Optional[str]:
"""Why this URL is not an Ichiba URL this scraper reads, or None.
- Refused WITH the reason (§5). Every entry in `OTHER_RAKUTEN_HOSTS` is a
+ Refused WITH the reason. Every entry in `OTHER_RAKUTEN_HOSTS` is a
real Rakuten property, so "not a Rakuten site" would be false and would
send the reader looking for a typo they did not make.
"""
@@ -723,7 +723,7 @@ def served_offset(html: Any) -> Optional[int]:
Both answer 200 with a full, well-formed page of real products. Asking
for page 151 and being handed `start: 0` is the site stating outright
that it served page 1, so "asked for N, got something else" is an
- unambiguous end of listing (§23, on a second site and in a second
+ unambiguous end of listing (on a second site and in a second
spelling).
"""
return _int(pagination(html).get("start"))
@@ -846,7 +846,7 @@ def _normalize_amount(raw: str) -> Optional[float]:
def prices_in(text: str) -> List[float]:
"""Every JPY amount in a piece of rendered text, in order.
- Percentages are removed BEFORE matching, not rejected afterwards (§4):
+ Percentages are removed BEFORE matching, not rejected afterwards:
Japanese discount badges write `50%OFF` and `-10％` with both ASCII and
fullwidth signs, and a rejected match has still consumed its
neighbouring yen symbol.
@@ -884,7 +884,7 @@ def _without_extension_scripts(text: str) -> str:
def detect_block_marker(html: Any) -> Optional[str]:
"""Which refusal this is, or None.
- Matched over a bounded prefix and with entities unescaped, per §20 — an
+ Matched over a bounded prefix and with entities unescaped — an
edge can entity-escape the punctuation of its own reference id on the way
to an HTTP client while a browser parses it back out plain.
"""
@@ -907,7 +907,7 @@ def detect_bot_challenge(html: Any, url: str = "") -> Optional[str]:
measurement (see `BOT_CHALLENGE_MARKERS`) rather than a shortcut: the
scan runs, and it runs against the markup with extension-injected
`' % _k, "u")
+ got_a = _static('' % _k, "u")
+ ok &= check("the static detector finds a v2 widget by its container "
+ "class alone", got_c is not None and got_c.kind == "recaptcha_v2"
+ and got_c.sitekey == _k)
+ ok &= check("...and by its anchor frame alone, with the frame's size",
+ got_a is not None and got_a.sitekey == _k
+ and got_a.kind == "recaptcha_v2_invisible")
+ served = [n for n in FIX if isinstance(FIX[n], dict)
+ and isinstance(FIX[n].get("html"), str)
+ and _static(FIX[n]["html"], FIX[n].get("url", ""))]
+ ok &= check("...and fires on none of this site's captures (%s)"
+ % (", ".join(served) or "none"), not served)
+
# Deliberately absent: no solver for a first-party image captcha. This
- # site has no such page — measured 2026-09-10, its refusal is not a page
- # at all: the HTTP/2 stream is reset and nothing arrives — so a solver
+ # site has no such page — its refusals are a 43-byte Akamai deny and a
+ # branded 403/503 page, none with a form or an image on it — so a solver
# for one would be dead code that looks load-bearing. Pinned so that
# reintroducing it is a decision rather than a drift.
import captcha_solver
@@ -2283,7 +2374,7 @@ def test_captcha():
# policy: which challenge a visitor meets depends on the exit country and
# on what the address has been doing.
# THE MARKER SET IS THE MEASUREMENT, and on this site the measurement
- # is zero. §18's question is not "did we meet a captcha" but "is one
+ # is zero. the question is not "did we meet a captcha" but "is one
# CONFIGURED, and would we recognise it if it appeared?" — so it was
# asked properly: across all 13 captures, served and refused alike,
# there are no reCAPTCHA, hCaptcha, Turnstile, DataDome, PerimeterX,
@@ -2309,19 +2400,19 @@ def test_captcha():
# loads, so it fires on GOOD pages fetched over --cdp-endpoint — and on
# two sibling sites it was measured to MISS the real challenge while
# doing so. A marker that fires on good pages and is absent from the bad
- # one is worse than no marker (§19).
+ # one is worse than no marker.
ok &= check("cf-turnstile is not carried",
not any("cf-turnstile" in m for m in markers))
# A bare CDN or vendor NAME is not carried either. Rakuten is fronted by
# Akamai, and "this site uses Akamai" is a fact about its
- # infrastructure rather than a signal about this response (§18).
+ # infrastructure rather than a signal about this response.
ok &= check("a bare vendor/CDN name is not a marker",
not any(m in ("akamai", "akam", "edgesuite", "cloudflare")
for m in markers))
# NOT ONE of them may appear on a page the site served. This is the
# check that would have caught the family's `cf-turnstile` mistake, run
# against the fixtures that matter rather than against a 404 fetched by
- # curl (§21: a guard is only as good as the fixture it runs against).
+ # curl (a guard is only as good as the fixture it runs against).
for name in ("search_p1", "search_p1_browser", "category_p1",
"search_deep", "search_sponsored", "item_page",
"item_discounted", "search_no_results"):
@@ -2339,7 +2430,7 @@ def test_captcha():
detect_bot_challenge(fx("search_p1")[0] + EXTENSION_TAGS)
is None)
# AND THE SAME THING AGAINST A PAGE ACTUALLY FETCHED THAT WAY, which is
- # the fixture §21 says a guard needs: a sibling repo's version of this
+ # the fixture a guard needs: a sibling repo's version of this
# check ran only against a curl-fetched 404, which carries no injection
# at all, so it passed for the wrong reason.
#
@@ -2401,7 +2492,7 @@ def test_captcha():
ok &= check("...and its rows parse",
len(parse_products(cdp_html, cdp_url)) >= 3)
# And the honest sentence about the paid product, which is the one this
- # family has got wrong before (§19). What may be written is "this repo
+ # family has got wrong before. What may be written is "this repo
# does not implement X", never "X cannot be solved" — 2Captcha solves
# enterprise reCAPTCHA (RecaptchaV2EnterpriseTaskProxyless) and
# Cloudflare Turnstile (TurnstileTaskProxyless), whatever this client
@@ -2690,7 +2781,7 @@ def test_proxy_pool():
# was written to show. Every caller here passes the URL as its own `%s`
# argument for that reason, and `_mask_credentials()` is what handles
# arbitrary text. Asserting the CURRENT behaviour makes a future swap a
- # failing check rather than an unreadable log (§10).
+ # failing check rather than an unreadable log.
sentence = "a http://u:supersecret@h.example:8080 b"
ok &= check("mask() on a sentence loses the host — the documented limit",
mask(sentence) == "?://?")
@@ -2793,7 +2884,7 @@ def test_engines(skips):
# "Not painted yet" is not a fault, and every engine has to make that
# distinction the same way — the first live search run of the
# Playwright engine reported 0 rows and exit 4 because it did not.
- # §17's rule, applied to this repo's own additions: a policy
+ # the rule, applied to this repo's own additions: a policy
# constant or a helper that nothing consults is the same defect as
# dead code, and harder to see because the prose reads like
# enforcement. All three of these were written, documented in the
@@ -2857,7 +2948,7 @@ def test_engines(skips):
# THE FLAG CONTRACT, and the exact ways the engines differ from it.
#
- # §9 lists the flags every engine must offer. Checked rather than
+ # The family contract lists the flags every engine must offer. Checked rather than
# trusted, because a flag one engine has and another does not is the
# drift page_flow.py and finish_run() exist to prevent, one level up —
# and because the README documents these differences by name, so a new
@@ -2890,7 +2981,7 @@ def test_engines(skips):
# there is nothing left to except: the family's contract flags are
# in all three engines. A sibling repo shipped twelve flags the
# primary engine had and its twins did not, nine of them predating
- # the work, while its README promised "same CLI" (§20) — so a new
+ # the work, while its README promised "same CLI" — so a new
# unshared flag has to fail here.
#
# This closed two real gaps rather than documenting them: pyppeteer
@@ -2910,7 +3001,7 @@ def test_engines(skips):
# And a flag every engine declares must be one every engine READS. A
# flag that is accepted and then ignored looks configurable and is
# not, which is the defect this family keeps finding in its own
- # copied core (§17).
+ # copied core.
for eng in ("playwright_scraper", "puppeteer_scraper",
"selenium_scraper"):
esrc = open(os.path.join(REPO_ROOT, eng + ".py"),
@@ -2947,7 +3038,7 @@ def test_engines(skips):
if "args.fingerprint = False" in ln]
ok &= check("%s forces it off in exactly one place" % eng,
len(forced) == 1)
- # THE BEHAVIOUR, not a comment saying so — §20's rule that an
+ # THE BEHAVIOUR, not a comment saying so — the rule that an
# assertion which is a proxy for behaviour goes stale. Forcing
# `args.fingerprint = False` only gates the identity if EVERY
# path that applies one is itself behind that flag.
@@ -2957,7 +3048,7 @@ def test_engines(skips):
# and guard its CALL SITE. A version of this check that looked
# only for calls lexically inside an `if ... fingerprint` block
# reported those two as ungated, which was the check being wrong
- # rather than the code (§22: ask what a check does when its
+ # rather than the code (ask what a check does when its
# resolution fails).
etree = ast.parse(esrc)
applies = ("get_fingerprint", "_apply_fingerprint",
@@ -3015,7 +3106,7 @@ def _calls_in(node):
# `pages_still_growing` — so a Selenium run's sidecar was MISSING the
# cap arithmetic, which on this site is the figure that keeps
# `status: complete` honest: the site serves 6,750 of 3,000,000 matches,
- # and a sidecar that says only "complete" is lying by omission (§21).
+ # and a sidecar that says only "complete" is lying by omission.
#
# Compared as the SET OF KEYS each engine builds, read off the source,
# because the alternative is three live runs and a diff — which is how
@@ -3067,14 +3158,14 @@ def _calls_in(node):
ok &= check("%s merges page 1's cap summary into the sidecar" % eng,
"page_one_cap" in esrc
and "extra.update(page_one_cap)" in esrc)
- # A dead dataclass field is the same defect as a dead column (§9).
+ # A dead dataclass field is the same defect as a dead column.
ok &= check("%s has no dead `scroll` outcome field" % eng,
"scroll: Optional" not in esrc)
# `--fp-tags` MUST DEFAULT TO ONE OS-FAMILY TAG. It shipped in this
# family as "Windows,Chrome,Desktop", which the fingerprint API rejects
# with HTTP 400 — so --fingerprint failed on every invocation, which is
- # one of the six defects §16 of the family notes lists. Measured
+ # one of the six defects found in the copied fingerprint client. Measured
# 2026-09-10 against the live API: `Windows` succeeds;
# `Windows,Chrome,Desktop`, `Chrome` and `Desktop` each 400.
for name in _ENGINE_MODULES:
@@ -3304,7 +3395,7 @@ def _unreachable_statements(path):
def test_public_names_have_consumers():
group("no public name is dead code or unenforced policy")
ok = True
- # §17 #5: grep every public name in a shared module for a consumer
+ # Grep every public name in a shared module for a consumer
# outside its own module, then READ what comes back. The ones that
# encode SITE POLICY rather than plumbing are pinned here, so deleting
# one fails instead of quietly changing behaviour — which is the failure
@@ -3353,7 +3444,7 @@ def test_public_names_have_consumers():
ok &= check("SOURCE_DEFAULT is what a row's `source` actually says",
Product().source == "rakuten.co.jp")
# THE THROTTLE POLICY IS CONSULTED, not merely declared. This is the
- # §17/§23 defect at its most expensive: `SOLVES_PER_PAGE` read like an
+ # The unenforced-cap defect at its most expensive: `SOLVES_PER_PAGE` read like an
# enforced cap in every repo of this family while each engine called the
# solver twice per attempt and counted once.
ok &= check("the throttle budget and its backoff agree in length",
@@ -3370,11 +3461,19 @@ def test_public_names_have_consumers():
ok &= check("%s consults RETRY_ON_BLOCKED rather than only "
"documenting it" % eng, "RETRY_ON_BLOCKED" in src)
# Count the solve call sites, the guards and the increments, and
- # assert the three are equal. That is the check §23 asks for by
- # name, and it is what a reading of the code does not give you.
+ # assert the three are equal: a call site the budget does not cover
+ # can buy a solve the cap was meant to refuse. The pre-classification
+ # site counts ATTEMPTED purchases (the `purchases` list the handler
+ # appends to before asking the solver), so a solver that raises
+ # cannot be asked twice for one page.
calls = src.count("handle_captcha_if_present(")
guards = src.count("page_flow.solve_budget(")
- bumps = src.count("solves_bought += 1")
+ bumps = (src.count("solves_bought += 1")
+ + src.count("solves_bought += len(purchases)"))
+ ok &= check("%s counts an attempted purchase, not only a successful "
+ "one, at the pre-classification call" % eng,
+ "purchases=purchases" in src
+ and "purchases.append(" in src)
# One of the "calls" is the function's own def.
ok &= check("%s: every solve call site is behind the budget "
"(%d call(s), %d guard(s), %d increment(s))"
@@ -3383,6 +3482,289 @@ def test_public_names_have_consumers():
return ok
+
+def _load_engine_with_stub_driver(eng):
+ """Import an engine even when its driver library is absent.
+
+ The fetch loop is pure Python around a handful of driver calls, so it can
+ be driven with a fake page. When the real driver is installed it is used;
+ otherwise a stub module stands in for the import only, and both the stub
+ and the engine are removed from sys.modules afterwards so the engine
+ groups elsewhere in this suite still see the library as absent.
+ """
+ import importlib
+ import types
+ stubs = {}
+ if eng == "playwright_scraper":
+ try:
+ import playwright.sync_api # noqa: F401
+ except ImportError:
+ api = types.ModuleType("playwright.sync_api")
+
+ class Error(Exception):
+ pass
+
+ class TimeoutError(Error): # noqa: A001 — mirrors the real name
+ pass
+ api.Error, api.TimeoutError = Error, TimeoutError
+ api.sync_playwright = lambda: None
+ stubs = {"playwright": types.ModuleType("playwright"),
+ "playwright.sync_api": api}
+ elif eng == "puppeteer_scraper":
+ try:
+ import pyppeteer # noqa: F401
+ except ImportError:
+ mod = types.ModuleType("pyppeteer")
+ mod.launch = mod.connect = lambda *a, **k: None
+ stubs = {"pyppeteer": mod}
+ had = eng in sys.modules
+ if had and not stubs:
+ return sys.modules[eng]
+ saved = dict((k, sys.modules.get(k)) for k in stubs)
+ sys.modules.update(stubs)
+ try:
+ sys.modules.pop(eng, None)
+ engine = importlib.import_module(eng)
+ finally:
+ for k, v in saved.items():
+ if v is None:
+ sys.modules.pop(k, None)
+ else:
+ sys.modules[k] = v
+ if stubs and not had:
+ sys.modules.pop(eng, None)
+ return engine
+
+
+class _LoopFakeResponse:
+ def __init__(self, status):
+ self.status = status
+
+
+class _LoopFakePage:
+ """Answers the driver calls both browser engines' fetch loops make.
+
+ pyppeteer's calls go through `bridge.run(...)`, which the fake bridge
+ below passes straight through, so plain return values serve both.
+ """
+ def __init__(self, url, html, status=200, goto_error=None):
+ self.url, self._html, self._status = url, html, status
+ self._goto_error = goto_error
+ self.gotos = 0
+
+ def goto(self, url, *a, **k):
+ self.gotos += 1
+ if self._goto_error is not None:
+ raise self._goto_error
+ self.url = url
+ return _LoopFakeResponse(self._status)
+
+ def content(self):
+ return self._html
+
+ def query_selector_all(self, sel):
+ return []
+
+ querySelectorAll = query_selector_all
+
+ def wait_for_timeout(self, ms):
+ return None
+
+ def evaluate(self, *a, **k):
+ return None
+
+ def reload(self, *a, **k):
+ return _LoopFakeResponse(self._status)
+
+
+class _LoopFakeBridge:
+ def run(self, value):
+ return value
+
+
+class _LoopFakeSession:
+ def __init__(self, page):
+ self.page, self.bridge, self.relaunches = page, _LoopFakeBridge(), 0
+
+ def relaunch(self):
+ self.relaunches += 1
+
+
+def test_engine_fetch_loop_offline():
+ group("browser engines' fetch loop, driven offline with a fake page")
+ ok = True
+ import types
+ import argparse as _ap
+ import time as _time
+ import page_flow as _pf
+ no_sleep = types.SimpleNamespace(sleep=lambda s: None, time=_time.time,
+ monotonic=_time.monotonic)
+ blk_html, blk_url, _ = fx("block_403")
+ for eng in ("playwright_scraper", "puppeteer_scraper"):
+ try:
+ engine = _load_engine_with_stub_driver(eng)
+ except Exception as e: # noqa: BLE001
+ ok &= check("%s loads with a stub driver (%s)" % (eng, e), False)
+ continue
+ real_time = engine.time
+ engine.time = no_sleep
+ try:
+ with tempfile.TemporaryDirectory() as d:
+ args = _ap.Namespace(
+ pages=1, retries=1, retry_delay=0.0, mode="listing",
+ out=os.path.join(d, "out"), dump_html=None,
+ proxy_block_retries=0, solve_captcha="when-blocked",
+ twocaptcha_key="", captcha_api="v2", min_score=0.7)
+
+ # 1. No proxy at all, and every navigation fails. Must end
+ # as a load failure, never as an AttributeError on the
+ # absent pool.
+ err_cls = getattr(engine, "PWTimeout", Exception)
+ page = _LoopFakePage(blk_url, "",
+ goto_error=err_cls("Navigation Timeout Exceeded"))
+ try:
+ out = engine._fetch_one_page(_LoopFakeSession(page), args,
+ None, 1, blk_url)
+ raised = None
+ except Exception as e: # noqa: BLE001
+ out, raised = None, e
+ ok &= check("%s: repeated load failures with no proxy pool "
+ "end as a load failure, not a crash (%r)"
+ % (eng, raised),
+ raised is None and out.load_failed)
+
+ # 2. The site's branded refusal under HTTP 403 is a BLOCK.
+ # Without the status it would read as a 503 throttle and
+ # be waited out at the same exit.
+ page = _LoopFakePage(blk_url, blk_html, status=403)
+ out = engine._fetch_one_page(_LoopFakeSession(page), args,
+ None, 1, blk_url)
+ ok &= check("%s: the branded page under HTTP 403 is "
+ "classified blocked, so the status reached the "
+ "classifier (state=%s)" % (eng, out.state),
+ out.state == "blocked"
+ and out.blocked_by is not None)
+ page = _LoopFakePage(blk_url, blk_html, status=503)
+ out = engine._fetch_one_page(_LoopFakeSession(page), args,
+ None, 1, blk_url)
+ ok &= check("%s: the same page under HTTP 503 is a throttle, "
+ "waited out rather than reported blocked "
+ "(state=%s, %d fetches)" % (eng, out.state,
+ page.gotos),
+ out.state == "throttled"
+ and page.gotos == 1 + _pf.THROTTLE_RETRIES)
+
+ # 3. A solver that RAISES still counts against the page's
+ # allowance: one page, a challenge on every fetch, a
+ # solver that fails every time -> at most SOLVES_PER_PAGE
+ # calls to it.
+ calls = []
+
+ def _solver(*a, **k):
+ calls.append(1)
+ raise RuntimeError("stub solver failure")
+ fake_challenge = types.SimpleNamespace(
+ kind="recaptcha_v2", source="html", sitekey="stub",
+ action=None)
+ saved = (engine.solve_recaptcha, engine.reconcile_detections,
+ _pf.classify)
+ engine.solve_recaptcha = _solver
+ engine.reconcile_detections = lambda *a: fake_challenge
+ _pf.classify = lambda *a, **k: "challenge"
+ args.twocaptcha_key = "stub-not-sent"
+ try:
+ page = _LoopFakePage(blk_url, "", status=200)
+ engine._fetch_one_page(_LoopFakeSession(page), args, None, 1,
+ blk_url)
+ finally:
+ (engine.solve_recaptcha, engine.reconcile_detections,
+ _pf.classify) = saved
+ args.twocaptcha_key = ""
+ ok &= check("%s: a failing solver is asked at most "
+ "SOLVES_PER_PAGE times for one page (%d call(s))"
+ % (eng, len(calls)),
+ 0 < len(calls) <= _pf.SOLVES_PER_PAGE)
+ finally:
+ engine.time = real_time
+
+ # Every classify call in every engine names a status argument, even
+ # Selenium's, which passes None on purpose because WebDriver has none.
+ import ast as _ast
+ for eng in ("playwright_scraper", "puppeteer_scraper", "selenium_scraper"):
+ tree = _ast.parse(open(os.path.join(REPO_ROOT, eng + ".py"),
+ encoding="utf-8").read())
+ bad, seen = [], 0
+ for node in _ast.walk(tree):
+ if not isinstance(node, _ast.Call):
+ continue
+ f = node.func
+ name = getattr(f, "attr", None) or getattr(f, "id", None)
+ if name not in ("classify", "_classify"):
+ continue
+ seen += 1
+ npos = len(node.args) - (1 if name == "_classify" else 0)
+ if npos < 2 and not any(k.arg == "status" for k in node.keywords):
+ bad.append(node.lineno)
+ ok &= check("%s: every classify call passes a status (%d call(s), "
+ "missing at lines %s)" % (eng, seen, bad),
+ seen > 0 and not bad)
+
+ # Numeric flags are validated at the command line, by argparse, in every
+ # entry point: read off disk so the check runs with no driver installed.
+ want = {"--pages": "at_least_one", "--concurrency": "at_least_one",
+ "--delay": "non_negative_seconds",
+ "--retry-delay": "non_negative_seconds",
+ "--proxy-block-retries": "non_negative_int",
+ "--retries": "attempts"}
+ api_want = {"--retries": "non_negative_int",
+ "--retry-delay": "non_negative_int",
+ "--timeout": "at_least_one"}
+ for eng, table in (("playwright_scraper", want),
+ ("puppeteer_scraper", want),
+ ("selenium_scraper", want),
+ ("scraper_api_client", api_want)):
+ tree = _ast.parse(open(os.path.join(REPO_ROOT, eng + ".py"),
+ encoding="utf-8").read())
+ types_ = {}
+ for node in _ast.walk(tree):
+ if (isinstance(node, _ast.Call)
+ and getattr(node.func, "attr", None) == "add_argument"
+ and node.args and isinstance(node.args[0], _ast.Constant)):
+ for k in node.keywords:
+ if k.arg == "type":
+ types_[node.args[0].value] = getattr(
+ k.value, "attr", getattr(k.value, "id", None))
+ wrong = sorted(f for f, t in table.items() if types_.get(f) != t)
+ ok &= check("%s validates its numeric flags with page_flow's "
+ "validators (wrong: %s)" % (eng, wrong or "none"),
+ not wrong)
+ import argparse as _ap2
+ probe = _ap2.ArgumentParser()
+ probe.add_argument("--pages", type=_pf.at_least_one)
+ probe.add_argument("--delay", type=_pf.non_negative_seconds)
+ probe.add_argument("--proxy-block-retries", type=_pf.non_negative_int)
+
+ def _rejects(argv, flag):
+ err = io.StringIO()
+ try:
+ with redirect_stderr(err):
+ probe.parse_args(argv)
+ except SystemExit as e:
+ return e.code == 2 and ("argument %s" % flag) in err.getvalue()
+ return False
+ ok &= check("--pages 0 is a usage error naming the flag",
+ _rejects(["--pages", "0"], "--pages"))
+ ok &= check("--delay -5 is a usage error naming the flag",
+ _rejects(["--delay", "-5"], "--delay"))
+ ok &= check("--proxy-block-retries -1 is a usage error naming the flag",
+ _rejects(["--proxy-block-retries", "-1"],
+ "--proxy-block-retries"))
+ ok &= check("...while --pages 1, --delay 0 and --proxy-block-retries 0 "
+ "parse", probe.parse_args(["--pages", "1", "--delay", "0",
+ "--proxy-block-retries", "0"]).pages == 1)
+ return ok
+
+
def test_log_format_strings_match_their_args():
group("every log call's placeholders match its arguments")
ok = True
@@ -3450,7 +3832,7 @@ def test_every_shared_call_binds():
group("every call into a shared module binds, including the paths a "
"credential gates")
ok = True
- # §17's check #1, widened to the case that matters most here: the
+ # the check #1, widened to the case that matters most here: the
# credential-gated modules. `fingerprint_client`, `scraper_api_client`
# and `captcha_solver` are reached only with a 2Captcha key, and no key
# was available to the work that built this repo — so a wrong call in
@@ -3528,7 +3910,7 @@ def test_every_shared_call_binds():
# A name the module does NOT define is the loudest thing this
# check can say, so it must not be skipped — a sibling repo
# resolved with getattr(..., None), skipped anything
- # not-callable, and stayed silent on exactly that case (§22).
+ # not-callable, and stayed silent on exactly that case.
if target is None:
bad.append("%s:%d %s.%s does not exist"
% (caller, node.lineno, module_name, real))
@@ -3593,7 +3975,7 @@ def test_sponsored_slots_do_not_shift_position():
# a click-tracking URL yields no sku and `_row_from_hit` drops the entry
# anyway — so that assertion alone would pass against a broken filter.
# (Found by planting exactly that fault and watching the suite stay
- # green; §22's "verify that your control actually failed".)
+ # green; the "verify that your control actually failed".)
#
# A sponsored entry wearing a product-shaped URL is the case only the
# filter catches, and Rakuten already puts `cpc` on every one of them.
@@ -3627,7 +4009,7 @@ def test_sponsored_slots_do_not_shift_position():
ok &= check("position starts at 1 even though the payload starts with ads",
rows[0].position == 1)
# page+position must be unique across a multi-page run — one line, and
- # the column is worthless without it (§18).
+ # the column is worthless without it.
multi = parse_products(html, url, page=1) + parse_products(html, url,
page=2)
pairs = [(r.page, r.position) for r in multi]
@@ -3683,7 +4065,7 @@ def test_detail_pages():
# A WAS-PRICE, and only where Rakuten's own verification flag allows one.
# Japan's double-pricing rules require a merchant's reference price to be
# substantiated, and the site carries the outcome as a flag — which is
- # the "use the site's own should-I-show-this flag" rule (§21).
+ # the "use the site's own should-I-show-this flag" rule.
ok &= check("a verified reference price becomes original_price",
row.original_price == 10398.0)
ok &= check("and WHICH reference price it was is recorded",
@@ -3691,7 +4073,7 @@ def test_detail_pages():
ok &= check("the discount is computed from the two prices",
row.discount_pct == 50.0)
# The other fixture has the flag FALSE, so no was-price at all — both
- # values of the flag are exercised, which §20 asks for.
+ # values of the flag are exercised.
plain = parse_product_page(*_fx_detail("item_page"))[0]
ok &= check("an UNVERIFIED reference price is not read",
plain.original_price is None
@@ -3824,7 +4206,7 @@ def test_parse_is_gated_on_the_policy():
# engine parsed whatever reached the parse line, so the `parse` column
# of STATE_POLICY decided nothing and an engine could disagree with the
# table — and with its twins — without anything noticing. That is the
- # defect CLAUDE.md §17 names for constants, with a function instead.
+ # defect of a policy constant nothing reads, with a function instead.
#
# Measured across the family on 2026-09-23 by counting definitions
# against readers: 7 of 24 repos defined it and none called it.
@@ -3957,6 +4339,7 @@ def main() -> int:
ok &= test_every_shared_call_binds()
ok &= test_x_debug_header_is_redacted()
ok &= test_scraper_api_waitfor_is_an_object()
+ ok &= test_engine_fetch_loop_offline()
print()
if _failures: