From 20ba803762044727a82f7a1ea9bdd826de653151 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:04:30 +0000 Subject: [PATCH] Read the newer EMI line, test the parser in the canary, validate numeric flags - The EMI figure is read whether or not the site prints "/m", and the "Save extra ... on" promotion beside it is no longer taken for the tag; before this, the newer card shape gave no EMI and put "EMI 15,916" in the hub column (canary runs of 2026-10-08, -10 and -15). - The canary asserts parser invariants instead of an EMI share and a strike/discount pairing, its checks live in .github/canary_check.py, and exit 5 (content never obtained) now fails instead of warning. - --pages, --concurrency, --proxy-block-retries, --delay and --retry-delay reject out-of-range values with exit 2 in all three engines (--retry-delay -1 used to crash the first retry); scraper_api_client.py validates --timeout, --retries and --retry-delay. - The credential scan compares excluded names relative to the repo root, fails when it scanned nothing, and reads .env backups, JSON, CSV, HTML and page dumps. - Root HTML dumps, NAME.pageN dumps and live/out/tmp-style directories are gitignored. - Playwright and pyppeteer pass the navigation's HTTP status to the classifier. - Comments and docs state rules in place instead of citing private notes by section; a suite guard keeps it that way. Co-Authored-By: Claude Opus 5.5 --- .github/canary_check.py | 258 ++++++++++++++++++++++++++++ .github/ci_checks.py | 42 ++++- .github/workflows/canary.yml | 255 +++------------------------- .gitignore | 21 ++- CHANGELOG.md | 42 +++++ CONTRIBUTING.md | 5 +- README.md | 12 +- captcha_solver.py | 4 +- env_config.py | 5 +- fingerprint_client.py | 5 +- output_writer.py | 9 +- page_flow.py | 77 +++++++-- playwright_scraper.py | 49 +++--- product_parser.py | 108 +++++++----- puppeteer_scraper.py | 22 ++- scraper_api_client.py | 10 +- selenium_scraper.py | 10 +- smoke_test.py | 315 ++++++++++++++++++++++++++++++++--- 18 files changed, 865 insertions(+), 384 deletions(-) create mode 100644 .github/canary_check.py diff --git a/.github/canary_check.py b/.github/canary_check.py new file mode 100644 index 0000000..b4a7f01 --- /dev/null +++ b/.github/canary_check.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +""" +The canary's data assertions, run against a finished scrape: + + python3 .github/canary_check.py canary_run + +reads `canary_run.json` and `canary_run.meta.json` and exits 1 with one +`::error::` line per failed assertion. + +It lives in a file rather than in the workflow so it can be RUN outside CI: +against a downloaded canary artefact, or by the offline suite. A heredoc in a +workflow is executed nowhere else, which makes it the least-tested code in +the repository, and it is exactly the code that decides whether the badge is +red. + +WHAT IS ASSERTED, AND WHAT IS DELIBERATELY NOT +---------------------------------------------- +Every check here tests the PARSER or the RUN — row count, fields every card +prints, invariants between two reads of the same card, the scroll and the +sidecar. None of them tests what Spinny chooses to show this week. + +Two floors used to, and both turned the badge red on runs that were complete +and fully priced (2026-10-08, -09, -10 and -15): + +* "emi_monthly on at least 90% of rows". The site changed the EMI line's + wording on those days, and the parser missed the new wording, so the floor + did catch a real parser miss, but it fails just as hard on a day the site + stops printing an EMI at all. It is replaced by the invariant that matters + to the parser: an EMI line never lands in another column (hub or tag), + which is what the miss actually did to the rows. +* "a strike price always travels with the site's own discount figure". On + 2026-10-10 the site printed a strike price without its rupee badge on 20 + of 82 cards. Whether a badge is printed is a merchandising choice. What is + kept is the arithmetic: where both ARE printed, they must agree. +""" + +import json +import re +import sys +from pathlib import Path + +BATCHES_REQUESTED = 3 +# Measured 2026-09-11: --pages 3 returned 62 cars, three times, through three +# different engines; 60 to 102 on the canary runs of October. The floor is +# two batches' worth, so a slow round cannot trip it. +MIN_ROWS = 40 +# Every card prints a price: 705 of 705 across three captures, and every row +# of every canary run. +MIN_PRICED_SHARE = 0.98 + +# Fields every card prints, so any of them collapsing means an anchor moved. +# Each is a fact about the CAR (or the row's own bookkeeping), not a +# promotion. `emi_monthly`, `original_price` and `discount_amount` are not +# here on purpose: see the module docstring. +CORE_FIELDS = (("title", 0.98), ("sku", 1.0), ("url", 1.0), + ("brand", 0.95), ("model", 0.95), ("variant", 0.95), + ("year", 0.95), ("currency", 0.98), ("category", 1.0), + ("km_driven", 0.95), ("fuel_type", 0.95), + ("transmission", 0.95), ("rto", 0.95), + ("assurance", 0.95), ("image_url", 0.98), + ("page", 1.0), ("position", 1.0)) + +DETAIL_ONLY = ("km_driven_exact", "owners", "registration_year", + "registration_month", "color", "seating_capacity", + "insurance_validity", "insurance_type", "in_stock") + +# Text that belongs to the EMI box. If either shape turns up as a hub or a +# tag, the EMI read has missed a wording again. +_EMI_LIKE = re.compile(r"^(EMI\b|Save\s+extra\b)", re.I) + + +def check(rows, meta): + fail = [] + n = len(rows) + + if n < MIN_ROWS: + fail.append(f"only {n} rows, expected at least {MIN_ROWS} " + f"from {BATCHES_REQUESTED} batches of 20") + + skus = [r.get("sku") for r in rows] + if len(set(skus)) != len(skus): + fail.append(f"{len(skus) - len(set(skus))} duplicate sku(s): one " + f"scrolled page can re-render a card as it hydrates, and " + f"the writer is meant to drop the repeats") + + priced = sum(1 for r in rows if r.get("price") is not None) + if n and priced / n < MIN_PRICED_SHARE: + fail.append(f"only {priced}/{n} rows have a price " + f"(floor {MIN_PRICED_SHARE:.0%}): every card prints one, " + f"so the price node moved") + + # A listing page carries no structured price for a car, so the price + # source is always the card's own node, with or without the attribute. + sources = {r.get("price_source") for r in rows} + if sources - {"dom+attr", "dom"}: + fail.append(f"listing rows carry price_source {sorted(sources)}; only " + f"'dom+attr' and 'dom' are possible on a listing page") + + # Two price BASES: the displayed figure excludes RC transfer and + # insurance, the site's own data-price attribute includes them, so the + # attribute is the higher of the two (482 of 482 measured cards). + pairs = [(r["price"], r["price_all_in"]) for r in rows + if r.get("price") is not None + and r.get("price_all_in") is not None] + crossed_basis = [p for p, a in pairs if a < p] + if pairs and len(crossed_basis) / len(pairs) > 0.02: + fail.append(f"{len(crossed_basis)}/{len(pairs)} rows have an all-in " + f"price BELOW the displayed one: the two price bases have " + f"been crossed") + if len(pairs) < 0.9 * n: + fail.append(f"only {len(pairs)}/{n} rows carry both price bases: the " + f"data-price attribute is on every card, so it moved") + + for field, floor in CORE_FIELDS: + got = sum(1 for r in rows if r.get(field) not in (None, "")) + if n and got / n < floor: + fail.append(f"{field} present on only {got}/{n} rows " + f"(floor {floor:.0%})") + + # `hub` is sparse by nature (10 of 705 cards print none), so the floor + # is well under 100%. + hubs = sum(1 for r in rows if r.get("hub")) + if n and hubs / n < 0.80: + fail.append(f"hub present on only {hubs}/{n} rows: about 2% of cards " + f"legitimately print none, so the hub read has broken") + + # The EMI box must not leak into the free-text columns. + leaked = [(r.get("sku"), col, r.get(col)) for r in rows + for col in ("hub", "tag", "variant") + if r.get(col) and _EMI_LIKE.match(str(r.get(col)))] + if leaked: + fail.append(f"{len(leaked)} row(s) carry EMI-box text in another " + f"column, e.g. {leaked[:2]}: the EMI read missed a wording") + + # Where the site prints BOTH a strike price and its own rupee discount, + # strike - price must equal the discount (the badge rounds at most to a + # thousand). Whether it prints them is the site's choice; that they agree + # when it does is the parser's job. + disagree = [r.get("sku") for r in rows + if r.get("price") is not None + and r.get("original_price") is not None + and r.get("discount_amount") is not None + and abs((r["original_price"] - r["price"]) + - r["discount_amount"]) > 1000] + if disagree: + fail.append(f"{len(disagree)} row(s) where strike minus price does " + f"not match the site's own discount figure: " + f"{disagree[:3]}") + + # No car-level rating or review count is published; if either fills up, + # Spinny's OWN aggregate rating is being read into a car's row. + for field in ("rating", "review_count"): + got = sum(1 for r in rows if r.get(field) is not None) + if got: + fail.append(f"{got}/{n} rows have `{field}`: the site publishes " + f"none per car") + + for field in DETAIL_ONLY: + got = sum(1 for r in rows if r.get(field) is not None) + if got: + fail.append(f"{got}/{n} listing rows have `{field}`, which only " + f"--mode detail can know") + + crossed = [r["sku"] for r in rows + if r.get("original_price") and r.get("price") + and r["original_price"] <= r["price"]] + if crossed: + fail.append(f"{len(crossed)} row(s) have an original_price at or below " + f"their price: {crossed[:3]}") + bad_discount = [r["sku"] for r in rows + if r.get("discount_pct") is not None + and not (0 < r["discount_pct"] < 100)] + if bad_discount: + fail.append(f"{len(bad_discount)} row(s) have a discount outside " + f"(0, 100): {bad_discount[:3]}") + + bad_sku = [r.get("sku") for r in rows + if not (r.get("sku") or "").isdigit() + or (r.get("url") or "").rstrip("/").rsplit("/", 1)[-1] + != r.get("sku")] + if bad_sku: + fail.append(f"{len(bad_sku)} sku(s) are not the car's numeric id from " + f"its own URL: {bad_sku[:3]}") + + keys = [(r.get("page"), r.get("position")) for r in rows] + if len(set(keys)) != len(keys): + fail.append(f"{len(keys) - len(set(keys))} row(s) share a (page, " + f"position) pair: the scroll's batch boundaries are not " + f"reaching the parser") + + currencies = {r.get("currency") for r in rows if r.get("currency")} + if currencies - {"INR"}: + fail.append(f"rows carry a currency other than INR " + f"({sorted(currencies)}); this site quotes one") + + if meta.get("pages_completed", 0) < BATCHES_REQUESTED: + fail.append(f"the run holds {meta.get('pages_completed')} batches of " + f"the {BATCHES_REQUESTED} requested (stop_reason=" + f"{meta.get('stop_reason')!r}): the lazy-load scroll is " + f"not reaching further into the grid") + batches_seen = {r.get("page") for r in rows} + if len(batches_seen) < 2: + fail.append(f"every row claims batch {sorted(batches_seen)}: the " + f"scroll's boundaries are not being recorded") + if meta.get("status") != "complete": + fail.append(f"run status is {meta.get('status')!r}, not 'complete'") + if meta.get("mode") != "listing": + fail.append(f"run mode is {meta.get('mode')!r}, not 'listing'") + + scroll = meta.get("scroll") or {} + if not scroll: + fail.append("the sidecar records no scroll at all: the lazy-load loop " + "did not run, and on this site the grid then never " + "hydrated") + elif not (scroll.get("settled") or scroll.get("reached_target")): + fail.append("the scroll neither settled nor reached its target, so " + "this run is a floor rather than the listing") + + # The advertised total moves between runs, so what is asserted is that + # it EXISTS and is sane, not its value. + comp = meta.get("completeness") or {} + advertised = comp.get("total_available") + if advertised is None: + fail.append("the listing's own advertised total was not read: the " + "heading node ([data-componentname=HeadingContent]) has " + "moved") + elif advertised < n: + fail.append(f"the site advertises {advertised} cars and this run holds " + f"{n}: the total is being read from the wrong node") + + # Reported, never asserted: these follow the site's merchandising. + emi = sum(1 for r in rows if r.get("emi_monthly") is not None) + struck = sum(1 for r in rows if r.get("original_price") is not None) + summary = (f"[canary] {n} rows, {priced} priced, {hubs} with a hub, " + f"{emi} with an EMI, {struck} with a strike price, advertised " + f"{advertised}, status {meta.get('status')!r}, stop_reason " + f"{meta.get('stop_reason')!r}") + return fail, summary + + +def main(argv): + prefix = argv[1] if len(argv) > 1 else "canary_run" + try: + rows = json.loads(Path(prefix + ".json").read_text(encoding="utf-8")) + meta = json.loads(Path(prefix + ".meta.json").read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + print(f"::error::the run's output could not be read ({e}): a run that " + f"exited 0 must have written both files") + return 1 + fail, summary = check(rows, meta) + for line in fail: + print(f"::error::{line}") + print(summary) + return 1 if fail else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.github/ci_checks.py b/.github/ci_checks.py index c827f22..894d8ec 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -92,7 +92,35 @@ HEX32_ALLOWED = ("sha", "hash", "nonce", "example", "md5", "digest", "checksum", "assets.spinny.com") -SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example") +SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example", + # Captures and data: a raw page dump, a --dump-html + # snapshot or a JSON fixture is where a front-end key or a + # pasted credential actually lands. + ".json", ".csv", ".html", ".htm", ".log", ".cfg", ".ini", + ".toml", ".sh") + +# Directory NAMES that are never scanned. Tested against the path RELATIVE to +# the repo root: tested against the absolute path, a clone that happens to +# live under a directory called `venv` (or any other name here) scanned 0 +# files and reported "nothing credential-shaped". +SKIPPED_DIRS = {".git", "__pycache__", ".venv", "venv", ".pytest_cache"} + +# A --dump-html run over several pages writes `.page2`, `.page3`, +# whatever the base name's own suffix was. +_PAGE_DUMP = re.compile(r"\.page[0-9]+$") + + +def _is_scanned_name(path): + """True for a file whose NAME says it may hold a credential or a capture.""" + name = path.name + # `.env`, `.env.bak`, `.env.local`, `.env.save`: a working copy renamed the + # way people actually rename them. Path.suffix of ".env.bak" is ".bak", + # so a suffix list alone never reads one. + if name.startswith(".env"): + return True + if _PAGE_DUMP.search(name): + return True + return path.suffix.lower() in SCANNED_SUFFIXES # A directory holding `pyvenv.cfg` is a virtualenv, whatever it is called. @@ -103,7 +131,7 @@ # into pip's vendored code and flags a 32-hex string in `_elffile.py` as # key-shaped. Correct about the string, wrong about the file — and it is the # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people -# have to argue with is one they learn to suppress (CLAUDE.md §22). +# have to argue with is one they learn to suppress. # # Deliberately NOT a narrowing of what gets scanned: an untracked file is # still read, because a key pasted into a scratch file beside the scripts is @@ -132,10 +160,9 @@ def _is_in_virtualenv(path): def scanned_files(): for path in sorted(REPO.rglob("*")): - if not path.is_file() or path.suffix not in SCANNED_SUFFIXES: + if not path.is_file() or not _is_scanned_name(path): continue - if any(part in {".git", "__pycache__", ".venv", "venv"} - for part in path.parts): + if any(part in SKIPPED_DIRS for part in path.relative_to(REPO).parts): continue if _is_in_virtualenv(path): continue @@ -227,6 +254,11 @@ def secret_check(): failed.append(f"{rel}:{lineno} contains {match[:6]}… — a " f"32-char hex string, the shape of a 2captcha key") + if not scanned: + # A scan that read nothing has proved nothing. Reported as a failure + # so a path rule that silently excludes the whole tree cannot pass. + failed.append("0 files scanned — the exclusion rules matched the " + "whole repository, so nothing was checked") if not failed: print(f"ok {scanned} files scanned, nothing credential-shaped") return failed diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 8f05ea6..0a23136 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -85,8 +85,8 @@ jobs: - name: Interpret the exit code id: verdict - # ACCESS CONDITIONS ARE NOT DEFECTS — but on THIS site almost nothing - # is an access condition, which changes what the badge means. + # ONLY A RUN THAT TESTED THE SITE ENDS GREEN. Exit 0 is the one code + # that passes, and only if the data assertions below pass too. # # A sibling repo's canary treats exit 3 as a warning because its site # refuses datacentre addresses as a matter of course. Spinny has never @@ -96,21 +96,24 @@ jobs: # honest fix is to change this and say why, not to leave a check that # is amber whatever happens. # - # Exit 5 stays a warning: it can only happen when the optional - # Scraping Browser secret is set, and `profile_locked` says nothing - # about the site. + # Exit 5 is "the content was never obtained": a navigation timeout, a + # dead connection, or a refused Scraping Browser endpoint when the + # optional secret is set. None of those tested the site, so none of + # them may end green. If the log says `profile_locked`, another run + # holds the pid; if the endpoint answered 401, the secret has expired + # and should be removed, since this canary does not need it. run: | set -e code="${{ steps.run.outputs.exit_code }}" - tested=true + tested=false case "$code" in 0) + tested=true echo "OK — cars parsed from a live run." ;; 5) - tested=false - echo "::warning::Remote API error (exit 5) — the optional Scraping Browser endpoint refused the connection, so nothing was tested and this run is not failed. If the log says 'profile_locked', another run still holds this pid: give CI its own. Otherwise the SPINNY_CDP_ENDPOINT secret has most likely expired — remove it and this canary goes back to running from the runner's own address, which works on this site." - echo "### Canary did NOT test anything (exit 5, endpoint refused)" >> "$GITHUB_STEP_SUMMARY" - echo "The optional \`SPINNY_CDP_ENDPOINT\` secret is set and did not authenticate. This canary does not need it." >> "$GITHUB_STEP_SUMMARY" ;; + echo "::error::The content was never obtained (exit 5): a navigation timeout, a dead connection, or (only when the optional SPINNY_CDP_ENDPOINT secret is set) a refused Scraping Browser endpoint. Nothing was tested, so this run is failed. If the log says 'profile_locked', another run holds this pid: give CI its own. If the endpoint answered 401, the secret has expired: remove it, and this canary runs from the runner's own address, which works on this site." + echo "### Canary did NOT test anything (exit 5, content never obtained)" >> "$GITHUB_STEP_SUMMARY" + exit 1 ;; 3) echo "::error::Blocked before parsing (exit 3). On this site that is NEW: no Spinny refusal has ever been observed, from a home connection, an Amsterdam datacentre exit, a Chennai residential exit or the Scraping Browser API. Check the byte count in the debug dump — 0 bytes means nothing came back at all, and a page that does not reference assets.spinny.com is Chromium's own error page rather than an interstitial. Either the site has started scoring addresses, or the runner's network is broken." echo "### Blocked (exit 3) — new behaviour for this site" >> "$GITHUB_STEP_SUMMARY" @@ -135,231 +138,13 @@ jobs: if: steps.verdict.outputs.tested == 'true' # Exit 0 only means "some rows were written". It says nothing about # whether the page actually rendered: a snapshot taken too early - # still writes rows, just with fields empty. Every floor below is set - # from a measured figure and deliberately under it, so this fails on - # a collapse rather than on normal variation. - run: | - python3 - <<'EOF' - import json, sys - from pathlib import Path - - BATCHES_REQUESTED = 3 - # Measured 2026-09-11: --pages 3 returned 62 cars, three times, - # through three different engines. The floor is two batches' worth, - # so a slow round cannot trip it. - MIN_ROWS = 40 - # 705 of 705 cards across three captures carried a price, and - # 62/62 on every live run. Every card prints one. - MIN_PRICED_SHARE = 0.98 - - rows = json.loads(Path("canary_run.json").read_text()) - meta = json.loads(Path("canary_run.meta.json").read_text()) - fail = [] - - if len(rows) < MIN_ROWS: - fail.append(f"only {len(rows)} rows, expected at least {MIN_ROWS} " - f"from {BATCHES_REQUESTED} batches of 20") - - skus = [r.get("sku") for r in rows] - if len(set(skus)) != len(skus): - fail.append(f"{len(skus) - len(set(skus))} duplicate sku(s) — one " - f"scrolled page can re-render a card as it hydrates, " - f"and the writer is meant to drop the repeats") - - priced = sum(1 for r in rows if r.get("price") is not None) - if rows and priced / len(rows) < MIN_PRICED_SHARE: - fail.append(f"only {priced}/{len(rows)} rows have a price " - f"(floor {MIN_PRICED_SHARE:.0%}) — every card prints " - f"one, so this means the price node moved") - - # THERE IS NO STRUCTURED-PRICE CONFIRMATION SHARE TO CHECK, and that - # absence is measured rather than an omission: a Spinny LISTING page - # carries five JSON-LD blocks and not one of them names a car. So - # `price_source` is "dom+attr" on every listing row, and a - # confirmation floor copied from a sibling repo would fail every run. - sources = {r.get("price_source") for r in rows} - if sources - {"dom+attr", "dom"}: - fail.append(f"listing rows carry price_source {sorted(sources)} — " - f"only 'dom+attr' and 'dom' are possible on a listing " - f"page here") - - # THE CHECK THIS SITE ACTUALLY NEEDS. Two price BASES: the displayed - # figure excludes RC transfer and insurance, the site's own - # data-price attribute includes them, so the attribute was the - # HIGHER of the two on 482 of 482 measured cards and 62 of 62 on - # every live run. A row where it is lower means the two reads have - # been crossed — which computes a negative discount and still looks - # entirely plausible in the output. - pairs = [(r["price"], r["price_all_in"]) for r in rows - if r.get("price") is not None - and r.get("price_all_in") is not None] - crossed_basis = [p for p, a in pairs if a < p] - if pairs and len(crossed_basis) / len(pairs) > 0.02: - fail.append(f"{len(crossed_basis)}/{len(pairs)} rows have an " - f"all-in price BELOW the displayed one — the two " - f"price bases have been crossed") - if len(pairs) < 0.9 * len(rows): - fail.append(f"only {len(pairs)}/{len(rows)} rows carry both price " - f"bases — the data-price attribute is measured on " - f"705 of 705 cards, so this means it moved") - - # Fields that were fully populated on the measured runs. Any of them - # collapsing means an anchor moved. - for field, floor in (("title", 0.98), ("sku", 1.0), ("url", 1.0), - ("brand", 0.95), ("model", 0.95), - ("variant", 0.95), ("year", 0.95), - ("currency", 0.98), ("category", 1.0), - ("km_driven", 0.95), ("fuel_type", 0.95), - ("transmission", 0.95), ("rto", 0.95), - ("assurance", 0.95), ("emi_monthly", 0.90), - ("image_url", 0.98), - ("page", 1.0), ("position", 1.0)): - got = sum(1 for r in rows if r.get(field) not in (None, "")) - if rows and got / len(rows) < floor: - fail.append(f"{field} present on only {got}/{len(rows)} rows " - f"(floor {floor:.0%})") - - # `hub` IS EXPECTED TO BE SPARSE — 10 of 705 cards print none — so - # the floor is well under 100%. A floor near it would fail healthy - # runs; no floor at all would miss the column emptying. - hubs = sum(1 for r in rows if r.get("hub")) - if rows and hubs / len(rows) < 0.80: - fail.append(f"hub present on only {hubs}/{len(rows)} rows — " - f"about 2% of cards legitimately print none, so this " - f"low means the hub read has broken") - - # ...and so is `original_price`: 20 of 705 cards carry no strike, - # and those same 20 carry no discount badge. The two travel - # together, so a row with one and not the other is a parsing fault. - mismatched = [r.get("sku") for r in rows - if (r.get("original_price") is None) - != (r.get("discount_amount") is None)] - if len(mismatched) > 0.10 * max(len(rows), 1): - fail.append(f"{len(mismatched)} row(s) have a strike price " - f"without the site's own discount figure or the " - f"reverse: {mismatched[:3]}") - - # Null on every row of every run, measured across 705 cards and one - # detail page: Spinny publishes no car-level rating and no per-car - # review count. If either starts filling up, something is reading - # Spinny's OWN aggregate rating into a car's row. - for field in ("rating", "review_count"): - got = sum(1 for r in rows if r.get(field) is not None) - if got: - fail.append(f"{got}/{len(rows)} rows have `{field}` — the " - f"site publishes none per car, so this is " - f"probably the marketing block's rating of " - f"SPINNY being read into a car") - - # Detail-only columns must stay null on a listing run, or a listing - # row is claiming facts only a car's own page states. - for field in ("km_driven_exact", "owners", "registration_year", - "registration_month", "color", "seating_capacity", - "insurance_validity", "insurance_type", "in_stock"): - got = sum(1 for r in rows if r.get(field) is not None) - if got: - fail.append(f"{got}/{len(rows)} listing rows have `{field}`, " - f"which only --mode detail can know") - - # THE INVARIANT, one line, and it catches a whole class forever. An - # original_price at or below the price means the sale price and the - # was-price are being read out of the wrong nodes — which is what a - # second KIND of struck-through price would produce, the way an EU - # Omnibus 30-day low did in a sibling repo. - crossed = [r["sku"] for r in rows - if r.get("original_price") and r.get("price") - and r["original_price"] <= r["price"]] - if crossed: - fail.append(f"{len(crossed)} row(s) have an original_price at or " - f"below their price: {crossed[:3]} — the sale price " - f"and the strikethrough are being confused") - bad_discount = [r["sku"] for r in rows - if r.get("discount_pct") is not None - and not (0 < r["discount_pct"] < 100)] - if bad_discount: - fail.append(f"{len(bad_discount)} row(s) have a discount outside " - f"(0, 100): {bad_discount[:3]}") - - # `sku` is the car's numeric id and the last segment of its own URL. - # If that changes, every downstream join breaks silently. - bad_sku = [r.get("sku") for r in rows - if not (r.get("sku") or "").isdigit() - or (r.get("url") or "").rstrip("/").rsplit("/", 1)[-1] - != r.get("sku")] - if bad_sku: - fail.append(f"{len(bad_sku)} sku(s) are not the car's numeric id " - f"from its own URL: {bad_sku[:3]}") - - # `page` and `position` TOGETHER have to identify a row. This run - # fetches ONE URL and reports batches through `page`, so a failure - # to thread the batch number in would give every row page 1 and a - # position another row already has — which a sibling repo shipped. - keys = [(r.get("page"), r.get("position")) for r in rows] - if len(set(keys)) != len(keys): - fail.append(f"{len(keys) - len(set(keys))} row(s) share a " - f"(page, position) pair — the scroll's batch " - f"boundaries are not reaching the parser") - - # One currency per run, always INR. The site sells in one country — - # measured identical from two exit countries — so more than one - # currency here means something is inventing them. - currencies = {r.get("currency") for r in rows if r.get("currency")} - if currencies - {"INR"}: - fail.append(f"rows carry a currency other than INR " - f"({sorted(currencies)}) — this site quotes one") - - # The point of asking for more than one batch. `pages_completed` - # counts batches of 20 that the run HOLDS. - if meta.get("pages_completed", 0) < BATCHES_REQUESTED: - fail.append(f"the run holds {meta.get('pages_completed')} batches " - f"of the {BATCHES_REQUESTED} requested (stop_reason=" - f"{meta.get('stop_reason')!r}) — the lazy-load scroll " - f"is not reaching further into the grid, and this " - f"site publishes no pagination to fall back on") - batches_seen = {r.get("page") for r in rows} - if len(batches_seen) < 2: - fail.append(f"every row claims batch {sorted(batches_seen)} — the " - f"scroll's boundaries are not being recorded, so the " - f"`page` column means nothing") - if meta.get("status") != "complete": - fail.append(f"run status is {meta.get('status')!r}, not 'complete'") - if meta.get("mode") != "listing": - fail.append(f"run mode is {meta.get('mode')!r}, not 'listing'") - - # THE SCROLL IS THE ONLY REASON THERE ARE ROWS AT ALL on this site: - # the first response is a shell with zero cars in it. - scroll = meta.get("scroll") or {} - if not scroll: - fail.append("the sidecar records no scroll at all — the lazy-load " - "loop did not run, and on this site that means the " - "grid never hydrated") - elif not (scroll.get("settled") or scroll.get("reached_target")): - fail.append("the scroll neither settled nor reached its target, " - "so this run is a floor rather than the listing") - - # COMPLETENESS IS ARITHMETIC HERE because Spinny prints its own - # total. The number itself moves between runs (1546 and 1566 eleven - # minutes apart), so what is asserted is that it EXISTS and is - # sane — not its value. - comp = meta.get("completeness") or {} - advertised = comp.get("total_available") - if advertised is None: - fail.append("the listing's own advertised total was not read — " - "the heading node ([data-componentname=HeadingContent]) " - "has moved, and with it the completeness check") - elif advertised < len(rows): - fail.append(f"the site advertises {advertised} cars and this run " - f"holds {len(rows)} — the total is being read from " - f"the wrong node") - - for line in fail: - print(f"::error::{line}") - print(f"[canary] {len(rows)} rows, {priced} priced, " - f"{hubs} with a hub, advertised {advertised}, " - f"status {meta.get('status')!r}, " - f"stop_reason {meta.get('stop_reason')!r}") - sys.exit(1 if fail else 0) - EOF + # still writes rows, just with fields empty. The assertions live in + # .github/canary_check.py so they can be run against a downloaded + # artefact and by the offline suite; each one tests the parser or + # the run, never a share of cars the site chose to discount or to + # advertise with an EMI. That file says which floors were removed + # and why. + run: python3 .github/canary_check.py canary_run - name: Upload what the run produced # On EVERY run, not only on failure: a green canary's artefacts are diff --git a/.gitignore b/.gitignore index 9ced0f1..47edd70 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,23 @@ live_results.zip *_debug.html *_debug.png *.html.page* +# --dump-html NAME writes NAME itself on a one-page run and NAME.pageN on a +# longer one, whatever NAME's own suffix is — so a dump at the repo root +# (`canary_run.html`, `foo.html`) and a `dump.page2` both need covering. +/*.html +*.page[0-9]* + +# --- working directories, by KIND rather than by one name --------------- +# A raw capture reaches `git add -A` through whichever directory nobody +# listed. These are the names such directories are actually given. +live/ +out/ +output/ +runs/ +results/ +tmp/ +scratch/ +dumps/ # --- captures kept for parser work --------------------------------------- # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py); @@ -73,7 +90,7 @@ secrets* # recoverable only because that repo was private at the time. On 2026-09-17 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC # repository, one `git add -A` from publication. Craigslist was fixed then -# and the fix never reached anyone else, which is §16's rule unapplied: when -# you fix one, check the siblings. +# and the fix never reached anyone else: when you fix one, check the +# siblings. .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fc96d7..7cb677f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,9 +19,51 @@ with it, so nobody discovers it from a bill or from a diff. > exit 3 or 4 with a 39-byte "capture", in tens of milliseconds. If a > script of yours passes 0, pass **1**: the flag counts ATTEMPTS, so 1 means > "try once and do not retry". +> +> **Behaviour change:** `--pages`, `--concurrency` (both at least 1), +> `--proxy-block-retries`, `--delay` and `--retry-delay` (all at least 0) now +> reject values outside those ranges with exit 2 and the flag's name, in all +> three engines; `scraper_api_client.py` does the same for `--timeout` (at +> least 1), `--retries` and `--retry-delay` (at least 0). `--retry-delay -1` +> used to crash the first retry with a traceback. +> +> **Behaviour change (canary):** the canary now fails on exit 5 (the content +> was never obtained) instead of passing with a warning, and its EMI-share +> and strike/discount-pairing floors are gone (see Fixed). ### Fixed +- **The EMI figure was missed, and its line read as the hub, on the newer + card shape.** Spinny now also prints "EMI 15,916" (no "/m") beside a + "Save extra ₹19.5K on" promotion. The pattern required "/m", so the + canary runs of 2026-10-08, -10 and -15 read no EMI on any card, and + "EMI 15,916" (it has a comma) became the `hub`. Both wordings are read + now and the promotion is not taken for the `tag`; a fixture cut from the + 2026-10-15 canary run pins the values. +- **The canary was red on merchandising and green on nothing.** Its floors + on the share of rows with an EMI, and on "a strike price always comes + with the site's own discount figure" (on 2026-10-10 the site printed 20 + strike prices without one), failed complete, fully priced runs. They are + replaced by parser invariants: EMI-box text never lands in another + column, and where both figures are printed they agree. The assertions + moved to `.github/canary_check.py`, which can be run against a downloaded + artefact. Exit 5 used to end green with a warning, though a navigation + timeout from the runner exits 5 too; it now fails. +- **The credential scan could scan nothing and say ok.** Its excluded + directory names were compared against the absolute path, so a clone under + any directory called `venv` scanned 0 files. They are compared relative + to the repo root now, a scan of 0 files fails, and `.env*` backups, + `.json`, `.csv`, `.html` and `--dump-html` page dumps are read too. +- **Root-level HTML dumps, `NAME.pageN` dumps and `live/`-style working + directories were not gitignored.** +- **The HTTP status was dropped before classification.** Playwright and + pyppeteer now pass the navigation's status, so a 404 or a non-200 answer + is judged by what the server said. Selenium exposes no status and keeps + relying on the page's own 404 copy. +- Comments and docs no longer cite private notes by section number; the + rule is stated in place. The README names the 2Captcha Scraping Browser + API and describes exit 5 as "the content was never obtained". + - **`--retries 0` fetched nothing and called it a result.** The load loop is `range(1, retries + 1)`, so 0 never runs it and the engine judged `about:blank`. Three third-party audits of this family ran exactly that and diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2a22d0c..515da3b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,8 +109,9 @@ Then the rest of the presentation, in the order that matters: is scheduled daily as well. The optional `SPINNY_CDP_ENDPOINT` secret, when set, routes the same run through the Scraping Browser API; no step is gated on it, so the canary keeps working the day it expires. -2. The repo description, homepage and topics set (see the family notes on - what those should say). +2. The repo description, homepage and topics set: a description naming what the site + publishes and the 2Captcha Scraping Browser API, the 2captcha.com + homepage, and topics people actually search for. 3. Only then the row in the org profile README — and check it with an ANONYMOUS request rather than your own logged-in browser. A row pointing at a private repo is a 404 for every visitor, which costs more trust than diff --git a/README.md b/README.md index bb87b78..eb11087 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ [![runs without an account](https://img.shields.io/badge/runs%20without%20an%20account-yes-brightgreen)](#do-you-need-anything-paid) Scrapes **Spinny** used-car listings and car detail pages into JSON or CSV, -through Playwright, Selenium, pyppeteer, or a remote browser over CDP. +through Playwright, Selenium, pyppeteer, or the 2Captcha Scraping Browser API +over CDP. ``` python playwright_scraper.py \ @@ -276,7 +277,8 @@ the two on **482 of 482** measured cards, and every run reports that share. ### Exit codes `0` ok · `1` crash · `2` bad usage · `3` blocked · `4` zero cars · -`5` remote API error · `6` partial. +`5` the content was never obtained (a navigation timeout, a dead proxy, a +refused remote endpoint) · `6` partial. A run that finds nothing **writes nothing**, so last night's good output is never replaced by `[]`. Pass `--allow-empty` if an empty result is the answer @@ -458,7 +460,11 @@ identically to the untrimmed original before being committed. `canary.yml` runs one real listing daily and asserts the row floor, the price coverage, the two price bases, the uniqueness of `page`+`position`, the -site's own advertised total and the run status. Unlike every sibling repo's, +site's own advertised total and the run status (the assertions are in +`.github/canary_check.py`; they test the parser and never a share of cars the +site chose to discount or show an EMI for). It is red on anything that did +not test the site: a block (exit 3), zero rows (exit 4) or content never +obtained (exit 5). Unlike every sibling repo's, it needs **no credential** — dispatched for the first time on 2026-09-11 from a bare GitHub runner it returned **82 cars, 82 priced, 79 with a hub, advertised 1546, status `complete`**. diff --git a/captcha_solver.py b/captcha_solver.py index 23fba50..6e8061d 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -553,7 +553,7 @@ def _solve_with_2captcha_v2(api_key: str, challenge: CaptchaChallenge, # with the endpoint constant it used. Nothing called it: this module has no # CLI, no engine consulted it before spending, and adding a paid API call to # every run to find out whether a run might spend anything would cost more -# than it saved. A public helper with no consumer is the defect §17 names — +# than it saved. A public helper with no consumer is dead code that looks load-bearing — # it reads like a feature and is not one — so it is removed rather than left # to look load-bearing. If a balance check is ever wanted, it belongs behind # a flag someone typed. @@ -730,4 +730,4 @@ class CaptchaUnsolvable(RuntimeError): # what the address has been doing — a narrow list is how a challenge gets # reported as an empty page months later. A solver for a challenge this site # has never been observed to serve is dead code; a DETECTOR for one is cheap -# insurance. See the family note in CLAUDE.md. +# insurance. diff --git a/env_config.py b/env_config.py index f1c33ca..2ff8b02 100644 --- a/env_config.py +++ b/env_config.py @@ -66,8 +66,9 @@ # A literal-only check reported both of those as CONFIGURED, so `cp # .env.example .env` followed by a run connected to cb.2captcha.com with the # string `{login}-zone-…` as its username and got a 401 — a confusing failure -# a long way from its cause, which is precisely what §3's rule exists to -# prevent. Any `{…}` left in a value means the line was never filled in. +# a long way from its cause, which is precisely what keeping credentials in +# .env, with placeholders read as unset, exists to prevent. Any `{…}` +# left in a value means the line was never filled in. _BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}") _loaded_from = None diff --git a/fingerprint_client.py b/fingerprint_client.py index fb6d933..4c9d34d 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -250,7 +250,7 @@ def playwright_context_kwargs(fp: dict) -> dict: # `window.devicePixelRatio === 1`, so the identity contradicted itself # on an axis any fingerprinter reads for free -- and a contradiction is # exactly what this flag exists to avoid. Same shape as the three - # defects the family notes already list for this file: a key the API + # defects already found in this file: a key the API # returns that nothing applies. Found by comparing a live browser # against the fingerprint rather than by reading the code. scale = (fp.get("screen") or {}).get("deviceScaleFactor") @@ -331,7 +331,8 @@ def main() -> int: # loader. It used to read `os.environ` alone, which meant a key put in # `.env` exactly as the README and .env.example instruct worked for every # engine and failed HERE with "No API key" — a documented mechanism not - # applied on one path, which is the shape of half the defects §16 lists. + # applied on one path, which is the shape of many defects found in + # code copied between scrapers. # # `.env` has to be LOADED before it can be read: `env_value` looks at # os.environ, and `load_env` is what fills that from the file. Calling it diff --git a/output_writer.py b/output_writer.py index c0b1798..b6f9a71 100644 --- a/output_writer.py +++ b/output_writer.py @@ -542,10 +542,11 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, failed — nothing was gathered at all `mode` and `source` are recorded because `mode` is not implied by the - repo: the same output prefix can hold a listing run or a product run, - and those populate different columns — `sold` is a FLOOR on a listing - row and exact on a product row, so diffing one against the other would - report every row as changed. diff_runs.py refuses a pair whose modes or + repo: the same output prefix can hold a listing run or a detail run, + and those populate different columns — `km_driven` is the card's rounded + figure on a listing row while a detail row also carries `km_driven_exact`, + owners and registration, so diffing one against the other would report + every row as changed. diff_runs.py refuses a pair whose modes or sources differ. `source` is `spinny.com` on every row of every run here, since the site has one hostname and one currency; it is kept because consumers read these columns by name across the family. diff --git a/page_flow.py b/page_flow.py index 876d1c2..6ac3510 100644 --- a/page_flow.py +++ b/page_flow.py @@ -100,8 +100,8 @@ READY_SELECTOR_DETAIL = '[data-base-component="Pricing"]' # How many matches mean "the grid rendered". Must be > 1: waiting for a -# single match resolves on an unrelated link long before the grid paints -# (§5). Spinny's first batch is 22 cars and arrives whole, so 4 is reached +# single match resolves on an unrelated link long before the grid paints. +# Spinny's first batch is 22 cars and arrives whole, so 4 is reached # the moment it lands and is nowhere near a batch boundary. MIN_CARD_MATCHES = 4 MIN_CARD_MATCHES_DETAIL = 1 @@ -129,7 +129,9 @@ def content_timeout_ms(mode: str) -> int: # --------------------------------------------------------------------------- # The lazy-load scroll — the only way any car is ever seen # --------------------------------------------------------------------------- -# §8's rule, and every clause of it is confirmed by the measurement below. +# Missing content is one of three things (not painted yet, lazy-loaded, or +# a different page served to this session), and every clause of that is +# confirmed by the measurement below. # Ten scroll rounds on /used-cars-in-delhi-ncr/s/, 2026-09-11: # # cards 22 42 82 122 142 162 182 182 222 242 302 @@ -302,9 +304,10 @@ def classify(html: Optional[str], status: Optional[int] = None, """The page's state, in one place, for all three engines. `status` is OPTIONAL, and that default is load-bearing rather than tidy. - Playwright hands back a response object with a status on it; pyppeteer - and Selenium do not expose one at the point this is called, so they pass - only the markup. When `status` was required positionally in a sibling + Playwright and pyppeteer pass the status of the navigation's response; + Selenium's `driver.get()` returns nothing and its driver exposes no + status, so it passes only the markup and relies on the page's own 404 + copy, which is why that copy is checked as well as the status. When `status` was required positionally in a sibling repo, both of those engines crashed with `TypeError` on their FIRST fetch — and that was invisible to import, to `--help`, to `compileall`, to the AST undefined-name walk and to 426 green offline checks, because @@ -315,6 +318,11 @@ def classify(html: Optional[str], status: Optional[int] = None, """ if html is None: return "blocked" + # Only a real integer is a status; anything else a driver hands back + # (None from a same-document navigation, a stub in a test) means "not + # known", never "not 200". + if isinstance(status, bool) or not isinstance(status, int): + status = None return detect_page_state(html, status, url) @@ -388,7 +396,7 @@ def is_unpainted(state: str, html: Optional[str]) -> bool: navigation, filter rail and footer with a grid container and nothing in it. - §8 says missing content is one of three things — not painted, + Missing content is one of three things — not painted, lazy-loaded, or a different page served to this session. On Spinny every listing starts as the first and continues as the second, and neither wants a retry: a page the site plainly served, with no zero-count @@ -413,7 +421,7 @@ def recaptcha_note() -> str: gating the listing on a score — which it has not been observed doing from either exit tested. So `recaptcha` and `g-recaptcha` are deliberately NOT in `product_parser.BOT_CHALLENGE_MARKERS`: a marker that matches every - page of the site it is meant to guard is worse than no marker (§18). + page of the site it is meant to guard is worse than no marker. Returned as a string rather than logged here so an engine can print it once, where a user asking "why didn't it solve the captcha" will see it. @@ -433,7 +441,7 @@ def recaptcha_note() -> str: # far more likely to be a dead proxy exit than a scored address. Rotating is # exactly the right response to that. # -# CONSULTED BY THE ENGINES, not decorative — §17 found a sibling repo's +# CONSULTED BY THE ENGINES, not decorative — a sibling repo had an # identical constant with a paragraph of justification and no reader. The # offline suite asserts that every engine reads this one. RETRY_ON_BLOCKED = True @@ -513,18 +521,18 @@ def block_retry_budget(has_pool: bool, retries: int) -> int: # `link[rel=next]`, zero `page=` hrefs, no "Load more" and no numbered # anchors on a fully scrolled 15 MB capture. # -# This is §7's silent-single-page failure in its most dangerous form. A +# This is the silent-single-page failure in its most dangerous form. A # `page_url()` used unconditionally would fetch page one N times, find no new # id after the first, conclude the listing was exhausted, and report a -# COMPLETE run holding a twentieth of a Delhi catalogue. Three of §7's four -# layers pass it: the selector layer finds nothing (correct), the constructed +# COMPLETE run holding a twentieth of a Delhi catalogue. Three of the four +# pagination defences pass it: the selector layer finds nothing (correct), the constructed # -URL layer builds something that returns 200 (wrong), and only "this page # added no new sku" notices. # # So the engines scroll one long page, and `--concurrency` above 1 is # refused with that reason. NEXT_PAGE_SELECTOR: List[str] = [ - # Ordered most-durable first, per §5 — standards-based signals before + # Ordered most-durable first — standards-based signals before # build artefacts. NONE of these has ever matched on Spinny: the list is # here so that if the site ever grows real pagination markup the engines # pick it up, and every entry is honestly marked as unobserved. @@ -569,7 +577,7 @@ def pagination_is_addressable(page1_url: str, # when it agrees with the URL convention" pair, and on this site neither can # ever return anything: `page_url` yields None above page 1, and no capture # of Spinny has ever carried a next-link to filter. Kept, they would have -# been two policy functions with no consumer — the defect §17 names, where +# been two policy functions with no consumer — dead code, where # the prose reads like enforcement and nothing enforces it. # # What survives is the pair that DOES have a reader in all three engines: @@ -602,7 +610,7 @@ def completeness(rows_collected: int, html: Optional[str], scroll: Optional[dict] = None) -> dict: """Whether the run got everything the listing says it has. - §8's "where the site publishes totals, a gap is arithmetic" — and Spinny + Where the site publishes a total, a gap is arithmetic — and Spinny does publish one, beside its own heading: "1559 Used cars in Delhi NCR". So 482 rows against an advertised 1559 is PROOF the scroll stopped early, with no threshold and no guess involved. @@ -666,8 +674,8 @@ def listing_stop_reason(scroll: Optional[dict], completeness: Optional[dict], if advertised is not None and rows_collected < advertised: return "scroll_stalled" # The grid stopped growing and the site's own count agrees there was - # no more to get. That is the DATA-based terminating condition §7 - # asks for rather than a selector's absence. + # no more to get. That is a DATA-based terminating condition + # rather than a selector's absence. return "no_new_products" # The round budget ran out with the page still growing. Partial, and it # has to say so: the missing tail would otherwise read as delisted cars @@ -726,7 +734,7 @@ def comparable(url: str) -> str: # --------------------------------------------------------------------------- # --retries # --------------------------------------------------------------------------- -import argparse as _argparse # noqa: E402 (kept beside its only user) +import argparse as _argparse # noqa: E402 (kept beside its users) def attempts(value) -> int: @@ -766,3 +774,36 @@ def attempts(value) -> int: f"With 0 the load loop never runs, nothing is fetched, and the " f"engine reports the browser's blank page as a result.") return number + + +def numeric_type(flag, cast, minimum, unit): + """Build an argparse `type=` that rejects a value below `minimum`. + + The message names the flag, because argparse prefixes it with the + argument's destination only, and a reader should not have to work out + which of several numeric flags was wrong. + """ + def convert(value): + try: + number = cast(value) + except (TypeError, ValueError): + raise _argparse.ArgumentTypeError( + f"{flag} must be {unit}, not {value!r}") + if number != number or number < minimum: # NaN fails both + raise _argparse.ArgumentTypeError( + f"{flag} must be at least {minimum}, not {value}") + return number + convert.__name__ = flag.lstrip("-").replace("-", "_") + return convert + + +# Numeric flags every engine takes. Each one was accepted at any value before: +# `--retry-delay -1` crashed the first retry with "sleep length must be +# non-negative" (a traceback where a retry was due), and `--pages 0`, +# `--concurrency 0` or a negative block-retry count ran with a meaning nobody +# wrote down. Rejected up front, before any browser starts. +pages_arg = numeric_type("--pages", int, 1, "a whole number") +concurrency_arg = numeric_type("--concurrency", int, 1, "a whole number") +block_retries_arg = numeric_type("--proxy-block-retries", int, 0, "a whole number") +delay_arg = numeric_type("--delay", float, 0, "a number of seconds") +retry_delay_arg = numeric_type("--retry-delay", float, 0, "a number of seconds") diff --git a/playwright_scraper.py b/playwright_scraper.py index 7ae47c1..94d3df4 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -35,7 +35,7 @@ infinitely scrolling page that hydrates 20 cars at a time, so `--pages N` means N batches of 20 and the engine reaches them by scrolling. A planner that built `?page=N` would fetch page one N times, find no new id, and - report a COMPLETE run holding a twentieth of the catalogue — §7's + report a COMPLETE run holding a twentieth of the catalogue — the silent-single-page failure with the site answering 200 the whole way. * **`--concurrency` above 1 is refused, and this engine ships no worker pool.** Page 5 of an infinitely scrolling grid has no address to hand a @@ -66,7 +66,7 @@ invisible widget with a hidden badge. v3 scores rather than challenges, so there is nothing to solve and `--solve-captcha` has nothing to buy — and `recaptcha` is deliberately NOT a block marker, because a marker matching - every page is worse than no marker (§18). + every page is worse than no marker. * **No refusal was ever observed.** An Amsterdam datacentre address and a Chennai residential address both got the full grid with no key and no proxy. So exit 3 on this site is far more likely to be a dead proxy exit @@ -165,7 +165,7 @@ class PageOutcome: state: Optional[str] = None # What the listing itself says its catalogue size is, read off Spinny's # own heading ("1559 Used cars in Delhi NCR"). This site DOES publish - # one, which makes §8's completeness check arithmetic rather than a + # one, which makes the completeness check arithmetic rather than a # threshold: 482 rows against an advertised 1559 is proof the scroll # stopped early. Note the number in the disagrees (1546 against # 1559 on one capture) — the heading is the one the site's own API @@ -296,7 +296,7 @@ def _advertised_next_hrefs(page, page_num: int = 1) -> List[str]: nothing reads into a live check. If Spinny ever grows real pagination markup, this logs it loudly and names the module to revisit — which is better than a constant with a paragraph of justification and no reader - (§17 found exactly that in a sibling repo). + (a sibling repo had exactly that). """ selector = page_flow.next_page_selector(page_num) return [el.get_attribute("href") @@ -634,7 +634,7 @@ def handle_captcha_if_present(page, args, budget=None) -> bool: # loader rather than guessed from the widget. Spinny wires reCAPTCHA # Enterprise v3, and an enterprise v3 score task is priced and solved # differently from a v2 checkbox — sending v3 parameters for a v2 widget - # buys a token the site rejects (§8). Logged before the spend, so a + # buys a token the site rejects. Logged before the spend, so a # rejected token has an explanation next to it in the log. configured = recaptcha_config(html) if configured: @@ -720,8 +720,8 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Computed by page_flow, not here, so the three engines cannot disagree # about how many attempts a blocked page is worth — and so that # `RETRY_ON_BLOCKED` has a READER rather than a paragraph of - # justification nobody consults. §17 found exactly that constant, unread, - # in a sibling repo: setting it False changed nothing. Setting it False + # justification nobody consults. A sibling repo had exactly that + # constant, unread: setting it False changed nothing. Setting it False # here really does stop the retry loop. block_retries = page_flow.block_retry_budget( has_pool, args.proxy_block_retries if has_pool @@ -737,9 +737,16 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Retry a navigation timeout rather than ending the run on it. One # network flap on page 12 of 50 should not break the loop. load_failed, exit_failed = False, None + status = None for attempt in range(1, args.retries + 1): try: - session.page.goto(url, wait_until="domcontentloaded", timeout=60000) + response = session.page.goto(url, wait_until="domcontentloaded", + timeout=60000) + # The HTTP status reaches the classifier, so a 404 or a + # non-200 answer is judged by what the server said rather + # than only by the markup. None when the driver returns no + # response (a same-document navigation). + status = getattr(response, "status", None) load_failed = False break except (PWTimeout, PWError) as e: @@ -773,11 +780,13 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if handle_captcha_if_present(session.page, args, budget): # A solve navigated the page. Give the destination a moment - # before judging what came back. + # before judging what came back — and forget the status, which + # belonged to the page the solve replaced. session.page.wait_for_timeout(1000) + status = None html = _content_when_settled(session.page) or "" - state = _classify(session.page, html) + state = _classify(session.page, html, status) # "Not painted yet" is not a fault, and on this site it is not an # edge case either: **Spinny's first response is ALWAYS a shell.** @@ -811,7 +820,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome logger.info("The grid still had not painted after %.0fs " "(%d match(es)).", wait_timeout / 1000, found) html = _content_when_settled(session.page) or html - state = _classify(session.page, html) + state = _classify(session.page, html, status) # No interstitial-settling step here, and its absence is measured # rather than an omission: no refusal of any kind was ever observed @@ -1082,7 +1091,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.car_facts = car_metadata(html, session.page.url) # Spinny DOES publish a total, beside its own heading — "1559 Used cars - # in Delhi NCR" — which makes §8's completeness check arithmetic rather + # in Delhi NCR" — which makes the completeness check arithmetic rather # than a threshold. Read it here so the sidecar can carry the gap. # # Two traps, both measured: the number in the <title> disagrees (1546 @@ -1126,7 +1135,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # LISTING page carries five JSON-LD blocks and not one of them names # a car, so `price_source` is "dom+attr" on every listing row and a # confirmation threshold would describe nothing. Porting the sibling - # repo's overlay would be dead code that looks load-bearing (§4). + # repo's overlay would be dead code that looks load-bearing. # # What replaces it is the check this site actually needs. There are # two price BASES here — the displayed figure excludes RC transfer @@ -1195,7 +1204,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # means scrolling past the first 299 in one session. # # So the machinery is not here. Shipping it would be dead code that looks -# load-bearing, which §17 names as the same defect as an unenforced policy +# load-bearing, the same defect as an unenforced policy # constant — and a `--concurrency 8` that quietly did nothing useful would be # worse than one that says no. The flag is still accepted, because the # family's flag contract has it, and `page_flow.concurrency_refusal` explains @@ -1390,7 +1399,7 @@ def parse_args(): "\"luxury-cars\", \"volvo-cars\", " "\"cars-under-1-lakh-rs\" — so the column is rarely " "empty just because the flag was omitted.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.pages_arg, default=1, help="How many BATCHES of 20 cars to gather. Applies to " "--mode listing; ignored in --mode detail. This is a " "SCROLL budget, not a URL plan: a Spinny listing has " @@ -1398,11 +1407,11 @@ def parse_args(): "ignored and returns page 1 under HTTP 200 — so the " "engine reaches batch N by scrolling one long page. " "--pages 5 asks for 100 cars.") - p.add_argument("--delay", type=float, default=2.0, + p.add_argument("--delay", type=page_flow.delay_arg, default=2.0, help="Delay between pages, seconds. Nothing to pace on " "this site, where a run fetches one URL; kept because " "the family's flag contract has it.") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--concurrency", type=page_flow.concurrency_arg, default=1, metavar="N", help="Accepted and REFUSED on this site, with the reason. " "A worker cannot be handed \"page 5\" because page 5 " "has no address, so N workers would fetch page one N " @@ -1417,7 +1426,7 @@ def parse_args(): "page_flow.STATE_POLICY — because an SEO landing page " "or a cityless URL having no grid is a correct " "answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", type=page_flow.retry_delay_arg, default=2.0, help="Seconds before the first page-load retry, doubling " "thereafter (default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") @@ -1442,7 +1451,7 @@ def parse_args(): p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup, so concurrent runs do not " "all begin on the first exit in the file.") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", type=page_flow.block_retries_arg, default=2, help="When a page comes back unusable, retry it from this " "many OTHER exits before giving up (default 2). Needs " "a pool of more than one; ignored otherwise. No Spinny " @@ -1524,7 +1533,7 @@ def parse_args(): # and look like an empty listing. why = unsupported_reason(args.url) if why: - # Named WITH THE REASON (§5). "is not a Spinny site" would be + # Named WITH THE REASON. "is not a Spinny site" would be # false for api.spinny.com and sends the reader hunting for a # typo that is not there. p.error(f"{site_host(args.url)} {why}.") diff --git a/product_parser.py b/product_parser.py index 9e82b5d..f164a3e 100644 --- a/product_parser.py +++ b/product_parser.py @@ -16,7 +16,7 @@ the rendered DOM. So the JSON-LD-first design the rest of this family uses has nothing to be -first about on a listing, and §4's *fallback* — a URL pattern — is promoted +first about on a listing, and the usual *fallback* — a URL pattern — is promoted to primary because there is nothing above it. Every car in the grid is wrapped in an `<a href="/buy-used-cars/{city}/{make}/{model}/{trim}-{year}/{id}/">`, 705 of 705 across three captures, and that path is a contract with search @@ -64,8 +64,8 @@ **`data-label` is not a car id.** It is the shortlist button's label, and on CTA buttons inside the same grid it holds `loan-learn-more`, `Buy Back` and -`notify-btn`. Anchoring on it would let a junk element steal a real car's row -(§4). The id comes from the URL; `data-label` is only ever used as a +`notify-btn`. Anchoring on it would let a junk element steal a real car's row. +The id comes from the URL; `data-label` is only ever used as a cross-check, and it agreed on 705 of 705. **Four prices per car, and they disagree by up to ₹300,000.** See the note at @@ -110,7 +110,7 @@ # Hosts that look like they belong and do not. Refuse them WITH THE REASON: # "is not a Spinny site" would be false for a Spinny subdomain that simply is -# not the storefront, and sends the reader hunting for a typo (§5). +# not the storefront, and sends the reader hunting for a typo. UNSUPPORTED: Dict[str, str] = { "api.spinny.com": "is Spinny's JSON API, not a page — this scraper " "drives a browser and reads the rendered grid", @@ -132,7 +132,7 @@ # The cities Spinny sells in, taken from the site's OWN footer rather than # guessed — 33 `/used-cars-in-{city}/s/` links on a detail page captured -# 2026-09-11. §5 says to read this off the site and then CHECK it, and the +# 2026-09-11. Read off the site and then CHECKED, and the # check matters here: `delhi-ncr` is in the list and is a REGION, not a city. # A /used-cars-in-delhi-ncr/ listing returned cars whose own city segments # were delhi, gurgaon, ghaziabad, noida, faridabad, sonipat and karnal. @@ -231,15 +231,15 @@ def host_currency(url: str) -> str: "card_text": 'span[data-base-component="text"]', "card_heading": 'h3', # The listing's own advertised total: `<span>1559</span><h1>Used cars in - # Delhi NCR</h1>`. This is the completeness oracle (§8) — 482 rows + # Delhi NCR</h1>`. This is the completeness oracle — 482 rows # against an advertised 1559 is arithmetic proof the scroll stopped # early, needing no threshold. "heading_content": '[data-componentname="HeadingContent"]', } # How many car links mean "the grid rendered". Must be > 1: waiting for a -# single match resolves on an unrelated link long before the grid paints -# (§5). Spinny hydrates ~20 cars per batch and 22 were present in the DOM +# single match resolves on an unrelated link long before the grid paints. +# Spinny hydrates ~20 cars per batch and 22 were present in the DOM # the moment the first batch landed, so 4 is reached immediately on a real # listing and never on an empty one. MIN_CARD_MATCHES = 4 @@ -280,7 +280,7 @@ def host_currency(url: str) -> str: # A written amount: digits with optional grouping separators and an optional # decimal part. Space grouping is included with all four of the space # characters a rendered page uses, because a no-break variant is what keeps a -# number from wrapping and missing them parses "1 234" as 234 (§4). +# number from wrapping and missing them parses "1 234" as 234. _GROUP_SPACES = "    " _AMOUNT = r'\d[\d,.' + _GROUP_SPACES + r']*' @@ -311,12 +311,12 @@ def host_currency(url: str) -> str: # There was an `_RS_RE` here for "Rs. 1.65 Lakh" in a page title, and it is # gone: nothing read it, and `scaled_price` already returns 165000.0 for # that string because the SCALE WORD is what it anchors on and the "Rs." -# lead-in is not in the way. A second pattern with no consumer is the defect -# §17 names — prose that reads like enforcement while nothing enforces it. +# lead-in is not in the way. A second pattern with no consumer is dead +# code — prose that reads like enforcement while nothing enforces it. # Percentages come out of the text BEFORE prices are matched, never after. A # rejected match has still consumed the currency symbol, so filtering -# afterwards loses the real price too (§4). Spinny has not been observed +# afterwards loses the real price too. Spinny has not been observed # printing a percentage inside a card — its discount badge is in rupees — # but the guard costs one substitution and the sibling repo that skipped it # read a 25,999 TRY product as costing 10.34. @@ -324,10 +324,27 @@ def host_currency(url: str) -> str: # An EMI line lives INSIDE the card, right next to the price: "EMI6,675/m*". # On a sibling site exactly this shape made a price read return a monthly -# payment (§4). Here the price is read from its own labelled node rather than +# payment. Here the price is read from its own labelled node rather than # from the card's text, so the EMI cannot be mistaken for it — but the EMI is # also worth having, so it gets its own pattern instead of being stripped. -_EMI_RE = re.compile(r'EMI\s*(' + _AMOUNT + r')\s*/\s*m', re.I) +# +# The "/m" suffix is OPTIONAL. Measured on the canary's own captures: on +# 2026-10-14 the line read "EMI 18,833/m *" on 62 of 62 cards, while on +# 2026-10-08, -10 and -15 it read "EMI 15,916" (no "/m", no asterisk) on 82 +# to 102 cards a run, beside a new promotion in the same box (below). A +# pattern that required "/m" read 0 EMIs on those days, and the unmatched +# "EMI 15,916" — which has a comma in it — was then taken for the HUB. +# Anchored at the start so a free-text tag that merely mentions EMI is not +# read as one. +_EMI_RE = re.compile(r'^EMI\s*(' + _AMOUNT + r')(?:\s*/\s*m(?:onth)?)?\s*\*?$', + re.I) + +# The promotion printed beside the EMI in the newer card shape: "Save extra +# ₹19.5K on" (101 of 102 cards on 2026-10-15, "₹1.1L" on one). It is a +# marketing line about financing, not a fact about the car, so it is +# recognised and dropped rather than left over — left over, it became the +# row's editorial `tag`. +_EMI_PROMO_RE = re.compile(r'^Save\s+extra\b.*\bon$', re.I) # "63.5K km" / "117K km" / "980 km". The K suffix is the site's own rounding # and is why `km_driven` is documented as approximate. @@ -552,7 +569,7 @@ def _slug_display(slug: Optional[str]) -> Optional[str]: MAKE_DISPLAY). A model name's casing is genuinely inconsistent on the site itself — "Grand i10" in a headline, "grand-i10" in a URL, "Grand i10" in the API — so anything cleverer here would be inventing a - convention the site does not have. §13 calls inconsistent brand casing a + convention the site does not have. Inconsistent brand casing is a trap worth documenting rather than one worth fixing. """ if not slug: @@ -732,17 +749,17 @@ def unknown_city_warning(url: str) -> Optional[str]: # `link[rel=next]`, zero `page=` hrefs, no "Load more" and no numbered # anchors on a fully scrolled 15 MB capture. # -# This is the §7 failure in its most dangerous form. A `page_url()` that +# This is the silent-single-page failure in its most dangerous form. A `page_url()` that # built `?page=N` unconditionally would fetch page one N times, find no new # id after the first, conclude the listing was exhausted, and report a # **complete** run holding one twentieth of a Delhi catalogue. Every layer of -# §7's defence except the last would pass: the selector layer finds nothing +# the pagination defence except the last would pass: the selector layer finds nothing # (correct), the constructed-URL layer builds something that returns 200 # (wrong), and only the data layer — "this page added no new sku" — notices. # # So `paginates_by_url` is False for every URL on this site, `page_url` # returns None above page 1, and the engines scroll one long page instead. -# `--concurrency` above 1 is refused with that reason (§18): page 5 of an +# `--concurrency` above 1 is refused with that reason: page 5 of an # infinitely scrolling grid has no address to hand a worker. PAGE_PARAM = "page" PAGINATED_KINDS: Tuple[str, ...] = () @@ -823,7 +840,7 @@ def category_from_url(url: str) -> Optional[str]: # --------------------------------------------------------------------------- # Spinny gives a refused request no branded page to detect — the development # machine never met a refusal at all, from either exit tested — so detection -# here is INVERTED, per §8 and §18: look for a marker every REAL page carries +# here is INVERTED: look for a marker every REAL page carries # and treat its absence as the signal. A served page is assembled out of the # site's own assets; an interstitial, and Chromium's own network-error page, # are not. @@ -858,10 +875,10 @@ def category_from_url(url: str) -> Optional[str]: # So `recaptcha`, `g-recaptcha`, `grecaptcha-badge` and # `recaptcha/api.js`-style loader markers are facts about the site, not # markers of a block — and a marker that matches every page is worse than no -# marker (§18, proven twice on a sibling site in two consecutive releases). +# marker (proven twice on a sibling site in two consecutive releases). # The sibling repos' lists contain `g-recaptcha` and `recaptcha/api.js` and # were copied here; both were REMOVED after counting them on a page known -# good, which is the check §18 asks for. +# good, which is the check every new marker needs. # # What IS a signal is a challenge actually RENDERED, which v3-invisible never # is on a served page: the anchor and bframe iframes of a v2 or of an @@ -889,10 +906,10 @@ def category_from_url(url: str) -> Optional[str]: # Spinny wires reCAPTCHA Enterprise v3 with sitekey # `6Lc_rqYoAAAAAHwcTbMntlDkC52H6QAYgYE7eUKp`, loaded as # `enterprise.js?render=<sitekey>`. `render=<sitekey>` means v3; -# `render=explicit` would mean v2 (§8). It is present and invisible on every +# `render=explicit` would mean v2. It is present and invisible on every # page, and no challenge has ever been rendered into it. # -# So the machinery is configured everywhere and fires nowhere, which is §18's +# So the machinery is configured everywhere and fires nowhere, which is # "no challenge rendered is not no captcha configured" exactly. What this # function is for is telling the solver WHICH task type to buy if one ever # appears: an enterprise v3 score task, not a v2 checkbox, and the two are @@ -908,7 +925,7 @@ def recaptcha_config(html: str) -> Optional[Dict[str, str]]: Returns {"sitekey", "version", "enterprise"}. Present on every page Spinny serves; `version` is "v3" when the loader carries a sitekey in `render=` and "v2" when it carries `explicit`, which is the loader-wins - reconciliation §8 describes. + reconciliation: the loader wins over the wrapper's declaration. """ for m in _RECAPTCHA_LOADER_RE.finditer(html or ""): render = m.group("render") @@ -932,7 +949,7 @@ def _strip_extension_tags(html: str) -> str: own captcha hunters into every page it loads, and its markup mentions the vendors it hunts — including `cf-turnstile`, which is why a sibling repo's first live run reported exit 3 on a 1.8 MB page holding the full - catalogue (§8). + catalogue. """ return _EXTENSION_TAG_RE.sub(" ", html or "") @@ -946,7 +963,7 @@ def _strip_scripts(html: str) -> str: no_car_found_data: "Oops. No cars found for your search" So a text search for that sentence matches a page holding 482 cars — the - §18 trap again, arriving through a JS blob rather than a vendor script. + count-it-on-a-good-page trap again, arriving through a JS blob rather than a vendor script. Measured: "No cars found" appears once in the raw markup of every listing capture including the full Delhi one, and ZERO times in any of them once scripts are removed. @@ -960,7 +977,7 @@ def _strip_scripts(html: str) -> str: # Spinny's own not-found page. It answers HTTP 404, carries its own branding # and three asset references, and says so in plain English. Detected by its # OWN copy rather than by the asset threshold, because it IS a Spinny page — -# §17's ordering rule: a marker only the real page can carry outranks a +# the ordering rule: a marker only the real page can carry outranks a # threshold, whichever is cheaper to check. _NOT_FOUND_MARKERS = ( "Spinny | Not Found Page", @@ -974,7 +991,8 @@ def _strip_scripts(html: str) -> str: # alert-signup card where the grid would be. # # **There is exactly one signal for it, and it is arithmetic: the heading's -# own count reading 0.** No text marker survives §18's test. +# own count reading 0.** No text marker survives being counted on a +# page known to be good. # # The obvious candidates were tried and all three failed it. Spinny's # no-results sentence ("Oops. No cars found for your search") ships inside @@ -985,7 +1003,7 @@ def _strip_scripts(html: str) -> str: # of a FILTERED listing rather than an empty one: it appears once on the # zero-result Volvo page and once on the luxury listing that rendered 41 # cars, and not at all on the two unfiltered city listings. Counted on a page -# known good, exactly as §18 requires, and discarded on the strength of that +# known good, as every candidate marker must be, and discarded on the strength of that # count. # # So the count it is. It is the site's own arithmetic, it cannot appear on a @@ -1001,7 +1019,7 @@ def total_available(html: str, shown: int = 0) -> Optional[int]: """The total Spinny itself advertises for this listing, or None. Read from the site's own heading — `<span>1559</span><h1>Used cars in - Delhi NCR</h1>` — and this is the completeness oracle §8 asks for. A run + Delhi NCR</h1>` — and this is the completeness oracle. A run that gathered 482 rows against an advertised 1559 is PARTIAL by arithmetic, with no threshold and no guess involved. @@ -1061,7 +1079,7 @@ def total_pages(html: str) -> Optional[int]: def detect_bot_challenge(html: str, url: str = "") -> Optional[str]: """The name of a challenge vendor found in the markup, or None. - Broad on purpose (§8) — different geos and scenarios surface different + Broad on purpose — different geos and scenarios surface different challenges, and a detection is only a warning unless the page also has no cars on it. But NOT so broad that it matches the site's own invisible reCAPTCHA, which is on every page: see BOT_CHALLENGE_MARKERS. @@ -1078,7 +1096,7 @@ def detect_bot_challenge(html: str, url: str = "") -> Optional[str]: def served_by_spinny(html: str) -> bool: """Whether this markup is a page Spinny itself built. - Inverted detection (§8, §18). Worth having even where no block page has + Inverted detection. Worth having even where no block page has been observed, because it is the only thing that classifies **Chromium's own network-error page** correctly — on a sibling site that page carried `<title>www.tokopedia.com`, so a title check called it real, and @@ -1094,8 +1112,8 @@ def is_not_found(html: str, status: Optional[int] = None) -> bool: """Whether this is Spinny's own 404 page. The status is the primary signal where a driver exposes one — Playwright - does, pyppeteer and Selenium do not at the point this is called — and the - page's own copy is the structural fallback, so all three engines agree. + and pyppeteer do, Selenium does not — and the page's own copy is the + structural fallback, so all three engines agree. """ if status == 404: return True @@ -1111,7 +1129,7 @@ def is_no_results(html: str) -> bool: The site's own heading count reading exactly 0 — see the note above `_NO_RESULTS_MARKERS` for why nothing textual is used here. The empty marker tuple is still consulted so that a future marker which DOES pass - §18's count-it-on-a-good-page test can be added in one place. + the count-it-on-a-good-page test can be added in one place. """ if not html: return False @@ -1134,7 +1152,7 @@ def detect_page_state(html: Optional[str], status: Optional[int] = None, retry/solve/blocked decision that follows lives in `page_flow.STATE_POLICY` as data. - **The order of these checks is the whole design**, per §17: signals are + **The order of these checks is the whole design**: signals are ordered by how much they PROVE, not by how cheap they are. Spinny's own 404 copy and its own zero-count heading are unambiguous positives that only a real Spinny page can carry, so they outrank the asset-reference @@ -1193,7 +1211,7 @@ def detect_page_state(html: Optional[str], status: Optional[int] = None, # If not, Spinny did not build it — an interstitial, or the browser's # own error page. # - # It sits BELOW the three checks above because of §17's ordering rule: + # It sits BELOW the three checks above because of the ordering rule: # signals go in order of how much they PROVE, and a threshold is # always weaker than a marker only the real page can carry. Cars in # the site's own grid container, the site's own car JSON-LD and the @@ -1271,7 +1289,7 @@ def _cards(soup, url: str = "") -> List: # A card that CONTAINS another card would double-count its child. # Not observed on any capture — Spinny's cards are siblings — but one # nested card would otherwise add a row holding its parent's data, - # which is §4's tile-scoping failure from the other direction. + # which is the tile-scoping failure from the other direction. if card.find(attrs={"data-base-component": "card"}) is not None: continue out.append(card) @@ -1353,6 +1371,8 @@ def _card_fields(card) -> Dict[str, object]: if val is not None: found["emi_monthly"] = int(round(val)) continue + if _EMI_PROMO_RE.match(text): + continue if text.lstrip().startswith(("₹", "₹")) and found["discount_amount"] is None: found["discount_amount"] = rupee_amount(text) continue @@ -1385,7 +1405,7 @@ def _card_price(card) -> Tuple[Optional[float], Optional[float]]: Both come from LABELLED nodes rather than from the card's text, which is what keeps the EMI line — "EMI6,675/m", inside the same card — from ever being read as the price. A sibling site's price node contained its own - instalment line and that is exactly the bug it caused (§4). + instalment line and that is exactly the bug it caused. The strike is taken from inside the card only. A detail page's "Similar cars" strip carries strikes belonging to other cars, and scoping is the @@ -1422,7 +1442,7 @@ def _card_image(card) -> Optional[str]: against: Spinny renders a real `assets.spinny.com` URL into `src` on every card in the grid, 705 of 705, painted or not. The host is still checked positively — a populated column of icon URLs is the failure mode - §10 warns about — and a protocol-relative `//assets.spinny.com/...` is + a coverage check cannot see — and a protocol-relative `//assets.spinny.com/...` is made absolute, because that is how the site writes it. """ img = card.select_one("img[src]") @@ -1657,7 +1677,7 @@ def _report_coverage(rows: Sequence[Product], url: str) -> None: # 2. `@type` is a LIST — `["Product", "Car"]` — so a reader comparing # `@type == "Product"` finds nothing. # -# And the §4 shapes that are legal schema.org and crash a naive parser apply +# And the JSON-LD shapes that are legal schema.org and crash a naive parser apply # here too: `offers` can legally be null or a list, `image` can be an # ImageObject, and products can sit under `@graph`. All are handled. _JSONLD_RE = re.compile( @@ -1721,7 +1741,7 @@ def _offer_of(node: dict) -> Optional[dict]: """The car's Offer, whatever legal shape it takes. `"offers": null` is legal schema.org and an explicit null is NOT covered - by a `.get()` default — that only applies to a MISSING key (§4). A list + by a `.get()` default — that only applies to a MISSING key. A list of offers, possibly containing non-dicts, is legal too. """ offers = node.get("offers") @@ -1751,7 +1771,7 @@ def _offer_price(offer: Optional[dict]) -> Optional[float]: def _jsonld_image(node: dict) -> Optional[str]: - """The car's image, in any of the four legal shapes (§4). + """The car's image, in any of the four legal shapes. Normalised to https, because the JSON-LD writes `http://assets.spinny.com/…` where a listing card writes `//assets.spinny.com/…`. Left as two @@ -1983,7 +2003,7 @@ def parse_detail_page(html: str, url: str, brand=brand or make_display(make_slug), price=price, # From the site's own `priceCurrency` when it states one — a FACT at - # the top of §4's trustworthiness order, never overwritten from the + # the top of the currency trustworthiness order, never overwritten from the # DOM. Falls back to INR only when a price was found and the page # stated no currency. currency=currency or (CURRENCY if price is not None else None), diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index b9614a1..06c85f6 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -580,10 +580,13 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome for block_attempt in range(block_retries + 1): logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) load_failed = False + status = None for attempt in range(1, args.retries + 1): try: - bridge.run(page.goto(url, {"waitUntil": "domcontentloaded", - "timeout": 60000})) + response = bridge.run(page.goto( + url, {"waitUntil": "domcontentloaded", "timeout": 60000})) + # Passed to the classifier, as the Playwright engine does. + status = getattr(response, "status", None) load_failed = False break except Exception as e: # noqa: BLE001 — pyppeteer raises many types @@ -621,9 +624,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if handle_captcha_if_present(session, args, budget): time.sleep(1) + status = None # it belonged to the page the solve replaced html = _content(session) or "" - state = page_flow.classify(html, url=page.url) + state = page_flow.classify(html, status, url=page.url) # "Not painted yet" is not a fault, and on this site it is the NORMAL # first response: domcontentloaded at 1.1 s with zero cars, the first @@ -648,7 +652,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome logger.info("The grid still had not painted after %.0fs " "(%d match(es)).", wait_timeout / 1000, found) html = _content(session) or html - state = page_flow.classify(html, url=page.url) + state = page_flow.classify(html, status, url=page.url) # No interstitial-settling step, and its absence is measured rather # than an omission: no refusal of any kind was ever observed on this @@ -1089,14 +1093,14 @@ def parse_args(): "insurance details — the columns a listing card " "cannot carry. No --pages in detail mode.") p.add_argument("--category", default=None, help="Label to tag output rows with.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.pages_arg, default=1, help="How many BATCHES of 20 cars to gather. A SCROLL " "budget, not a URL plan: a Spinny listing has no " "per-page addresses — ?page=N returns page 1 under " "HTTP 200 — so batch N is reached by scrolling one " "long page.") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--delay", type=page_flow.delay_arg, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.concurrency_arg, default=1, metavar="N", help="Accepted and REFUSED with the reason, in every " "engine: page 5 of an infinitely scrolling grid has " "no address to hand a worker. Split the work across " @@ -1106,7 +1110,7 @@ def parse_args(): "A page that comes back EMPTY is not retried: an SEO " "landing page or a cityless URL having no grid is a " "correct answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", type=page_flow.retry_delay_arg, default=2.0, help="Seconds before the first retry, doubling thereafter") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="spinny_products", help="Output file prefix") @@ -1119,7 +1123,7 @@ def parse_args(): "Wins over --proxy.") p.add_argument("--proxy-rotate", choices=list(ROTATE_MODES), default="per-run") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2) + p.add_argument("--proxy-block-retries", type=page_flow.block_retries_arg, default=2) p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 rows were found.") diff --git a/scraper_api_client.py b/scraper_api_client.py index 0b0f292..b55563c 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -47,7 +47,8 @@ honest. And a browserless fetch is genuinely the right tool on other sites in this family, where the grid is server-rendered — this client is family core, and family core that only exists in four repos and runs in three is how the -untested-inheritance defects in §16 happen. +untested-inheritance defects happen: code copied between repos and +never run in this one. If you are reading this because your run came back with zero rows: use `playwright_scraper.py`, which scrolls. @@ -83,6 +84,7 @@ detect_page_state) from output_writer import save import env_config +import page_flow logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("scraper_api_client") @@ -347,7 +349,7 @@ def parse_args(): p.add_argument("--category", default=None, help="Label to tag output rows with. Defaults to the category segment of the URL, so the column is never empty just because the flag was omitted.") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="spinny_products_scraperapi", help="Output file prefix") - p.add_argument("--timeout", type=int, default=60, + p.add_argument("--timeout", type=page_flow.numeric_type("--timeout", int, 1, "a whole number"), default=60, help=f"API-side task timeout in seconds (1-{MAX_API_TIMEOUT}, default 60)") p.add_argument("--cdp-url", default=None, help="Route the fetch through an existing browser session over CDP " @@ -364,11 +366,11 @@ def parse_args(): p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 products were parsed. Off by " "default so a failed fetch can't overwrite a good result.") - p.add_argument("--retries", type=int, default=1, + p.add_argument("--retries", type=page_flow.numeric_type("--retries", int, 0, "a whole number"), default=1, help="Extra attempts if a bot-challenge page comes back. One retry is " "usually worth it. Each attempt is a separate billable task, so " "this defaults to 1.") - p.add_argument("--retry-delay", type=int, default=10, + p.add_argument("--retry-delay", type=page_flow.numeric_type("--retry-delay", int, 0, "a whole number"), default=10, help="Seconds between retries (default 10)") p.add_argument("--dump-html", default=None, help="Also write the raw returned HTML to this path (always, even on success)") diff --git a/selenium_scraper.py b/selenium_scraper.py index f45b1c6..3ae2cd9 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -1050,14 +1050,14 @@ def parse_args(): "insurance details — the columns a listing card " "cannot carry. No --pages in detail mode.") p.add_argument("--category", default=None, help="Label to tag output rows with.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.pages_arg, default=1, help="How many BATCHES of 20 cars to gather. A SCROLL " "budget, not a URL plan: a Spinny listing has no " "per-page addresses — ?page=N returns page 1 under " "HTTP 200 — so batch N is reached by scrolling one " "long page.") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--delay", type=page_flow.delay_arg, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.concurrency_arg, default=1, metavar="N", help="Accepted and REFUSED with the reason, in every " "engine: page 5 of an infinitely scrolling grid has " "no address to hand a worker. Split the work across " @@ -1067,7 +1067,7 @@ def parse_args(): "A page that comes back EMPTY is not retried: an SEO " "landing page or a cityless URL having no grid is a " "correct answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, + p.add_argument("--retry-delay", type=page_flow.retry_delay_arg, default=2.0, help="Seconds before the first retry, doubling thereafter") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="spinny_products", help="Output file prefix") @@ -1081,7 +1081,7 @@ def parse_args(): "Wins over --proxy.") p.add_argument("--proxy-rotate", choices=list(ROTATE_MODES), default="per-run") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2) + p.add_argument("--proxy-block-retries", type=page_flow.block_retries_arg, default=2) p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 rows were found.") diff --git a/smoke_test.py b/smoke_test.py index c147eeb..00a703b 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -32,7 +32,8 @@ `data-*` attribute, every class and every text node the parser reads is byte-for-byte what Spinny served. -Six cards, each here because it pins a specific behaviour, named above it. +Seven cards, each here because it pins a specific behaviour, named above it +(six from 2026-09-11, one from the canary run of 2026-10-15). One detail page, reduced from 2.1 MB to its own ["Product","Car"] JSON-LD, its price node, its overview table and its inspection-report line. @@ -291,6 +292,20 @@ def heading(count, what="Used cars in Delhi NCR"): DETAIL_PAGE = ( '3.88 Lakh
Make Year
Apr 2019
Registration Year
Aug 2019
Fuel Type
petrol (BSIV)
Km driven
63K km
Transmission
manual (regular)
No. of Owner
1st Owner
Insurance Validity
Jul 2027
Insurance Type
Third Party
RTO
HR26
Car Location
Sector-27, Faridabad
1573 parts evaluated by 5 automotive experts
' ) +# 33112227 — the NEWER card shape, cut from the canary's own run of +# 2026-10-15 (the same shape served on 2026-10-08 and -10), trimmed the same +# way as the cards above and verified to parse identically to the untrimmed +# card. Two differences from the 2026-09-11 captures, both in the EMI box: +# the line reads "EMI 15,916" with no "/m" and no asterisk, and a financing +# promotion, "Save extra ₹19.5K on", sits beside it. The parser required +# "/m" and so read no EMI on 0 of 82 to 102 cards those days, and the +# unmatched "EMI 15,916" — it has a comma — was taken for the HUB. +CARD_SLAVIA_EMI_PROMO_URL = 'https://www.spinny.com/used-cars-in-delhi-ncr/s/' +CARD_SLAVIA_EMI_PROMO = ( + '
Used 2022 Skoda Slavia Style 1.0L TSI AT Petrol Automatic Image

2022 Skoda Slavia

9.34 Lakh9.48 Lakh
Style 1.0L TSI AT
Save extra ₹19.5K on
EMI 15,916
39.5K kmPetrolAutomaticHR98
Sohna Road, Gurgaon
₹14,000
' +) + + def _one(card_html, url): """Parse a single card fixture as the grid the site would serve it in.""" rows = parse_products(grid(card_html), url) @@ -304,6 +319,7 @@ def _one(card_html, url): ("CARD_EON", CARD_EON, CARD_EON_URL), ("CARD_E_CLASS", CARD_E_CLASS, CARD_E_CLASS_URL), ("CARD_RANGE_ROVER", CARD_RANGE_ROVER, CARD_RANGE_ROVER_URL), + ("CARD_SLAVIA_EMI_PROMO", CARD_SLAVIA_EMI_PROMO, CARD_SLAVIA_EMI_PROMO_URL), ] @@ -380,7 +396,7 @@ def test_price_parsing(): # Listing rows — pinned VALUES, on real cards # --------------------------------------------------------------------------- def test_listing_values(): - group("listing rows: the exact values six real cards must produce") + group("listing rows: the exact values seven real cards must produce") ok = True r = _one(CARD_GRAND_I10, CARD_GRAND_I10_URL) @@ -478,7 +494,26 @@ def test_listing_values(): r.category == "luxury-cars") ok &= check("Range Rover: luxury tier", r.assurance == "luxury") - # INVARIANTS ACROSS ALL SIX, checked as a set rather than one at a time. + # THE NEWER EMI BOX: "EMI 15,916" without "/m", beside a promotion. + r = _one(CARD_SLAVIA_EMI_PROMO, CARD_SLAVIA_EMI_PROMO_URL) + ok &= check("Slavia: EMI read without the '/m' suffix", + r.emi_monthly == 15916) + ok &= check("Slavia: the hub is the place, not the EMI line", + r.hub == "Sohna Road, Gurgaon") + ok &= check("Slavia: the financing promotion is not the tag", + r.tag is None) + ok &= check("Slavia: variant", r.variant == "Style 1.0L TSI AT") + ok &= check("Slavia: price and strike", + (r.price, r.original_price) == (934000.0, 948000.0)) + ok &= check("Slavia: the site's own discount figure", + r.discount_amount == 14000.0) + ok &= check("Slavia: odometer, fuel, transmission, RTO", + (r.km_driven, r.fuel_type, r.transmission, r.rto) + == (39500, "Petrol", "Automatic", "HR98")) + ok &= check("an EMI pattern is anchored: a tag mentioning EMI is not one", + product_parser._EMI_RE.search("No-cost EMI 6,000 offer") is None) + + # INVARIANTS ACROSS ALL CARDS, checked as a set rather than one at a time. rows = [_one(h, u) for _, h, u in ALL_CARDS] ok &= check("every card yields exactly one row", all(rows)) ok &= check("every row has a price", all(r.price for r in rows)) @@ -659,7 +694,7 @@ def test_urls(): # Refused WITH THE REASON. "is not a Spinny site" would be FALSE for # api.spinny.com and would send the reader hunting for a typo that is not - # there (§5). + # there. why = unsupported_reason("https://api.spinny.com/v3/api/listing/v3/") ok &= check("api.spinny.com is refused with its own reason", why is not None and "API" in why) @@ -962,7 +997,7 @@ def test_recaptcha_is_not_a_block_marker(): group("the site's own reCAPTCHA is a fact about the site, not a marker") ok = True - # §18, and this is the check that rule asks for: COUNT A MARKER ON A PAGE + # The rule for markers, and this is the check it asks for: COUNT A MARKER ON A PAGE # YOU KNOW IS GOOD before adding it. Spinny loads reCAPTCHA Enterprise v3 # on every page it serves — 31 occurrences of "recaptcha" on a capture # that had just delivered 482 cars — so `recaptcha` and `g-recaptcha` as @@ -982,7 +1017,7 @@ def test_recaptcha_is_not_a_block_marker(): ok &= check("...and no vendor marker fires on it", detect_bot_challenge(good, LISTING_URL) is None) - # "No challenge rendered" is not "no captcha configured" (§18). What IS + # "No challenge rendered" is not "no captcha configured". What IS # read off the page is which task type a solve would have to buy — an # enterprise v3 score task, not a v2 checkbox, and the two are priced # and solved differently. @@ -994,7 +1029,7 @@ def test_recaptcha_is_not_a_block_marker(): cfg and cfg["sitekey"].startswith("6Lc_rqYo")) # render=explicit means v2, and the LOADER wins over anything the # wrapper declares: v3 parameters bought for a v2 widget produce a token - # the site rejects (§8). + # the site rejects. v2 = recaptcha_config('') ok &= check("render=explicit reads as v2", v2 and v2["version"] == "v2") @@ -1274,7 +1309,7 @@ def test_output_contract(): "seating_capacity", "insurance_validity", "insurance_type"]) ok &= check("both modes map to a row class", ROW_CLASS_BY_MODE == {"listing": Product, "detail": Product}) - # A column that is null on every row of every run should not exist (§9). + # A column that is null on every row of every run should not exist. # These eight are null on every LISTING row and populated in --mode # detail, which is a different thing and is why they are kept — pinned # so that removing them needs a measurement rather than a hunch. @@ -1561,7 +1596,7 @@ def default_exception_handler(self, context): def test_canary_separates_access_from_defect(): - group("the canary fails on defects and only WARNS on access conditions") + group("the canary is green only when it tested the site") ok = True wf_path = os.path.join(REPO_ROOT, ".github", "workflows", "canary.yml") wf = open(wf_path, encoding="utf-8").read() @@ -1593,10 +1628,10 @@ def test_canary_separates_access_from_defect(): # new information and is failed rather than excused. # # 0 got in and parsed -> pass, and the assertions then run - # 5 endpoint refused -> ACCESS, and only possible when the - # OPTIONAL Scraping Browser secret is set. `profile_locked` and an - # expired credential both say nothing about the site, and this - # canary does not need the secret at all. + # 5 content never obtained -> FAIL. It used to be a warning on the + # theory that only the optional Scraping Browser secret could cause + # it, but a navigation timeout from the runner's own address exits 5 + # too, and a canary that went green on that had tested nothing. # 3 blocked before parsing -> DEFECT, or news. See above. # 6 partial -> DEFECT: the target is 60 cars and the # measured run passes it in two scroll rounds, so a partial means @@ -1605,7 +1640,7 @@ def test_canary_separates_access_from_defect(): # 2 bad arguments -> DEFECT (in the workflow itself). # 4 served a page, ZERO rows -> DEFECT, and precisely the regression # this canary exists to catch: the card anchor moved. - expected = {0: "pass", 5: "warn", + expected = {0: "pass", 5: "fail", 1: "fail", 2: "fail", 3: "fail", 4: "fail", 6: "fail", 99: "fail"} for code, want in sorted(expected.items()): @@ -1637,8 +1672,56 @@ def test_canary_separates_access_from_defect(): got = failed and errored ok &= check("exit %-2d is treated as %s" % (code, want), got) - ok &= check("the reason access is not a defect is written down", - "ACCESS CONDITIONS ARE NOT DEFECTS" in wf) + ok &= check("the rule that only a tested run is green is written down", + "ONLY A RUN THAT TESTED THE SITE ENDS GREEN" in wf) + + # THE DATA ASSERTIONS ARE A FILE, RUN BY THE WORKFLOW, AND THEY TEST THE + # PARSER. Two floors on merchandising (an EMI share and "a strike price + # always comes with the site's discount figure") turned the badge red on + # complete, fully priced runs; neither may come back. + cc_path = os.path.join(REPO_ROOT, ".github", "canary_check.py") + ok &= check("the canary runs .github/canary_check.py", + "python3 .github/canary_check.py canary_run" in wf + and os.path.exists(cc_path)) + if os.path.exists(cc_path): + import importlib.util as _ilu + spec = _ilu.spec_from_file_location("_canary_check", cc_path) + cc = _ilu.module_from_spec(spec) + spec.loader.exec_module(cc) + floors = dict(cc.CORE_FIELDS) + ok &= check("no floor on the EMI share, a merchandising choice", + "emi_monthly" not in floors) + ok &= check("no floor on strike/discount shares", + "original_price" not in floors + and "discount_amount" not in floors) + # Run it: a healthy run of the newer card shape passes, and the same + # rows with the EMI line read as the hub (what the parser did on + # 2026-10-08/-10/-15) fail on the named invariant. + base = _one(CARD_SLAVIA_EMI_PROMO, CARD_SLAVIA_EMI_PROMO_URL) + rows = [] + for i in range(60): + r = dataclasses_asdict(base) + r.update(sku=str(33112227 + i), page=1 + i // 20, + position=1 + i % 20, + url=r["url"].rsplit("/", 2)[0] + "/%d/" % (33112227 + i)) + rows.append(r) + meta = {"pages_completed": 3, "status": "complete", "mode": "listing", + "scroll": {"reached_target": True}, + "completeness": {"total_available": 1782}} + fail, _ = cc.check(rows, meta) + ok &= check("canary_check passes a healthy run of the newer cards", + fail == []) + if fail: + print(" " + "; ".join(fail)[:400]) + leaked = [dict(r, hub="EMI 15,916", emi_monthly=None) for r in rows] + fail, _ = cc.check(leaked, meta) + ok &= check("...and fails when the EMI line lands in the hub", + any("EMI-box text" in f for f in fail)) + no_badge = [dict(r, discount_amount=None, emi_monthly=None) + for r in rows] + fail, _ = cc.check(no_badge, meta) + ok &= check("...and does NOT fail when the site prints no discount " + "badge and no EMI", fail == []) # THIS CANARY IS SCHEDULED, and that too differs from its siblings for a # measured reason: it needs no credential, so there is none to expire. @@ -1892,10 +1975,10 @@ def test_fingerprint_client_reads_env(): # THE DEFECT THIS PINS, found the first time --fingerprint was run live # here and confirmed present in five sibling repos: `--key` defaulted to # `os.environ.get("TWOCAPTCHA_KEY")` alone. So a key put in `.env` — - # which is exactly what §3, the README and .env.example instruct — worked + # which is exactly what the README and .env.example instruct — worked # for every engine and failed HERE with "No API key". A documented - # mechanism not applied on one path, which is the shape of half the - # defects §16 lists. + # mechanism not applied on one path, which is the shape of many + # defects found in code copied between scrapers. src = inspect.getsource(fpc.main) ok &= check("it loads .env itself, rather than hoping an engine did", "env_config.load_env()" in src) @@ -1959,7 +2042,7 @@ def test_fingerprint_application(): # possible, but it is not what this fingerprint describes, and a locale # that contradicts the rest of the identity is the mismatch again. This # was one of the six defects a sibling repo inherited from copied core - # and never ran (§16). + # and never ran. ok &= check("the locale is the fingerprint's own, not en-", kw.get("locale") == "en-IN") ok &= check("the timezone is carried, so the browser cannot contradict it", @@ -2612,7 +2695,7 @@ def test_captcha(): # # `recaptcha` / `g-recaptcha` — inherited from the sibling repos and # REMOVED here. Spinny loads reCAPTCHA Enterprise v3 on every page it - # serves, so as markers they would make every good page blocked. §18's + # serves, so as markers they would make every good page blocked. The # rule, and the check that rule asks for: count it on a page you know # is good before adding it. # @@ -2698,7 +2781,7 @@ def test_env_config(): # CONFIGURED, so `cp .env.example .env` and a run connected to # cb.2captcha.com with the string `{login}-zone-…` as its username and # got a 401 — a confusing failure a long way from its cause, which is - # what §3's rule exists to prevent. + # what reading placeholders as unset exists to prevent. for raw in ('ws://{login}-zone-scraping_browser-country-id-pid-' '{profileId}:{password}@cb.2captcha.com:9222', 'http://{user}:{password}@ap.proxy.2captcha.com:2334', @@ -2791,7 +2874,7 @@ def test_proxy_pool(): # and the check that used to live here went with it. `--concurrency` is # refused on this site (page 5 of an infinitely scrolling grid has no # address to hand a worker), so a per-worker pool would be machinery with - # no caller: the defect §17 names, tested into looking load-bearing. + # no caller: dead code, tested into looking load-bearing. # # What replaces it is the rotation a SEQUENTIAL run actually uses. A pool # here spreads several runs, or several block-retries within one, rather @@ -2869,7 +2952,7 @@ def test_proxy_pool(): # was written to show. Every caller here passes the URL as its own `%s` # argument for that reason, and `_mask_credentials()` is what handles # arbitrary text. Asserting the CURRENT behaviour makes a future swap a - # failing check rather than an unreadable log (§10). + # failing check rather than an unreadable log. sentence = "a http://u:supersecret@h.example:8080 b" ok &= check("mask() on a sentence loses the host — the documented limit", mask(sentence) == "?://?") @@ -2994,7 +3077,7 @@ def test_engines(skips): # THE FLAG CONTRACT, and the exact ways the engines differ from it. # - # §9 lists the flags every engine must offer. Checked rather than + # The family's flag contract lists the flags every engine must offer. Checked rather than # trusted, because a flag one engine has and another does not is the # drift page_flow.py and finish_run() exist to prevent, one level up — # and because the README documents these differences by name, so a new @@ -3034,7 +3117,7 @@ def test_engines(skips): # `--fp-tags` MUST DEFAULT TO ONE OS-FAMILY TAG. It shipped in this # family as "Windows,Chrome,Desktop", which the fingerprint API rejects # with HTTP 400 — so --fingerprint failed on every invocation, which is - # one of the six defects §16 of the family notes lists. Measured + # one of six defects found in copied fingerprint code. Measured # 2026-09-10 against the live API: `Windows` succeeds; # `Windows,Chrome,Desktop`, `Chrome` and `Desktop` each 400. for name in _ENGINE_MODULES: @@ -3344,7 +3427,7 @@ def test_parse_is_gated_on_the_policy(): # engine parsed whatever reached the parse line, so the `parse` column # of STATE_POLICY decided nothing and an engine could disagree with the # table — and with its twins — without anything noticing. That is the - # defect CLAUDE.md §17 names for constants, with a function instead. + # defect of a policy constant nothing reads, with a function instead. # # Measured across the family on 2026-09-23 by counting definitions # against readers: 7 of 24 repos defined it and none called it. @@ -3429,6 +3512,180 @@ def test_retries_is_validated_in_every_entry_point(): return check("--retries is validated in every entry point, so 0 is a usage error", not problems) +def _load_ci_checks(): + import importlib.util as _ilu + path = os.path.join(REPO_ROOT, ".github", "ci_checks.py") + spec = _ilu.spec_from_file_location("_ci_checks_probe", path) + mod = _ilu.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def test_credential_scan_reads_what_matters(): + group("the credential scan: relative exclusions, backups, captures") + ok = True + if not os.path.exists(os.path.join(REPO_ROOT, ".github", "ci_checks.py")): + # The Docker image carries no .github; nothing to test there. + return check("ci_checks.py is absent with the whole .github dir", + not os.path.isdir(os.path.join(REPO_ROOT, ".github"))) + key = "a" * 32 + with tempfile.TemporaryDirectory() as td: + # A clone that lives UNDER a directory called `venv`: exclusions + # tested against the absolute path scanned 0 files here and said ok. + root = os.path.join(td, "venv", "clone") + os.makedirs(os.path.join(root, "myenv", "lib")) + open(os.path.join(root, "myenv", "pyvenv.cfg"), "w").close() + planted = {"note.txt": key, ".env.bak": "TWOCAPTCHA_KEY=" + key, + "dump.json": '{"k": "%s"}' % key, + "foo.html": "

%s

" % key, + "canary_run.json.page2": key} + for name, body in planted.items(): + with open(os.path.join(root, name), "w") as fh: + fh.write(body + "\n") + # Inside a real virtualenv: must NOT be read. + with open(os.path.join(root, "myenv", "lib", "vendored.py"), "w") as fh: + fh.write("# %s\n" % key) + + cc = _load_ci_checks() + cc.REPO = __import__("pathlib").Path(root) + cc._VENV_CACHE.clear() + with redirect_stdout(io.StringIO()): + found = cc.secret_check() + blob = "\n".join(found) + for name in planted: + ok &= check("a key planted in %s is found, under a `venv` parent" + % name, (name + ":") in blob) + ok &= check("a directory holding pyvenv.cfg is still skipped", + "vendored.py" not in blob) + + empty = os.path.join(td, "empty") + os.makedirs(empty) + cc.REPO = __import__("pathlib").Path(empty) + cc._VENV_CACHE.clear() + with redirect_stdout(io.StringIO()): + found = cc.secret_check() + ok &= check("a scan that read 0 files FAILS instead of saying ok", + any("0 files scanned" in f for f in found)) + return ok + + +def _public_files(): + """Every file this repo publishes: git's index if there is one.""" + try: + out = subprocess.run(["git", "ls-files"], cwd=REPO_ROOT, + capture_output=True, text=True, check=True).stdout + names = [n for n in out.splitlines() if n] + if names: + return names + except (OSError, subprocess.CalledProcessError): + pass + names = [] + for dirpath, dirnames, filenames in os.walk(REPO_ROOT): + dirnames[:] = [d for d in dirnames + if d not in (".git", "__pycache__") + and not os.path.exists(os.path.join(dirpath, d, "pyvenv.cfg"))] + for f in filenames: + names.append(os.path.relpath(os.path.join(dirpath, f), REPO_ROOT)) + return names + + +def test_no_private_citations(): + group("public files cite no private notes") + # Assembled from pieces so this file does not match itself. + banned = (chr(0xA7), "CLAUDE" + ".md", "claude_" + "2scraper", + "2scraper_" + "repo_rules", "play" + "book") + hits = [] + for name in _public_files(): + path = os.path.join(REPO_ROOT, name) + try: + text = open(path, encoding="utf-8").read() + except (OSError, UnicodeDecodeError): + continue + for lineno, line in enumerate(text.splitlines(), 1): + for token in banned: + if token in line: + hits.append("%s:%d (%r)" % (name, lineno, token)) + for h in hits[:10]: + print(" " + h) + return check("no section-sign references or private file names in any " + "published file (%d found)" % len(hits), not hits) + + +def test_numeric_flags_are_validated(): + group("numeric flags reject values that cannot mean anything") + import argparse as _ap + ok = True + want = {"--pages": "page_flow.pages_arg", "--delay": "page_flow.delay_arg", + "--retry-delay": "page_flow.retry_delay_arg", + "--concurrency": "page_flow.concurrency_arg", + "--proxy-block-retries": "page_flow.block_retries_arg", + "--retries": "page_flow.attempts"} + for eng in ENGINE_FILES: + tree = ast.parse(open(os.path.join(REPO_ROOT, eng), encoding="utf-8").read()) + got = {} + for node in ast.walk(tree): + if (isinstance(node, ast.Call) + and getattr(node.func, "attr", "") == "add_argument" + and node.args and isinstance(node.args[0], ast.Constant) + and node.args[0].value in want): + got[node.args[0].value] = next( + (ast.unparse(k.value) for k in node.keywords + if k.arg == "type"), None) + ok &= check("%s validates every counted flag" % eng, got == want) + if got != want: + print(" %s" % {k: v for k, v in got.items() if want.get(k) != v}) + bad = [("pages_arg", "0"), ("pages_arg", "-2"), ("concurrency_arg", "0"), + ("block_retries_arg", "-1"), ("delay_arg", "-5"), + ("retry_delay_arg", "-1"), ("retry_delay_arg", "nan"), + ("pages_arg", "x")] + for fn, value in bad: + try: + getattr(page_flow, fn)(value) + rejected, msg = False, "" + except _ap.ArgumentTypeError as e: + rejected, msg = True, str(e) + flag = "--" + fn[:-4].replace("_", "-").replace("block-retries", + "proxy-block-retries") + ok &= check("%s rejects %r, naming the flag" % (flag, value), + rejected and flag in msg) + good = [("pages_arg", "1", 1), ("block_retries_arg", "0", 0), + ("delay_arg", "0", 0.0), ("retry_delay_arg", "2.5", 2.5), + ("concurrency_arg", "1", 1)] + for fn, value, out in good: + ok &= check("%s accepts %r" % (fn, value), + getattr(page_flow, fn)(value) == out) + return ok + + +def test_status_reaches_the_classifier(): + group("the HTTP status of the navigation reaches the classifier") + ok = True + good = grid(*[h for _, h, _ in ALL_CARDS]) + ok &= check("200 with cars is content", + page_flow.classify(good, 200, LISTING_URL) == "content") + ok &= check("404 is notfound whatever the body says", + page_flow.classify(good, 404, LISTING_URL) == "notfound") + ok &= check("503 is blocked, not content", + page_flow.classify(good, 503, LISTING_URL) == "blocked") + ok &= check("a status that is not an int is read as unknown", + page_flow.classify(good, object(), LISTING_URL) == "content") + # Each browser engine whose driver returns a response passes its status + # on every classify call that judges the navigated page. + for eng, needle in (("playwright_scraper.py", "_classify"), + ("puppeteer_scraper.py", "classify")): + tree = ast.parse(open(os.path.join(REPO_ROOT, eng), encoding="utf-8").read()) + calls = [n for n in ast.walk(tree) + if isinstance(n, ast.Call) + and getattr(n.func, "attr", getattr(n.func, "id", "")) == needle] + with_status = [c for c in calls + if any(isinstance(a, ast.Name) and a.id == "status" + for a in c.args) + or any(k.arg == "status" for k in c.keywords)] + ok &= check("%s passes a status to at least two classify calls" + % eng, len(with_status) >= 2) + return ok + + def main() -> int: ok = True # Checks that could not run because an optional engine library is absent. @@ -3473,6 +3730,10 @@ def main() -> int: ok &= test_sample_output() ok &= test_x_debug_header_is_redacted() ok &= test_scraper_api_waitfor_is_an_object() + ok &= test_credential_scan_reads_what_matters() + ok &= test_no_private_citations() + ok &= test_numeric_flags_are_validated() + ok &= test_status_reaches_the_classifier() print() if _failures: