diff --git a/CHANGELOG.md b/CHANGELOG.md index a62a04e..ca93533 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,20 @@ that is a fix, and the release notes lead with it. ## [Unreleased] +### Fixed + +- **`--retries 0` sent no request at all and reported exit 3.** Every engine + loops `range(1, retries + 1)` around the navigation, so 0 meant zero + navigations: the page stayed `about:blank` (39 bytes of empty DOM), the + classifier called that `blocked`, and the run exited 3 with a 39-byte debug + dump having never contacted the site. `--retries` now refuses anything + below 1 (`page_flow.attempts_arg`), in all three engines. +- **The block advice and three help/log texts blamed the browser build.** They + still said Vrbo "reads the CLIENT before the address" and that bundled + Chromium is refused, which the README's own 12-of-12 interleaved + measurement had already retracted. They now say the exit's score is noisy + and the browser is not the first lever. + ### CI - **The canary asks for three pages, not two.** Two exercises the next-button diff --git a/README.md b/README.md index 6ad4281..906df43 100644 --- a/README.md +++ b/README.md @@ -222,7 +222,7 @@ last night's good output. `--allow-empty` is the opt-out. |---|---| | `playwright_scraper.py` | **Recommended.** Needs `--browser-channel chrome` (the default) — `playwright install chrome` | | `selenium_scraper.py` | Drives the Chrome you already have, so it gets the right browser for free. **Cannot authenticate a proxy** (`--proxy-server` has nowhere to put a password) and **cannot use an authenticated CDP endpoint** (`debuggerAddress` is a bare `host:port`) | -| `puppeteer_scraper.py` | pyppeteer is effectively unmaintained. It downloads its **own Chromium**, which this site refuses, so `--chromium-path /path/to/chrome` is required in practice. It warns before the run rather than after a blocked page | +| `puppeteer_scraper.py` | pyppeteer is effectively unmaintained. It downloads its **own, old Chromium**; an installed Chrome is the more faithful client, so `--chromium-path /path/to/chrome` is recommended (browser build was not measured to decide access, see the note at the top). It warns before the run rather than after a blocked page | | `scraper_api_client.py` | One HTTP request per page via the 2Captcha Scraper API, no local browser. **Measured on this site**: HTTP 200, 3.2 MB, and **3 of 50 cards** at $0.0005 — see below | All three browser engines produce the same rows. Measured on three live runs diff --git a/page_flow.py b/page_flow.py index d6c5ba4..b166574 100644 --- a/page_flow.py +++ b/page_flow.py @@ -296,6 +296,26 @@ def is_unpainted(state: str, html: Optional[str]) -> bool: # nothing consults is the same defect as dead code (§17). RETRY_ON_BLOCKED = True BLOCK_RETRIES_WITHOUT_POOL = 2 + + +def attempts_arg(value: str) -> int: + """argparse type for `--retries`: an ATTEMPT count, so at least 1. + + Every engine loops `range(1, retries + 1)` around its navigation, so 0 + meant zero navigations: the page stayed `about:blank` (39 bytes of empty + DOM), classified as `blocked`, and the run reported exit 3 having never + sent a request to the site. Refusing it at the parser is cheaper than + teaching that loop what zero means. + """ + import argparse + try: + n = int(value) + except ValueError: + raise argparse.ArgumentTypeError(f"{value!r} is not an integer") + if n < 1: + raise argparse.ArgumentTypeError( + f"{n} attempts would never load the page; use 1 for no retry") + return n BLOCK_RETRIES_WITH_POOL = 3 # One solve per page. Detection is broad on purpose, but a second solve on @@ -307,8 +327,11 @@ def block_advice(html: Optional[str], headless: bool, has_pool: bool) -> str: """What a reader should actually DO about this block. Exists because the honest first answer on this site is almost never "get - a better proxy" — it is "use a real Chrome" — and a message that says so - saves an afternoon and a proxy bill. + a better proxy" — it is "slow down, the 429 is a rate response" — and a + message that says so saves an afternoon and a proxy bill. It used to say + "use a real Chrome" instead, on the strength of three single requests; + the interleaved re-measurement (12 of 12 served on both builds) found no + browser effect, so the browser is no longer named as the cause. """ vendor = challenge_vendor(html or "") marker = detect_block_marker(html or "") or "HTTP 429" @@ -318,14 +341,15 @@ def block_advice(html: Optional[str], headless: bool, has_pool: bool) -> str: f"{vendor!r} this time, which this repo has no solver for, so " f"no solve was attempted and nothing was charged") hints = [ - "this site reads the CLIENT before the address: a bundled Chromium " - "was answered 429 and a real Chrome 200 from the same exit seconds " - "apart, so run the Playwright engine with its default " - "channel=chrome before reaching for anything else", + "how this site scores the exit address is noisy: the same address " + "was served and challenged minutes apart, and bundled Chromium and " + "real Chrome were served 12 of 12 when interleaved on a clean one, " + "so changing the browser is not the first lever", ] if headless: - hints.append("and with a real window — --headful is the default here " - "for that reason") + hints.append("a real window (--headful, the default here) is still " + "a more faithful client, but it was not what decided " + "access when measured") if not has_pool: hints.append("the 429 is a RATE response and it clears: the same URL " "was served minutes later from the same address. Slow " diff --git a/playwright_scraper.py b/playwright_scraper.py index 4c408d7..4ec5025 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -352,12 +352,12 @@ def _proxy_failure(exc) -> str: def _launch_local(pw, args, pool): """Launch a browser on `pool`'s current exit; return (browser, context, page). - Uses REAL Chrome by default, and that is the single most load-bearing - line in this engine. Measured on this site, same address, seconds apart: - bundled Chromium 429, real Chrome 200. If the channel is unavailable the - launch falls back to the bundled Chromium and says clearly that the run - is now very likely to be refused — silently falling back would turn a - missing browser into "the site blocked us". + Uses REAL Chrome by default because it is the more faithful client, not + because it was shown to be the gate: three single requests once said so, + and twelve interleaved ones (6 + 6, 2026-09-14) found no difference. If + the channel is unavailable the launch falls back to the bundled Chromium + and says so — silently falling back would turn a missing browser into + "the site blocked us". Factored out so a proxy rotation can tear the whole browser down and call it again. Swapping the proxy under a live session would be cheaper and @@ -377,16 +377,18 @@ def _launch_local(pw, args, pool): except (PWError, PWTimeout) as e: logger.warning( "Could not launch the %r channel (%s) — falling back to " - "Playwright's bundled Chromium. EXPECT HTTP 429: this site " - "was measured refusing the bundled Chromium and serving real " - "Chrome from the same address seconds apart. Install it with " - "`playwright install chrome`, or point --browser-channel at " - "one you have (msedge also works).", + "Playwright's bundled Chromium. A single early reading had " + "it refused where real Chrome was served, but an interleaved " + "re-measurement (12 of 12 on both) did not reproduce that; " + "real Chrome is still the more faithful client. Install it " + "with `playwright install chrome`, or point " + "--browser-channel at one you have (msedge also works).", channel, str(e)[:160]) browser = pw.chromium.launch(**launch_kwargs) else: - logger.warning("--browser-channel '' launches the bundled Chromium, " - "which this site was measured refusing with HTTP 429.") + logger.warning("--browser-channel '' launches the bundled Chromium: " + "a less faithful client than real Chrome, though not " + "measured to be refused more often (see README).") browser = pw.chromium.launch(**launch_kwargs) ctx_kwargs = {"user_agent": _chrome_ua(browser.version), @@ -1262,7 +1264,7 @@ def parse_args(): "sequential button presses and there is nothing to " "hand a second worker. Run several searches in " "parallel instead, one process each.") - p.add_argument("--retries", type=int, default=3, + p.add_argument("--retries", type=page_flow.attempts_arg, default=3, help="Attempts per page load before giving up (default 3). " "The pause between attempts doubles each time. A page " "that comes back EMPTY is not retried — see " @@ -1281,13 +1283,12 @@ def parse_args(): p.add_argument("--browser-channel", default=DEFAULT_BROWSER_CHANNEL, metavar="CHANNEL", help=f"Which installed browser to drive (default " - f"{DEFAULT_BROWSER_CHANNEL!r}). THE flag that decides " - f"whether this works: measured on the same address " - f"seconds apart, Playwright's bundled Chromium was " - f"answered HTTP 429 and real Chrome HTTP 200 with the " - f"full grid. Install it with `playwright install " - f"chrome`. Pass an empty string to use the bundled " - f"Chromium anyway.") + f"{DEFAULT_BROWSER_CHANNEL!r}). A more faithful client " + f"than the bundled Chromium; interleaved on a clean " + f"address both were served 12 of 12, so it is not " + f"what decides access. Install it with `playwright " + f"install chrome`. Pass an empty string to use the " + f"bundled Chromium anyway.") p.add_argument("--proxy", default=None, help="Proxy URL, e.g. http://ACCOUNT:PASSWORD@HOST:9999 " "(2captcha.com/proxy)") @@ -1379,10 +1380,9 @@ def parse_args(): "as well as failure. Useful when the row count is " "right but a column comes back empty — see " "TROUBLESHOOTING.md.") - # HEADFUL by default. The measured discriminator on this site is the - # browser BUILD rather than the window (bundled Chromium was refused - # headful), but a real window costs nothing next to real Chrome and - # removes one variable from a refusal. + # HEADFUL by default. Neither the window nor the browser build was shown + # to decide access (see README), but a real window costs nothing next to + # real Chrome and removes one variable from a refusal. p.add_argument("--headful", dest="headless", action="store_false", default=False, help="Run with a real browser window. THE DEFAULT here.") diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index f9364e5..536ea8c 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -639,10 +639,10 @@ def _fetch_one_page(session, args, pool, page_num: int, debug_html) if not args.chromium_path and not args.cdp_endpoint: logger.error( - "This engine launched pyppeteer's OWN Chromium, which is the " - "build this site was measured refusing. Pass --chromium-path " - "pointing at an installed Chrome before concluding anything " - "about the address.") + "This engine launched pyppeteer's OWN Chromium, an old build " + "and a less faithful client than an installed Chrome. It was " + "not measured to be what decides access, but pass " + "--chromium-path to rule it out before blaming the address.") logger.error("%s", page_flow.block_advice( html, headless=bool(getattr(args, "headless", False)), has_pool=has_pool)) @@ -885,7 +885,7 @@ def parse_args(): p.add_argument("--concurrency", type=int, default=1, metavar="N", help="Accepted for family compatibility and REFUSED above " "1: a Vrbo listing has no per-page address.") - p.add_argument("--retries", type=int, default=3, + p.add_argument("--retries", type=page_flow.attempts_arg, default=3, help="Attempts per page load before giving up (default 3).") p.add_argument("--retry-delay", type=float, default=2.0, help="Seconds before the first page-load retry, doubling " diff --git a/selenium_scraper.py b/selenium_scraper.py index 42b7fdd..325a42d 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -11,10 +11,10 @@ Two things are different about this ENGINE: * **It uses real Chrome for free.** chromedriver drives the Chrome that is - installed, which is exactly what this site wants — the Playwright engine - has to be told `--browser-channel chrome` to get there, and its bundled - Chromium is refused with HTTP 429. So there is no browser-channel flag - here: there is nothing to choose. + installed — the Playwright engine has to be told `--browser-channel + chrome` to get there. So there is no browser-channel flag here: there is + nothing to choose. (Real Chrome is the more faithful client; it was not + measured to be what decides access.) * **It cannot authenticate a proxy, and it cannot use an authenticated CDP endpoint.** `--proxy-server` takes an address with nowhere to put a @@ -874,7 +874,7 @@ def parse_args(): help="Accepted for family compatibility and REFUSED above " "1: a Vrbo listing has no per-page address, so there " "is nothing to hand a second worker.") - p.add_argument("--retries", type=int, default=3, + p.add_argument("--retries", type=page_flow.attempts_arg, default=3, help="Attempts per page load before giving up (default 3).") p.add_argument("--retry-delay", type=float, default=2.0, help="Seconds before the first page-load retry, doubling " diff --git a/smoke_test.py b/smoke_test.py index 76b07de..3f69425 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -504,8 +504,10 @@ def test_challenge_is_not_always_solvable(): group("block_advice says what to DO") advice = page_flow.block_advice(block, headless=False, has_pool=False) - ok &= check("it leads with the browser, not the proxy", - "channel=chrome" in advice) + ok &= check("it does NOT blame the browser build (12 of 12 served on both)", + "channel=chrome" not in advice and "reads the CLIENT" not in advice) + ok &= check("it says the exit's score is noisy and the browser is not the first lever", + "noisy" in advice and "not the first lever" in advice) ok &= check("it says the vendor could not be solved and nothing was charged", "nothing was charged" in advice) ok &= check("it names the rate response as clearable", @@ -513,6 +515,26 @@ def test_challenge_is_not_always_solvable(): return ok +def test_retries_zero_is_refused(): + group("--retries 0 would never navigate, so the parser refuses it") + ok = True + import argparse + ok &= check("1 is accepted", page_flow.attempts_arg("1") == 1) + for bad in ("0", "-1", "x"): + try: + page_flow.attempts_arg(bad) + raised = False + except argparse.ArgumentTypeError: + raised = True + ok &= check(f"{bad!r} is refused", raised) + import ast + for eng in ("playwright_scraper", "puppeteer_scraper", "selenium_scraper"): + src = open(eng + ".py", encoding="utf-8").read() + ok &= check(f"{eng} wires --retries through attempts_arg", + '"--retries", type=page_flow.attempts_arg' in src) + return ok + + def test_page_flow_policy(): group("STATE_POLICY — the triage as DATA, not three if-chains") ok = True @@ -1559,6 +1581,7 @@ def main() -> int: ok &= test_pagination() ok &= test_page_state() ok &= test_challenge_is_not_always_solvable() + ok &= test_retries_zero_is_refused() ok &= test_page_flow_policy() ok &= test_scroll_loop() ok &= test_throttle_is_not_completion()