From 0891766151de2d077814548565ad219440430414 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:28:58 +0000 Subject: [PATCH] Stop a run on a refused proxy before the first request, and skip media through a proxy A proxy that answers 407 or 402 to a one-request pre-check (CONNECT plus HEAD /robots.txt on www.sleepnumber.com) now ends the run with exit 5 and .latest_attempt.meta.json (status failed, stop_reason proxy_auth_failed / proxy_balance_exhausted), in http_scraper.py and all three browser engines, instead of failing every retry. Through a local proxy the browser engines abort image, media and font requests; --load-media or SLEEPNUMBER_LOAD_MEDIA=1 loads them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CKsd7q25N19WzUZjrAdzx8 --- .env.example | 6 + CHANGELOG.md | 32 +++ Dockerfile | 2 +- README.md | 43 +++- env_config.py | 2 + http_scraper.py | 14 +- output_writer.py | 29 +++ playwright_scraper.py | 17 +- proxy_check.py | 383 ++++++++++++++++++++++++++++++++++ puppeteer_scraper.py | 17 +- selenium_scraper.py | 17 +- smoke_test.py | 466 ++++++++++++++++++++++++++++++++++++++++++ 12 files changed, 1020 insertions(+), 8 deletions(-) create mode 100644 proxy_check.py diff --git a/.env.example b/.env.example index ec64db4..c3eadac 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,12 @@ TWOCAPTCHA_KEY={your_2captcha_api_key} # --------------------------------------------------------------------------- SLEEPNUMBER_PROXY=http://{login}-zone-resi-region-us:{password}@proxy.2captcha.com:2334 +# Through a proxy, the browser engines do not download images, video/audio +# or fonts: the proxy is billed by traffic and the catalogue is read from the +# page's inline payload. Set to 1 to load them anyway (same as --load-media). +# Left empty = block. http_scraper.py fetches HTML only, so it is unaffected. +SLEEPNUMBER_LOAD_MEDIA= + # --------------------------------------------------------------------------- # SLEEPNUMBER_CDP_ENDPOINT — the 2Captcha Scraping Browser API, over CDP. # diff --git a/CHANGELOG.md b/CHANGELOG.md index d5c232b..015de05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,38 @@ nobody discovers it from a bill or a broken cron job. ## [Unreleased] +> **Behaviour changes.** (1) A run through a proxy now makes one pre-check +> request before its first page request, in `http_scraper.py` and all three +> browser engines. A proxy that answers **HTTP 407** or **402** ends the run +> with **exit 5** and `.latest_attempt.meta.json` (`status: failed`, +> `stop_reason: proxy_auth_failed` / `proxy_balance_exhausted`) instead of +> failing every retry. (2) Through a local proxy, the browser engines **no +> longer download** images, video/audio or fonts; `--load-media` (or +> `SLEEPNUMBER_LOAD_MEDIA=1`) restores them. Runs without a proxy, over +> `--cdp-endpoint`, and the HTML-only `http_scraper.py` fetch are otherwise +> unchanged. + +### Added + +- `proxy_check.py`: the proxy pre-check (one CONNECT plus `HEAD /robots.txt` + on www.sleepnumber.com, no third-party service), shared by + `http_scraper.py` and the browser engines, plus media blocking for the + three browser engines. The 407 message names `SLEEPNUMBER_PROXY` and says + the gateway refused the credential (or the traffic balance is exhausted); + a 402 says the proxy traffic/balance is exhausted. Neither prints the + credential. A proxy that does not answer keeps the existing retry + behaviour. +- `--load-media` on the three browser engines, and `SLEEPNUMBER_LOAD_MEDIA` + in `.env.example`. +- Offline checks: + - a stub proxy that answers 407/402/200 or nothing; + - `http_scraper.py` and each engine driven to exit 5 on a 407, with no + page request and no browser; + - the attempt sidecar; + - the Playwright route handler, the pyppeteer Fetch patterns and the + Selenium blocked-URL list; + - when each engine installs blocking. + ### Changed - CI only: the canary bills the `SLEEPNUMBER_PROXY` exit, so it now runs weekly diff --git a/Dockerfile b/Dockerfile index 5913ea7..0e26d91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,7 @@ RUN pip install --no-cache-dir -r requirements.txt -r requirements-playwright.tx # because nothing built it. COPY captcha_solver.py env_config.py fingerprint_client.py output_writer.py \ page_flow.py playwright_scraper.py product_parser.py proxy_pool.py \ - scraper_api_client.py diff_runs.py http_scraper.py ./ + scraper_api_client.py diff_runs.py http_scraper.py proxy_check.py ./ # No test suite, no fixtures, no .github/ and no .env — see .dockerignore. ENTRYPOINT ["python", "playwright_scraper.py"] diff --git a/README.md b/README.md index e4441fe..11e816e 100644 --- a/README.md +++ b/README.md @@ -458,6 +458,39 @@ address and does not change between attempts, so retrying buys a second copy of a certainty and is billed for it. A challenge page still retries, because that one can clear. +### Proxy traffic savers + +A proxy is billed by traffic, so a run through `--proxy` / `--proxy-file` / +`SLEEPNUMBER_PROXY` does two things to avoid paying for bytes it does not +use: + +- **One pre-check before the first page request, in every entry point** + (`http_scraper.py` and the three browser engines). It is a CONNECT through + the proxy and a `HEAD /robots.txt` on www.sleepnumber.com. Chromium + reports a proxy's HTTP 407 as a navigation timeout, and `requests` reports + it inside a `ProxyError`, so a wrong or exhausted credential used to spend + every retry before ending as `page_load_timeout`. Now a **407** (the + gateway refused the credential, or the traffic balance is exhausted) or a + **402** (proxy traffic/balance exhausted) ends the run in seconds with + **exit 5** and writes `.latest_attempt.meta.json` saying + `status: failed`, `stop_reason: proxy_auth_failed` (or + `proxy_balance_exhausted`). Any previous output and its `.meta.json` are + left as they were. The message names the variable and never prints the + credential. If the proxy does not answer at all, the run goes on as + before. No pre-check runs without a proxy or with `--cdp-endpoint`. +- **Images, video/audio and web fonts are not downloaded** by the browser + engines through a local proxy. Playwright aborts them by resource type, + pyppeteer through CDP's Fetch domain, and Selenium by file extension with + CDP `Network.setBlockedURLs`. The rows come from the page's inline + payload, which is why `http_scraper.py` (which never loads a sub-resource) + produces the same rows. `http_scraper.py` fetches HTML only, so blocking + does not apply to it. On 2026-10-20 a run with no proxy from a datacentre + got the same 923-byte CloudFront "Request blocked" 403 with media loaded + and with media blocked, so the refusal does not depend on what the page + loads; a served comparison needs a residential exit and was not made. + `--load-media` (or `SLEEPNUMBER_LOAD_MEDIA=1`) loads media. Without a + proxy, or over `--cdp-endpoint`, nothing is blocked. + ### Install exactly one engine `playwright` and `pyppeteer` pin mutually unsatisfiable `pyee` versions, and @@ -484,7 +517,9 @@ python3 playwright_scraper.py --mode product --category cm-mattress never replaced with `[]`. `--allow-empty` is the opt-out. - **Exit codes:** `0` ok · `1` crash · `2` bad usage · `3` blocked · `4` zero products · `5` the content was never obtained (a remote API - error, a navigation timeout, a dead proxy) · `6` partial. + error, a navigation timeout, a dead proxy, or a proxy that refused the + credential or the balance before the run began: `stop_reason: + proxy_auth_failed` / `proxy_balance_exhausted`) · `6` partial. - **`.meta.json` beside every run that WRITES ROWS**, recording `status`, `stop_reason`, `transport`, which pages failed by number, and the site's own arithmetic about the result set — `total_results`, `per_page`, @@ -504,7 +539,9 @@ python3 playwright_scraper.py --mode product --category cm-mattress "per run" and it meant "per run that writes rows". If you want a machine- readable record of attempts including failures, read the exit code and keep your own log; this repo will not put a contradicting file next to good - data. + data. The one exception is a run the proxy pre-check stopped (see "Proxy + traffic savers"): it writes `.latest_attempt.meta.json`, a file with + its own name, and leaves `.meta.json` alone. - **An empty CSV still carries its header.** - **Writes are atomic.** Each file is written beside its target and renamed over it, so a crash or a full disk mid-write leaves the previous file @@ -544,7 +581,7 @@ Precedence, highest first: explicit flag → exported environment variable → ## Tests ```bash -python3 smoke_test.py -v # 59 checks, no network, no engine needed +python3 smoke_test.py -v # 112 checks, no network, no engine needed pytest # the same checks, wrapped ``` diff --git a/env_config.py b/env_config.py index 7db2673..5eeebf6 100644 --- a/env_config.py +++ b/env_config.py @@ -41,6 +41,8 @@ "SLEEPNUMBER_CDP_ENDPOINT": "cdp_endpoint", "SLEEPNUMBER_PROXY": "proxy", "SLEEPNUMBER_URL": "url", + # "1" keeps images/video/fonts loading through a proxy (--load-media). + "SLEEPNUMBER_LOAD_MEDIA": "load_media", } # Deliberately NOT here: an output prefix. `--out` already carries a non-empty # default, so `apply()` would never see it as unset and the variable would be diff --git a/http_scraper.py b/http_scraper.py index 08cb55b..daef419 100644 --- a/http_scraper.py +++ b/http_scraper.py @@ -84,13 +84,15 @@ import requests -from output_writer import dedupe_by_key, finish_run, SOURCE_DEFAULT +from output_writer import (dedupe_by_key, finish_run, finish_without_fetch, + SOURCE_DEFAULT) import page_flow from product_parser import (parse_products, detect_bot_challenge, is_supported_url, normalise_url, category_url, product_url, SITE_ASSET_MARKERS) from proxy_pool import from_args as proxy_pool_from_args, mask, ROTATE_MODES, ProxyError import env_config +import proxy_check logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("http_scraper") @@ -294,6 +296,16 @@ def scrape(args) -> int: pool = proxy_pool_from_args(args) + # One request through the proxy before the first page request: a 407 or 402 + # from the gateway ends the run here, in seconds, instead of surfacing + # as a ProxyError on every retry. + refused = proxy_check.gate(pool, args.url) + if refused: + return finish_without_fetch( + args.out, stop_reason=refused.stop_reason, + pages_requested=args.pages, start_url=args.url, mode=args.mode, + source=SOURCE_DEFAULT) + with requests.Session() as session: first = _fetch_one_page(session, args, pool, 1, args.url) outcomes.append(first) diff --git a/output_writer.py b/output_writer.py index 8ceac9d..ec81264 100644 --- a/output_writer.py +++ b/output_writer.py @@ -456,6 +456,35 @@ def write_run_meta(out_prefix: str, meta: dict) -> str: return path +ATTEMPT_META_SUFFIX = ".latest_attempt.meta.json" + + +def finish_without_fetch(out_prefix: str, *, stop_reason: str, + pages_requested: int, start_url: str, + mode: str = "listing", source: str = SOURCE_DEFAULT, + message: str = "") -> int: + """End a run that was stopped before anything was fetched; exit 5. + + Used by the proxy pre-check (a 407 or 402 from the proxy). The record + goes to `.latest_attempt.meta.json`, NOT `.meta.json`: the + latter describes the data file beside it, and a previous run's good + output is still there, untouched, so overwriting its metadata with + `status: failed` would make the pair contradict each other. + """ + meta = run_meta(status="failed", stop_reason=stop_reason, + pages_requested=pages_requested, pages_completed=0, + start_url=start_url, final_url=start_url, products=0, + mode=mode, source=source) + path = f"{out_prefix}{ATTEMPT_META_SUFFIX}" + with _atomic(path) as f: + json.dump(meta, f, ensure_ascii=False, indent=2) + print(f"[+] Wrote attempt metadata -> {path} (status=failed, " + f"stop_reason={stop_reason})") + print(f"[!] Failed run: nothing was fetched ({stop_reason}). " + f"{message} Any previous output and its .meta.json are untouched.") + return EXIT_FETCH_FAILED + + def run_meta(status: str, stop_reason: str, pages_requested: int, pages_completed: int, start_url: str, final_url: str, products: int, pages_failed: Optional[List[int]] = None, diff --git a/playwright_scraper.py b/playwright_scraper.py index 16951ed..89d4995 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -79,11 +79,13 @@ category_url, product_url, CATEGORY_URL, CANONICAL_HOST) from output_writer import (dedupe_by_key, finish_run, RemoteAPIError, - EXIT_REMOTE_API_ERROR, SOURCE_DEFAULT) + EXIT_REMOTE_API_ERROR, SOURCE_DEFAULT, + finish_without_fetch) import page_flow from proxy_pool import (from_args as proxy_pool_from_args, to_playwright, mask, ROTATE_MODES, ProxyError, ProxyPool) import env_config +import proxy_check logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("playwright_scraper") @@ -188,6 +190,8 @@ def _launch_local(pw, args, pool): context = browser.new_context(**ctx_kwargs) if init_script: context.add_init_script(init_script) + if proxy_check.block_media(args, pool, remote=False): + proxy_check.install_playwright(context) return browser, context, context.new_page() @@ -847,6 +851,16 @@ def scrape(args) -> int: "its whole lifetime by design, " "rotated only per WORKER, not per page.") + # One request through the proxy before a browser exists: a 407 or 402 + # from the gateway ends the run here, in seconds, instead of surfacing + # as a navigation timeout after every retry. + refused = proxy_check.gate(pool, args.url) + if refused: + return finish_without_fetch( + args.out, stop_reason=refused.stop_reason, + pages_requested=args.pages, start_url=args.url, mode=args.mode, + source=SOURCE_DEFAULT) + with sync_playwright() as pw: session = _BrowserSession(pw, args, pool, remote=bool(args.cdp_endpoint)).open() @@ -1077,6 +1091,7 @@ def parse_args(): p.add_argument("--captcha-api", choices=["v2", "v1"], default="v2") p.add_argument("--solve-captcha", choices=["when-blocked", "always"], default="when-blocked") p.add_argument("--min-score", type=float, default=0.7) + proxy_check.add_media_flags(p) p.add_argument("--cdp-endpoint", default=None, help="Connect to an already-running browser over CDP, " "e.g. ws://user:pass@host:port — the Scraping " diff --git a/proxy_check.py b/proxy_check.py new file mode 100644 index 0000000..f94feab --- /dev/null +++ b/proxy_check.py @@ -0,0 +1,383 @@ +""" +proxy_check.py +--------------- +Two traffic savers for runs that go out through a local proxy. + +A residential proxy is billed by the byte, and both of these exist because +of what a run used to spend before it produced anything: + +1. **A pre-check before any browser starts.** Chromium does not report a + proxy's 407 as a 407. A rejected credential surfaced as + `Page.goto: Timeout ... exceeded`, retried, rotated and timed out again, + so a run with a wrong or exhausted proxy login spent minutes on retries + and ended "page_load_timeout", which points at the site. One CONNECT + through the proxy, followed by a HEAD for the target's own robots.txt, + names the real cause in a second or two: + + 407 the gateway refused the credential (or the traffic balance is + exhausted, which some gateways also answer with 407) + 402 the proxy's traffic or balance is exhausted + + Both end the run before a browser is launched: exit 5 (content not + obtained) and `.latest_attempt.meta.json` with `status: failed` and + `stop_reason: proxy_auth_failed` / `proxy_balance_exhausted`. That file + is separate from `.meta.json` on purpose: a failed attempt must not + overwrite the metadata of the previous run's good output, which `save` + leaves in place. + + Anything else, including a transport error (the proxy host does not + answer, DNS fails, TLS breaks), is logged and the run goes on exactly as + it did before this check existed: the browser then makes its own attempt + and the existing retry and classification rules apply. Only an answer the + proxy gave about the credential or the balance is treated as final. + + The target is the site's robots.txt rather than a third-party "what is my + IP" service: it is tiny, it is on the host the run is about to fetch + anyway, and it does not send the run's traffic to anyone else. + +2. **Media blocking.** Images, video/audio and web fonts are most of the + bytes of a page and none of the data this scraper reads. When (and only + when) the browser's traffic goes through a local proxy, requests of those + resource types are aborted. Not over --cdp-endpoint (the remote browser + has its own exit and its own billing, and is not ours to rewrite), and + not without a proxy (nothing is metered). `--load-media` (or + `SLEEPNUMBER_LOAD_MEDIA=1`) turns it off. + +The credential is never printed: every message names the proxy through +`proxy_pool.mask`, which keeps the host and port and replaces the login. +""" + +from __future__ import annotations + +import base64 +import logging +import socket +import ssl +from dataclasses import dataclass +from typing import Optional, Tuple +from urllib.parse import urlparse + +from proxy_pool import mask + +logger = logging.getLogger("proxy_check") + +# Where the proxy credential comes from, for the failure message. Named +# rather than "your proxy" because the person reading the log is often +# looking at a CI secret, not at a command line. +PROXY_SOURCES = "SLEEPNUMBER_PROXY (or --proxy / --proxy-file)" +LOAD_MEDIA_ENV = "SLEEPNUMBER_LOAD_MEDIA" + +# The rows come from the page's inline React Router payload, never from an +# image or a font: http_scraper.py, which loads no sub-resource at all, +# produces the same rows as the browser engines. The site's refusal is a +# CloudFront deny on the address (measured 2026-10-20: the same 403 with +# media loaded and blocked), not a script that watches what a page loads. +BLOCK_MEDIA_BY_DEFAULT = True + +PRECHECK_TIMEOUT = 15.0 + +STOP_AUTH_FAILED = "proxy_auth_failed" +STOP_BALANCE_EXHAUSTED = "proxy_balance_exhausted" + +# Playwright's `request.resource_type` names, and the same three as CDP's +# Fetch domain spells them (pyppeteer goes through that domain directly). +BLOCKED_RESOURCE_TYPES = frozenset({"image", "media", "font"}) +CDP_BLOCKED_RESOURCE_TYPES = ("Image", "Media", "Font") + +# Selenium has no request interception, but Chromium's CDP +# `Network.setBlockedURLs` takes wildcard URL patterns. Extensions are a +# coarser test than a resource type, so the list is limited to formats that +# are only ever media; a trailing `*` also catches `?width=...` variants. +BLOCKED_URL_PATTERNS = tuple( + f"*.{ext}{tail}" + for ext in ("png", "jpg", "jpeg", "gif", "webp", "avif", "bmp", "ico", + "svg", "mp4", "webm", "m4v", "mov", "m3u8", "mp3", "m4a", + "ogg", "wav", "woff", "woff2", "ttf", "otf", "eot") + for tail in ("", "?*")) + +_FALSE_WORDS = {"", "0", "false", "no", "off"} + + +@dataclass(frozen=True) +class ProxyCheck: + """What the pre-check learned. `fatal` is True only for 407 and 402.""" + + outcome: str # ok | auth_failed | balance_exhausted | transport_error | skipped + status: Optional[int] # the proxy's (or, through the tunnel, the site's) status + detail: str = "" + + @property + def fatal(self) -> bool: + return self.outcome in ("auth_failed", "balance_exhausted") + + @property + def stop_reason(self) -> Optional[str]: + return {"auth_failed": STOP_AUTH_FAILED, + "balance_exhausted": STOP_BALANCE_EXHAUSTED}.get(self.outcome) + + +def _read_head(sock, limit: int = 65536) -> Tuple[int, str]: + """Read an HTTP response head; return (status, reason).""" + buf = b"" + while b"\r\n\r\n" not in buf and b"\n\n" not in buf: + chunk = sock.recv(4096) + if not chunk: + break + buf += chunk + if len(buf) > limit: + break + line = buf.split(b"\n", 1)[0].decode("latin-1").strip() + parts = line.split(" ", 2) + if len(parts) < 2 or not parts[0].startswith("HTTP/") or not parts[1].isdigit(): + raise ValueError(f"not an HTTP response: {line[:60]!r}") + return int(parts[1]), (parts[2] if len(parts) > 2 else "") + + +def _auth_header(proxy) -> str: + # The username and password are sent exactly as the browser engines send + # them (proxy_pool.to_playwright / split_credentials pass urlparse's + # fields through unchanged), so the check judges the same credential the + # browser would present rather than a decoded variant of it. + if not (proxy.username or proxy.password): + return "" + token = base64.b64encode( + f"{proxy.username or ''}:{proxy.password or ''}".encode()).decode() + return f"Proxy-Authorization: Basic {token}\r\n" + + +def _classify(status: int, reason: str) -> Optional[ProxyCheck]: + if status == 407: + return ProxyCheck("auth_failed", 407, reason) + if status == 402: + return ProxyCheck("balance_exhausted", 402, reason) + return None + + +def precheck(proxy_url: Optional[str], target_url: str, + timeout: float = PRECHECK_TIMEOUT) -> ProxyCheck: + """One request through `proxy_url`: a HEAD for `target_url`'s robots.txt. + + For an https target that is a CONNECT to the target host, then the HEAD + inside the tunnel. A 407 or 402 is the proxy's own answer about the + credential or the balance; the site never sees the request. Any status + the SITE gives (a 403 to a datacentre exit, a 404 for a missing + robots.txt) is not this check's business and counts as `ok`: the proxy + let the request through. + """ + if not proxy_url: + return ProxyCheck("skipped", None, "no proxy") + proxy = urlparse(proxy_url) + if proxy.scheme not in ("http", "https"): + # SOCKS5 carries no credentials in any engine here (proxy_pool + # refuses an authenticated socks5 entry), so there is nothing to + # check that the browser would not report itself. + return ProxyCheck("skipped", None, f"{proxy.scheme} proxy") + target = urlparse(target_url or "") + if target.scheme not in ("http", "https") or not target.hostname: + return ProxyCheck("skipped", None, "no http(s) target URL") + try: + proxy_port = proxy.port or (443 if proxy.scheme == "https" else 80) + target_port = target.port or (443 if target.scheme == "https" else 80) + except ValueError: + return ProxyCheck("skipped", None, "unparseable port") + host_header = (target.hostname if target.port is None + else f"{target.hostname}:{target.port}") + auth = _auth_header(proxy) + sock = None + try: + sock = socket.create_connection((proxy.hostname, proxy_port), + timeout=timeout) + if proxy.scheme == "https": + sock = ssl.create_default_context().wrap_socket( + sock, server_hostname=proxy.hostname) + if target.scheme == "https": + sock.sendall((f"CONNECT {target.hostname}:{target_port} HTTP/1.1\r\n" + f"Host: {target.hostname}:{target_port}\r\n" + f"{auth}\r\n").encode()) + status, reason = _read_head(sock) + verdict = _classify(status, reason) + if verdict: + return verdict + if status != 200: + # The proxy refused the tunnel for another reason (a 403 for + # a blocked port, a 502 from an upstream). Not about the + # credential, so not final: the browser gets its own try. + return ProxyCheck("transport_error", status, + f"CONNECT answered {status} {reason}".strip()) + # The tunnel is up, so the proxy accepted the credential: that + # is the verdict. The HEAD below only completes the one request; + # whatever the site does with it (a refusal, a reset) is the + # browser's business and does not change the answer. + if proxy.scheme == "https": + # TLS inside TLS is not something the ssl module does on a + # wrapped socket; the CONNECT answer already says enough. + return ProxyCheck("ok", None, "tunnel established") + try: + tls = ssl.create_default_context().wrap_socket( + sock, server_hostname=target.hostname) + sock = tls + sock.sendall((f"HEAD /robots.txt HTTP/1.1\r\n" + f"Host: {host_header}\r\n" + "User-Agent: Mozilla/5.0\r\nAccept: */*\r\n" + "Connection: close\r\n\r\n").encode()) + status, reason = _read_head(sock) + except (OSError, ssl.SSLError, ValueError) as e: + return ProxyCheck("ok", None, + f"tunnel established; site: {type(e).__name__}") + return ProxyCheck("ok", status, reason) + sock.sendall((f"HEAD http://{host_header}/robots.txt HTTP/1.1\r\n" + f"Host: {host_header}\r\n{auth}" + "User-Agent: Mozilla/5.0\r\nAccept: */*\r\n" + "Connection: close\r\n\r\n").encode()) + status, reason = _read_head(sock) + return _classify(status, reason) or ProxyCheck("ok", status, reason) + except (OSError, ssl.SSLError, ValueError) as e: + return ProxyCheck("transport_error", None, type(e).__name__) + finally: + if sock is not None: + try: + sock.close() + except OSError: + pass + + +def failure_message(check: ProxyCheck, proxy_url: Optional[str]) -> str: + """The one-line explanation for a fatal pre-check. Never holds the login.""" + shown = mask(proxy_url) + if check.outcome == "auth_failed": + return (f"Proxy pre-check: {shown} answered HTTP 407 — the gateway " + f"refused the credential (or the traffic balance is " + f"exhausted). Check {PROXY_SOURCES} and the proxy account. " + f"No browser was started and nothing was fetched.") + return (f"Proxy pre-check: {shown} answered HTTP 402 — proxy " + f"traffic/balance exhausted. Top up the proxy account or change " + f"{PROXY_SOURCES}. No browser was started and nothing was fetched.") + + +def gate(pool, target_url: str, timeout: float = PRECHECK_TIMEOUT + ) -> Optional[ProxyCheck]: + """Run the pre-check on the pool's current exit; return it if fatal. + + One request for the whole run, on the exit the browser is about to use. + A pool's other exits share the account, so one refusal speaks for all. + """ + if not pool: + return None + exit_url = pool.current + check = precheck(exit_url, target_url, timeout=timeout) + if check.fatal: + logger.error("%s", failure_message(check, exit_url)) + return check + if check.outcome == "transport_error": + logger.warning("Proxy pre-check through %s did not complete (%s); " + "continuing, the browser will make its own attempt.", + mask(exit_url), check.detail) + elif check.outcome == "ok": + logger.info("Proxy pre-check through %s: accepted (%s).", + mask(exit_url), + f"site answered {check.status}" if check.status + else check.detail) + return None + + +# --------------------------------------------------------------------------- +# Media blocking +# --------------------------------------------------------------------------- + +def _truthy(value) -> bool: + if isinstance(value, str): + return value.strip().lower() not in _FALSE_WORDS + return bool(value) + + +def block_media(args, pool, remote: bool) -> bool: + """Whether this run should abort image/media/font requests. + + `--block-media` (hidden; for tests and measurements) forces it on even + without a proxy. Otherwise: only through a local proxy, only when the + site's default allows it, and never when `--load-media` / + `SLEEPNUMBER_LOAD_MEDIA` asks for media. + """ + if remote: + return False + if getattr(args, "block_media", False): + return True + if _truthy(getattr(args, "load_media", False)): + return False + return BLOCK_MEDIA_BY_DEFAULT and bool(pool) + + +def add_media_flags(parser) -> None: + """`--load-media` (public) and `--block-media` (hidden test switch).""" + import argparse + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--load-media", action="store_true", default=False, + help=("Load images, video/audio and fonts through the proxy. By " + "default they are blocked whenever a local --proxy/--proxy-file " + "is in use, to save metered proxy traffic; the data this " + "scraper reads does not depend on them. Also " + f"{LOAD_MEDIA_ENV}=1. No effect without a proxy or with " + "--cdp-endpoint.")) + group.add_argument("--block-media", action="store_true", default=False, + help=argparse.SUPPRESS) + + +def _should_abort(resource_type: str) -> bool: + return resource_type in BLOCKED_RESOURCE_TYPES + + +def install_playwright(context) -> None: + """Abort image/media/font requests on a Playwright context.""" + def _route(route): + if _should_abort(route.request.resource_type): + route.abort() + else: + route.continue_() + context.route("**/*", _route) + logger.info("Blocking image/media/font requests to save proxy traffic " + "(--load-media to load them).") + + +async def install_pyppeteer(page) -> None: + """Abort image/media/font requests on a pyppeteer page. + + Not `page.setRequestInterception`: that sends `Network. + setRequestInterception`, which current Chromium no longer has, and the + call fails with "wasn't found". The Fetch domain is its replacement. + It runs on a CDP session of its own, so this does not replace any Fetch + handling the engine's page session already has (proxy auth), and its + patterns pause ONLY the three blocked resource types: nothing else is + held back waiting for a continue. + """ + import asyncio + session = await page.target.createCDPSession() + + def _on_paused(event): + asyncio.ensure_future(session.send("Fetch.failRequest", { + "requestId": event["requestId"], "errorReason": "BlockedByClient"})) + session.on("Fetch.requestPaused", _on_paused) + await session.send("Fetch.enable", {"patterns": [ + {"urlPattern": "*", "resourceType": rtype, "requestStage": "Request"} + for rtype in CDP_BLOCKED_RESOURCE_TYPES]}) + logger.info("Blocking image/media/font requests to save proxy traffic " + "(--load-media to load them).") + + +def install_selenium(driver) -> bool: + """Block media URLs on a local chromedriver over CDP. False if refused. + + By file extension, not by resource type: Selenium has no request + interception, so a media file served without an extension still loads. + """ + try: + driver.execute_cdp_cmd("Network.enable", {}) + driver.execute_cdp_cmd("Network.setBlockedURLs", + {"urls": list(BLOCKED_URL_PATTERNS)}) + except Exception as e: # noqa: BLE001 — a saver must never break a run + logger.warning("Could not block media over CDP (%s); loading it.", + type(e).__name__) + return False + logger.info("Blocking image/media/font URLs to save proxy traffic " + "(--load-media to load them).") + return True diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index cb69069..e75c059 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -51,11 +51,13 @@ category_url, product_url, CATEGORY_URL, CANONICAL_HOST) from output_writer import (dedupe_by_key, finish_run, RemoteAPIError, - EXIT_REMOTE_API_ERROR, SOURCE_DEFAULT) + EXIT_REMOTE_API_ERROR, SOURCE_DEFAULT, + finish_without_fetch) import page_flow from proxy_pool import (from_args as proxy_pool_from_args, mask, ROTATE_MODES, ProxyError, split_credentials) import env_config +import proxy_check logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("puppeteer_scraper") @@ -238,6 +240,8 @@ def open(self): if credentials: self.bridge.run(self.page.authenticate( {"username": credentials[0], "password": credentials[1]})) + if proxy_check.block_media(self.args, self.pool, remote=False): + self.bridge.run(proxy_check.install_pyppeteer(self.page)) if self.args.fingerprint: self._apply_fingerprint() return self @@ -614,6 +618,16 @@ def scrape(args) -> int: "page fetching (--mode transfers) is implemented in " "playwright_scraper.py. Running sequentially.") + # One request through the proxy before a browser exists: a 407 or 402 + # from the gateway ends the run here, in seconds, instead of surfacing + # as a navigation timeout after every retry. + refused = proxy_check.gate(pool, args.url) + if refused: + return finish_without_fetch( + args.out, stop_reason=refused.stop_reason, + pages_requested=args.pages, start_url=args.url, mode=args.mode, + source=SOURCE_DEFAULT) + bridge = _AsyncBridge() session = _Session(bridge, args, pool).open() try: @@ -778,6 +792,7 @@ def parse_args(): p.add_argument("--captcha-api", choices=["v2", "v1"], default="v2") p.add_argument("--solve-captcha", choices=["when-blocked", "always"], default="when-blocked") p.add_argument("--min-score", type=float, default=0.7) + proxy_check.add_media_flags(p) p.add_argument("--cdp-endpoint", default=None, help="Connect to an existing browser, e.g. " "ws://user:pass@host:port — the Scraping Browser " diff --git a/selenium_scraper.py b/selenium_scraper.py index 29e06f5..b85f771 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -45,11 +45,13 @@ category_url, product_url, CATEGORY_URL, CANONICAL_HOST) from output_writer import (dedupe_by_key, finish_run, RemoteAPIError, - EXIT_REMOTE_API_ERROR, SOURCE_DEFAULT) + EXIT_REMOTE_API_ERROR, SOURCE_DEFAULT, + finish_without_fetch) import page_flow from proxy_pool import (from_args as proxy_pool_from_args, mask, ROTATE_MODES, ProxyError, split_credentials) import env_config +import proxy_check logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("selenium_scraper") @@ -173,6 +175,8 @@ def open(self): self.driver = webdriver.Chrome(options=options) self._apply_timeouts() + if proxy_check.block_media(self.args, self.pool, remote=False): + proxy_check.install_selenium(self.driver) version = self.driver.capabilities.get("browserVersion", "") if version: @@ -557,6 +561,16 @@ def scrape(args) -> int: "page fetching (--mode transfers) is implemented in " "playwright_scraper.py. Running sequentially.") + # One request through the proxy before a browser exists: a 407 or 402 + # from the gateway ends the run here, in seconds, instead of surfacing + # as a navigation timeout after every retry. + refused = proxy_check.gate(pool, args.url) + if refused: + return finish_without_fetch( + args.out, stop_reason=refused.stop_reason, + pages_requested=args.pages, start_url=args.url, mode=args.mode, + source=SOURCE_DEFAULT) + session = _Session(args, pool).open() try: first = _fetch_one_page(session, args, pool, 1, args.url) @@ -714,6 +728,7 @@ def parse_args(): p.add_argument("--captcha-api", choices=["v2", "v1"], default="v2") p.add_argument("--solve-captcha", choices=["when-blocked", "always"], default="when-blocked") p.add_argument("--min-score", type=float, default=0.7) + proxy_check.add_media_flags(p) p.add_argument("--cdp-endpoint", default=None, help="A CREDENTIAL-FREE debuggerAddress only — Selenium " "cannot authenticate on the WebSocket upgrade the " diff --git a/smoke_test.py b/smoke_test.py index c64bc06..90a5bca 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2020,6 +2020,472 @@ def test_no_private_citations_in_shipped_files(): assert not hits, f"private citations in shipped files: {hits[:10]}" +# =========================================================================== +# Proxy traffic savers: the pre-check and media blocking +# =========================================================================== +# Each sub-check is recorded under its own label, so a planted fault shows +# up by name rather than as one opaque failure. +import contextlib # noqa: E402 + + +def _savers_check(label, condition): + if condition: + PASSES.append(label) + if VERBOSE: + print(f"ok {label}") + else: + FAILURES.append((label, "failed")) + print(f"FAIL {label}") + return bool(condition) + + +def _savers_group(title): + if VERBOSE: + print(f"\n== {title}") + + +class _SaversSkips(list): + def append(self, msg): + skip("proxy savers", msg) +_SAVERS_URL = LISTING_URL +_SAVERS_SECRET = "s3cret-Pw-not-real" + + +class _StubProxy: + """A one-port HTTP proxy on 127.0.0.1 that answers every request with a + fixed status and records the request heads it received.""" + + def __init__(self, status): + import socket + import threading + self.status, self.heads = status, [] + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(8) + self.port = self.sock.getsockname()[1] + threading.Thread(target=self._serve, daemon=True).start() + + def _serve(self): + while True: + try: + conn, _ = self.sock.accept() + except OSError: + return + with conn: + data = b"" + while b"\r\n\r\n" not in data: + chunk = conn.recv(4096) + if not chunk: + break + data += chunk + self.heads.append(data.decode("latin-1")) + reason = {407: "Proxy Authentication Required", + 402: "Payment Required", 200: "OK"}.get( + self.status, "X") + conn.sendall(f"HTTP/1.1 {self.status} {reason}\r\n" + "Content-Length: 0\r\nConnection: close\r\n\r\n" + .encode()) + + def url(self, secret=_SAVERS_SECRET): + # Concatenated, so no line here looks like a credentialled URL. + return "http://user-zone-x:" + secret + "@127.0.0.1:" + str(self.port) + + def close(self): + self.sock.close() + + +def _closed_port(): + import socket + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +@contextlib.contextmanager +def _captured_output(): + """stdout and every log record, as one string.""" + import logging + buf = io.StringIO() + handler = logging.StreamHandler(buf) + root = logging.getLogger() + root.addHandler(handler) + try: + with contextlib.redirect_stdout(buf): + yield buf + finally: + root.removeHandler(handler) + + +class _FakeRoute: + def __init__(self, resource_type): + self.request = type("R", (), {"resource_type": resource_type})() + self.verdict = None + + def abort(self): + self.verdict = "abort" + + def continue_(self): + self.verdict = "continue" + + +def _media_verdicts(handler): + out = {} + for rtype in ("image", "media", "font", "document", "script", "xhr", + "fetch", "stylesheet"): + route = _FakeRoute(rtype) + handler(route) + out[rtype] = route.verdict + return out + + +def _engine_args(engine, tmp, proxy_url, *extra): + """The engine's own parser, on a stubbed argv, with no .env in play.""" + from unittest import mock + argv = [engine.__name__, "--url", _SAVERS_URL, "--out", + os.path.join(tmp, "out")] + if proxy_url: + argv += ["--proxy", proxy_url] + argv += list(extra) + with mock.patch.object(sys, "argv", argv), \ + mock.patch.object(env_config, "load_env", lambda *a, **k: None): + return engine.parse_args() + + +def test_proxy_traffic_savers(skips, check, group): + group("Proxy traffic savers: pre-check before the browser, media " + "blocking through a proxy") + import argparse + import asyncio + from unittest import mock + import output_writer + import proxy_check + from proxy_pool import ProxyPool + ok = True + + # -- the pre-check, against a stub proxy -------------------------------- + stub = _StubProxy(407) + try: + res = proxy_check.precheck(stub.url(), _SAVERS_URL, timeout=5) + ok &= check("a 407 from the proxy is auth_failed and fatal " + f"({res.outcome})", res.outcome == "auth_failed" + and res.fatal and res.stop_reason == "proxy_auth_failed") + head = stub.heads[-1] if stub.heads else "" + ok &= check("the pre-check is ONE CONNECT to the target host, with " + "the credential", len(stub.heads) == 1 + and head.startswith("CONNECT www.sleepnumber.com:443 ") + and "Proxy-Authorization: Basic " in head) + with _captured_output() as buf: + gated = proxy_check.gate(ProxyPool([stub.url()]), _SAVERS_URL, + timeout=5) + text = buf.getvalue() + ok &= check("gate() returns the fatal check on a 407", + gated is not None and gated.fatal) + ok &= check("the 407 message names the variable and the cause", + proxy_check.PROXY_SOURCES.split()[0] in text + and "refused the credential" in text + and "traffic balance is exhausted" in text) + ok &= check("the 407 message never prints the credential", + _SAVERS_SECRET not in text and "user-zone-x" not in text) + finally: + stub.close() + + stub = _StubProxy(402) + try: + res = proxy_check.precheck(stub.url(), _SAVERS_URL, timeout=5) + msg = proxy_check.failure_message(res, stub.url()) + ok &= check("a 402 is balance_exhausted and fatal", + res.outcome == "balance_exhausted" and res.fatal + and res.stop_reason == "proxy_balance_exhausted") + ok &= check("the 402 message says traffic/balance exhausted, no " + "credential", "traffic/balance exhausted" in msg + and _SAVERS_SECRET not in msg) + finally: + stub.close() + + stub = _StubProxy(200) + try: + res = proxy_check.precheck(stub.url(), "http://127.0.0.1:9/x", + timeout=5) + head = stub.heads[-1] if stub.heads else "" + ok &= check("a plain-http target is one HEAD for its robots.txt", + res.outcome == "ok" and not res.fatal + and head.startswith("HEAD http://127.0.0.1:9/robots.txt ")) + finally: + stub.close() + + dead = "http://u:" + _SAVERS_SECRET + "@127.0.0.1:" + str(_closed_port()) + res = proxy_check.precheck(dead, _SAVERS_URL, timeout=5) + ok &= check("a proxy that does not answer is a transport error, NOT " + "fatal (existing retry behaviour applies)", + res.outcome == "transport_error" and not res.fatal) + with _captured_output() as buf: + gated = proxy_check.gate(ProxyPool([dead]), _SAVERS_URL, timeout=5) + ok &= check("gate() lets a transport error through, credential unprinted", + gated is None and _SAVERS_SECRET not in buf.getvalue()) + ok &= check("no pool means no pre-check", + proxy_check.gate(None, _SAVERS_URL) is None) + ok &= check("a socks5 exit is skipped, not refused", + proxy_check.precheck("socks5://127.0.0.1:1", _SAVERS_URL) + .outcome == "skipped") + + # -- the attempt sidecar ------------------------------------------------ + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "out") + for name, body in (("out.json", "[1]"), ("out.meta.json", "{}")): + with open(os.path.join(tmp, name), "w") as f: + f.write(body) + with contextlib.redirect_stdout(io.StringIO()): + rc = output_writer.finish_without_fetch( + out, stop_reason="proxy_auth_failed", pages_requested=3, + start_url=_SAVERS_URL) + with open(out + ".latest_attempt.meta.json") as f: + meta = json.load(f) + untouched = (open(out + ".json").read() == "[1]" + and open(out + ".meta.json").read() == "{}") + ok &= check(f"a refused proxy exits 5 ({rc})", rc == 5) + ok &= check("...and records status failed / proxy_auth_failed in the " + "attempt sidecar", meta.get("status") == "failed" + and meta.get("stop_reason") == "proxy_auth_failed" + and meta.get("pages_completed") == 0) + ok &= check("...leaving the previous output and its .meta.json alone", + untouched) + + # -- when media is blocked ---------------------------------------------- + pool = ProxyPool(["http://127.0.0.1:1"]) + ns = argparse.Namespace + ok &= check("blocked through a local proxy by default", + proxy_check.block_media(ns(load_media=False, block_media=False), + pool, remote=False) + is proxy_check.BLOCK_MEDIA_BY_DEFAULT) + ok &= check("not blocked without a proxy", + not proxy_check.block_media( + ns(load_media=False, block_media=False), None, False)) + ok &= check("not blocked over --cdp-endpoint, even when forced", + not proxy_check.block_media( + ns(load_media=False, block_media=True), pool, True)) + ok &= check("--load-media turns it off", + not proxy_check.block_media( + ns(load_media=True, block_media=False), pool, False)) + ok &= check("an env value of 0 does not count as --load-media", + proxy_check.block_media(ns(load_media="0", block_media=False), + pool, False) + is proxy_check.BLOCK_MEDIA_BY_DEFAULT) + ok &= check("the hidden --block-media forces it without a proxy", + proxy_check.block_media(ns(load_media=False, block_media=True), + None, False)) + parser = argparse.ArgumentParser() + proxy_check.add_media_flags(parser) + try: + with contextlib.redirect_stderr(io.StringIO()): + parser.parse_args(["--load-media", "--block-media"]) + exclusive = False + except SystemExit: + exclusive = True + ok &= check("--load-media and --block-media are mutually exclusive", + exclusive) + ok &= check("--block-media is hidden from --help", + "--block-media" not in parser.format_help() + and "--load-media" in parser.format_help()) + ok &= check(f"{proxy_check.LOAD_MEDIA_ENV} maps onto --load-media", + env_config.ENV_KEYS.get(proxy_check.LOAD_MEDIA_ENV) + == "load_media") + + # -- the route handler, on a stub page ---------------------------------- + ctx = mock.MagicMock() + with _captured_output(): + proxy_check.install_playwright(ctx) + pattern, handler = ctx.route.call_args[0] + verdicts = _media_verdicts(handler) + ok &= check(f"Playwright: route on every URL ({pattern})", pattern == "**/*") + ok &= check("Playwright: image/media/font aborted, everything else " + f"continues {verdicts}", + all(verdicts[t] == "abort" for t in ("image", "media", "font")) + and all(v == "continue" for t, v in verdicts.items() + if t not in ("image", "media", "font"))) + + cdp = mock.MagicMock() + cdp.send = mock.AsyncMock() + page = mock.MagicMock() + page.target.createCDPSession = mock.AsyncMock(return_value=cdp) + + async def _drive_pyppeteer(): + await proxy_check.install_pyppeteer(page) + on_paused = cdp.on.call_args[0][1] + on_paused({"requestId": "r1", "resourceType": "Image"}) + await asyncio.sleep(0) + + with _captured_output(): + asyncio.run(_drive_pyppeteer()) + calls = [c[0] for c in cdp.send.await_args_list] + enable = next((c[1] for c in calls if c[0] == "Fetch.enable"), {}) + kinds = sorted(pt.get("resourceType") for pt in enable.get("patterns", [])) + ok &= check("pyppeteer: Fetch pauses ONLY Image/Media/Font, on its own " + f"CDP session {kinds}", cdp.on.call_args[0][0] == + "Fetch.requestPaused" and kinds == ["Font", "Image", "Media"]) + ok &= check("pyppeteer: a paused request is failed, not continued", + ("Fetch.failRequest", {"requestId": "r1", + "errorReason": "BlockedByClient"}) + in calls) + ok &= check("pyppeteer: never the removed Network.setRequestInterception", + "setRequestInterception" not in inspect.getsource( + proxy_check.install_pyppeteer).split('"""')[-1]) + + driver = mock.MagicMock() + with _captured_output(): + installed = proxy_check.install_selenium(driver) + sent = {c[0][0]: c[0][1] for c in driver.execute_cdp_cmd.call_args_list} + urls = sent.get("Network.setBlockedURLs", {}).get("urls", []) + ok &= check("Selenium: Network.setBlockedURLs with image/video/font " + "patterns", installed and "*.png" in urls and "*.woff2" in urls + and "*.mp4?*" in urls) + ok &= check("Selenium: no pattern touches a document, script, JSON or " + "stylesheet", not [u for u in urls if re.search( + r"\.(html?|js|json|css)\b", u)]) + broken = mock.MagicMock() + broken.execute_cdp_cmd.side_effect = RuntimeError("no CDP") + with _captured_output(): + ok &= check("Selenium: a driver that refuses CDP is not fatal", + proxy_check.install_selenium(broken) is False) + + # -- the requests transport, driven: no page request on a 407 ---------- + import http_scraper + stub = _StubProxy(407) + fetched = [] + try: + with tempfile.TemporaryDirectory() as tmp: + args = _engine_args(http_scraper, tmp, stub.url()) + with mock.patch.object( + http_scraper, "_fetch_one_page", + side_effect=lambda *a, **k: fetched.append(1)), \ + _captured_output() as buf: + try: + rc = http_scraper.scrape(args) + except Exception as e: # noqa: BLE001 — fail BY NAME + rc = f"raised {type(e).__name__}" + side = os.path.join(tmp, "out.latest_attempt.meta.json") + meta = json.load(open(side)) if os.path.exists(side) else {} + finally: + stub.close() + ok &= check("http_scraper: a 407 exits 5 before any page request " + f"(rc={rc}, fetched={len(fetched)})", rc == 5 and not fetched) + ok &= check("http_scraper: ...stop_reason proxy_auth_failed in the " + "attempt sidecar, credential unprinted", + meta.get("stop_reason") == "proxy_auth_failed" + and _SAVERS_SECRET not in buf.getvalue()) + + # -- each engine, driven: no browser is started on a 407 ---------------- + for name in ENGINE_NAMES: + try: + engine = __import__(name) + except ImportError as e: + skips.append(f"{name} proxy savers driving — {e}") + print(" SKIP %s proxy savers — %s" % (name, e)) + continue + stub = _StubProxy(407) + launched = [] + if name == "playwright_scraper": + target = "sync_playwright" + else: + target = "_Session" + try: + with tempfile.TemporaryDirectory() as tmp: + args = _engine_args(engine, tmp, stub.url()) + with mock.patch.object( + engine, target, + side_effect=lambda *a, **k: launched.append(1)), \ + _captured_output() as buf: + try: + rc = engine.scrape(args) + except Exception as e: # noqa: BLE001 — fail BY NAME + rc = f"raised {type(e).__name__}" + side = os.path.join(tmp, "out.latest_attempt.meta.json") + meta = json.load(open(side)) if os.path.exists(side) else {} + wrote_rows = any(os.path.exists(os.path.join(tmp, "out" + s)) + for s in (".json", ".csv", ".meta.json")) + finally: + stub.close() + ok &= check(f"{name}: a 407 exits 5 without starting a browser " + f"(rc={rc}, launched={len(launched)})", + rc == 5 and not launched) + ok &= check(f"{name}: ...with stop_reason proxy_auth_failed in the " + "attempt sidecar and no output written", + meta.get("stop_reason") == "proxy_auth_failed" + and meta.get("status") == "failed" and not wrote_rows) + ok &= check(f"{name}: ...and the credential never reaches the log", + _SAVERS_SECRET not in buf.getvalue()) + + # -- each engine installs blocking only through a local proxy ----------- + for name in ENGINE_NAMES: + try: + engine = __import__(name) + except ImportError: + continue # already recorded as a skip above + for label, proxy, extra, want in ( + ("through a proxy", "http://127.0.0.1:1", (), None), + ("with --load-media", "http://127.0.0.1:1", + ("--load-media",), False), + ("without a proxy", None, (), False), + ("forced, no proxy", None, ("--block-media",), True)): + if want is None: + want = proxy_check.BLOCK_MEDIA_BY_DEFAULT + with tempfile.TemporaryDirectory() as tmp: + args = _engine_args(engine, tmp, proxy, *extra) + pool = ProxyPool([proxy]) if proxy else None + got = _engine_installs_blocking(engine, args, pool) + ok &= check(f"{name}: media blocking {label} = {want} (got {got})", + got is want) + return ok + + +def _engine_installs_blocking(engine, args, pool): + """Open the engine's local session on fakes; True if blocking went in.""" + from unittest import mock + name = engine.__name__ + with _captured_output(): + if name == "playwright_scraper": + pw = mock.MagicMock() + browser = pw.chromium.launch.return_value + browser.version = "120.0.0.0" + ctx = browser.new_context.return_value + engine._launch_local(pw, args, pool) + return ctx.route.called + if name == "selenium_scraper": + with mock.patch.object(engine.webdriver, "Chrome") as chrome: + driver = chrome.return_value + driver.capabilities = {"browserVersion": "120.0.0.0"} + engine._Session(args, pool).open() + return any(c[0][0] == "Network.setBlockedURLs" + for c in driver.execute_cdp_cmd.call_args_list) + page = mock.MagicMock() + for attr in ("setUserAgent", "setViewport", "authenticate", + "setRequestInterception", "evaluateOnNewDocument", + "setExtraHTTPHeaders"): + setattr(page, attr, mock.AsyncMock()) + cdp = mock.MagicMock() + cdp.send = mock.AsyncMock() + page.target.createCDPSession = mock.AsyncMock(return_value=cdp) + browser = mock.MagicMock() + browser.newPage = mock.AsyncMock(return_value=page) + browser.version = mock.AsyncMock(return_value="HeadlessChrome/120.0") + + async def fake_launch(*a, **k): + return browser + bridge = engine._AsyncBridge() + try: + with mock.patch.object(engine, "launch", fake_launch): + engine._Session(bridge, args, pool).open() + finally: + closer = getattr(bridge, "close", None) + if closer: + closer() + return any(c[0][0] == "Fetch.enable" for c in cdp.send.await_args_list) + + +test_proxy_traffic_savers(_SaversSkips(), _savers_check, _savers_group) + + def main() -> int: total = len(PASSES) + len(FAILURES) print()