diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 9881cb5..4fe7f01 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -206,9 +206,29 @@ jobs: if len(rows) < 15: fail.append("only %d rows; a measured tag feed gave 42-53" % len(rows)) - if not any(r.get("claps") is not None for r in rows): - fail.append("no row carries a clap count, and the modern " - "payload carries one on every story it describes") + # Claps and responses ride on the Apollo rows ONLY: the ~20 rows + # that come from JSON-LD never carry them, so the share over all + # rows measures the mix of sources and says nothing about the + # read. Judge each source on what it should hold. Measured over + # four live runs of /tag/python on 2026-10-05: Apollo rows 34-35 + # per run with claps and responses on 100% of them in three runs + # and 30 of 35 (86%) in the fourth, JSON-LD rows 20 with neither. + # Floors, with room for that fourth run. + apollo = [r for r in rows if r.get("data_source") == "apollo"] + if len(apollo) < 15: + fail.append("only %d rows came from the Apollo payload; " + "measured 34-35, and the clap check below would " + "pass on an empty set" % len(apollo)) + for field in ("claps", "responses"): + got = sum(1 for r in apollo if r.get(field) is not None) + if apollo and got / len(apollo) < 0.8: + fail.append("%s on only %d of %d Apollo rows; measured " + "86-100%%" % (field, got, len(apollo))) + if any(r.get("claps") is not None or r.get("responses") is not None + for r in rows if r.get("data_source") == "jsonld"): + fail.append("a JSON-LD row carried claps or responses — the " + "payload changed, and what the README says about " + "these columns by mode needs rewriting") # The documented asymmetry, asserted rather than assumed: a # tag-feed row has NO reading time because the payload does not # contain one. If this ever starts being populated, the README is diff --git a/CHANGELOG.md b/CHANGELOG.md index 32fc6de..8502031 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,25 @@ empty output file. ### Fixed +- **A refused `--cdp-endpoint` explained the wrong cause.** Every connect + failure was followed by the same paragraph about `profile_locked`, so an + expired login (HTTP 401, `deny_no_user`) sent the reader looking for a + run holding the pid. The text is now chosen by what the failure says: + expired credentials, a held profile, an unreachable host, or — for a bare + timeout — an honest "no status, try fresh credentials first". pyppeteer + also crashed with a traceback and exit 1 on the same failure; it now + exits 5 like Playwright, with the endpoint's password masked. Pinned by + offline checks that drive each engine's real connect path with the + driver stubbed; verified by planting four faults and requiring the named + check to go red. +- **The tag-feed canary asserted that ONE row had claps.** Claps and + responses ride on the Apollo rows only; the 20 rows that come from + JSON-LD never carry them, so a share over all rows tracks the source mix, + not the read. The canary now judges the Apollo rows (at least 15, claps + and responses on at least 80% of them) and fails if a JSON-LD row ever + carries either. Measured 2026-10-05 over four live runs of `/tag/python`: + 34-35 Apollo rows, 100% in three runs and 30 of 35 in the fourth, 0 of 20 + JSON-LD rows in every run. - **The Scraper API path sent `waitFor` in a form the live API rejects, and read the wrong field as the target's status.** Measured 2026-09-23 against `scraper.2captcha.com/tasks/sync`: `waitFor` sent as a diff --git a/page_flow.py b/page_flow.py index 1fa618c..c70a349 100644 --- a/page_flow.py +++ b/page_flow.py @@ -51,6 +51,7 @@ from __future__ import annotations import logging +import re from typing import Callable, Dict, List, Optional from product_parser import (SELECTORS, PAGE_CAP, CONCURRENCY_REASON, @@ -301,6 +302,46 @@ def is_unpainted(state: str, html: Optional[str]) -> bool: RETRY_NEEDS_FRESH_CONTEXT = True +def cdp_connect_advice(error_text: str) -> str: + """Say what a refused --cdp-endpoint connection most likely MEANS. + + The same sentence used to follow every connect failure, and it explained + `profile_locked`. A 401 is not that: the credentials are expired or + wrong (a profile's last about a day), and a reader told to look for a + run holding the pid goes looking in the wrong place. Classified by what + the failure SAYS, not by which driver raised it. Pure text in, text out, + and no secret ever goes into the answer. + + pyppeteer never surfaces the HTTP status of a rejected upgrade — only a + timeout ends its connect — so a bare timeout is named as ambiguous + rather than guessed at. + """ + text = (error_text or "").lower() + if re.search(r"\b401\b", text) or "deny_no_user" in text or "unauthorized" in text: + return ("The endpoint refused the credentials (HTTP 401). A Scraping " + "Browser profile's login expires after about a day, so fetch " + "fresh ones from the 2Captcha dashboard and check the " + "`ws://{login}-zone-scraping_browser-country-xx-pid-{id}:" + "{password}@cb.2captcha.com:9222` shape. This is not " + "`profile_locked`, and another pid will not help.") + if "profile_locked" in text or re.search(r"\b500\b", text): + return ("A Scraping Browser profile allows ONE live connection, so " + "`profile_locked` means something holds this `pid` — it stays " + "locked for a couple of seconds after a clean disconnect. " + "Two profiles were also observed entering that state and not " + "leaving it, one for over forty minutes; if waiting does not " + "clear it, use a different pid or reset the profile from the " + "2Captcha dashboard.") + if any(k in text for k in ("econnrefused", "getaddrinfo", "name or service", + "not known", "unreachable", "econnreset")): + return ("The endpoint could not be reached at all — a network or DNS " + "problem, not a rejected login. Check the host and port.") + return ("The connection did not complete and the failure carries no " + "status. Expired credentials (HTTP 401) and a locked profile " + "(`profile_locked`) both look like this from some drivers; try " + "fresh credentials first, then a different pid.") + + def block_advice(html: Optional[str], headless: bool, has_pool: bool) -> str: """What a reader should actually DO about this block. diff --git a/playwright_scraper.py b/playwright_scraper.py index 7aa67eb..6f1dd3e 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -492,16 +492,7 @@ def _connect_remote(pw, args): f"could not connect to --cdp-endpoint " f"{_mask_credentials(args.cdp_endpoint)}: " f"{_mask_credentials(str(e))}\n" - f"A Scraping Browser profile allows ONE live connection at a " - f"time, so `profile_locked` means something holds this `pid`.\n" - f"Worth knowing before you go looking for it on your side: two " - f"profiles were observed here entering that state and NOT leaving " - f"it — one for over forty minutes, one still locked after four " - f"minutes of no requests at all, having locked on its very first " - f"connection attempt. Waiting did not clear either. If that is " - f"what you are seeing, it is not another run of this tool holding " - f"it, and nothing on this side will free it: use a different pid, " - f"or reset the profile from the 2Captcha dashboard." + f"{page_flow.cdp_connect_advice(_mask_credentials(str(e)))}" ) from None context = browser.contexts[0] if browser.contexts else browser.new_context() page = context.new_page() diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index db99256..c1607b4 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -65,7 +65,7 @@ detect_bot_challenge, listing_kind, normalize_url, page_url, paginates_by_url, redirected_away, served_by_medium, source_of, unsupported_reason) -from output_writer import dedupe_by_key, finish_run +from output_writer import dedupe_by_key, finish_run, EXIT_API_ERROR import page_flow from proxy_pool import (from_args as proxy_pool_from_args, split_credentials, mask, ROTATE_MODES, ProxyError) @@ -90,6 +90,14 @@ CONNECT_TIMEOUT = 150 +class CdpConnectError(RuntimeError): + """A remote browser that would not accept the connection. + + A remote API failure (exit 5), not a crash in this code (exit 1): the + Playwright engine already reports it that way. + """ + + class _AsyncBridge: """Runs pyppeteer's coroutines on a private event loop, synchronously. @@ -233,11 +241,20 @@ def open(self): # form and authenticates on the WebSocket upgrade, so an # authenticated Scraping Browser endpoint works here — unlike # Selenium's debuggerAddress, which has nowhere to put a password. - self.browser = self.bridge.run( - connect(browserWSEndpoint=self.args.cdp_endpoint, - ignoreHTTPSErrors=True), - timeout=getattr(self.args, "cdp_connect_timeout", - CONNECT_TIMEOUT)) + try: + self.browser = self.bridge.run( + connect(browserWSEndpoint=self.args.cdp_endpoint, + ignoreHTTPSErrors=True), + timeout=getattr(self.args, "cdp_connect_timeout", + CONNECT_TIMEOUT)) + except Exception as e: # noqa: BLE001 — any failure to connect + # The library's text can carry the endpoint, password and + # all, so mask before it is put anywhere. + text = _mask_credentials(str(e)) + raise CdpConnectError( + f"could not connect to --cdp-endpoint " + f"{_mask_credentials(self.args.cdp_endpoint)}: {text}\n" + f"{page_flow.cdp_connect_advice(text)}") from None self.page = self.bridge.run(self.browser.newPage()) _watch_graphql(self) return self @@ -1110,3 +1127,6 @@ def parse_args(): except ProxyError as e: logger.error("%s", e) sys.exit(2) + except CdpConnectError as e: + logger.error("%s", e) + sys.exit(EXIT_API_ERROR) diff --git a/smoke_test.py b/smoke_test.py index d2a9b15..316eec2 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1642,6 +1642,106 @@ def test_credentials_never_reach_a_log(): return ok +def test_cdp_connect_failure_names_its_cause(): + group("A refused --cdp-endpoint says WHY, and a 401 is not profile_locked") + ok = True + secret = "hunter2" + # Concatenated, so no line here is a complete scheme://user:pass@host + # literal (see test_credentials_never_reach_a_log). + endpoint = "ws://user:" + secret + "@cb.2captcha.com:9222" + advice = page_flow.cdp_connect_advice + unauthorized = "WebSocket error: " + endpoint + " 401 Unauthorized (deny_no_user)" + ok &= check("a 401 is named as expired credentials", + "expires after" in advice(unauthorized)) + # Each signal alone, so removing one of the three cannot hide behind the + # other two in the line above. + ok &= check("a bare status 401 is enough", + "expires after" in advice("server returned HTTP 401")) + ok &= check("deny_no_user alone is enough", + "expires after" in advice("deny_no_user")) + ok &= check("a 401 is NOT explained as a locked profile", + "means something holds" not in advice(unauthorized)) + ok &= check("a 500 / profile_locked is explained as a held pid", + "means something holds" in advice("HTTP 500 profile_locked")) + ok &= check("an unreachable host is a network problem", + "network" in advice("getaddrinfo ENOTFOUND cb.example")) + ok &= check("a bare timeout is called ambiguous, not guessed", + "no status" in advice("Timeout 30000ms exceeded")) + # A port that merely contains the numerals must not read as a status. + ok &= check("numerals inside a port are not an HTTP status", + "expires" not in advice("connect to host:14019 failed")) + ok &= check("the advice never carries a secret", + all(secret not in advice(t) for t in (unauthorized, endpoint))) + + # Through each engine that CAN connect remotely: the message that + # reaches the log names the cause, and carries no password. + try: + import playwright_scraper as pw_engine + except ImportError: + pw_engine = None + if pw_engine is not None: + class _Chromium: + def connect_over_cdp(self, *a, **k): + raise pw_engine.PWError(unauthorized) + + class _Pw: + chromium = _Chromium() + + args = type("A", (), {"cdp_endpoint": endpoint, + "cdp_connect_timeout": 1})() + try: + pw_engine._connect_remote(_Pw(), args) + message = "" + except pw_engine.PWError as e: + message = str(e) + ok &= check("playwright: a 401 reaches the log as a 401", + "expires after" in message + and "means something holds" not in message) + ok &= check("playwright: no password in the connect error", + bool(message) and secret not in message) + + try: + import puppeteer_scraper as pp_engine + except ImportError: + pp_engine = None + if pp_engine is not None: + class _Bridge: + def run(self, coro, timeout=None): + # A library error repeats the endpoint, password and all. + raise TimeoutError("pyppeteer call did not return within 10s: " + + " ".join([endpoint] * 3)) + + def _connect(**kwargs): + return None + + real = pp_engine.connect + pp_engine.connect = _connect + try: + args = type("A", (), {"cdp_endpoint": endpoint, + "cdp_connect_timeout": 1})() + try: + pp_engine._Session(_Bridge(), args, None).open() + message = "" + except pp_engine.CdpConnectError as e: + message = str(e) + finally: + pp_engine.connect = real + ok &= check("pyppeteer: a failed connect is a CdpConnectError", + bool(message)) + ok &= check("pyppeteer: no password in the connect error", + bool(message) and secret not in message) + ok &= check("pyppeteer: a timeout with no status is called ambiguous", + "no status" in message) + ok &= check("pyppeteer: the host and port survive the masking", + "cb.2captcha.com:9222" in message) + source = _engine_source("puppeteer_scraper") or "" + ok &= check("pyppeteer: CdpConnectError exits 5, not a traceback", + "except CdpConnectError" in source + and "EXIT_API_ERROR" in + source.split("except CdpConnectError")[1][:120]) + return ok + + def test_driver_primitives_tolerate_a_navigation(): group("Every driver primitive survives the page moving under it") ok = True @@ -2635,6 +2735,7 @@ def main() -> int: ok &= test_fingerprint_cache_is_key_aware() ok &= test_proxy_pool() ok &= test_credentials_never_reach_a_log() + ok &= test_cdp_connect_failure_names_its_cause() ok &= test_driver_primitives_tolerate_a_navigation() ok &= test_concurrency_machinery(skips) ok &= test_engine_parity(skips)