From b8612b370862737bb0eb3d6a25de01dc4d6ea078 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:49:01 +0000 Subject: [PATCH] Name the real cause of a refused --cdp-endpoint; judge claps per source in the tag canary A refused CDP connection always explained profile_locked, so an expired login (HTTP 401) sent the reader looking for a run holding the pid. The text now follows what the failure says. pyppeteer also exited 1 with a traceback on the same failure; it now exits 5 like Playwright, with the password masked. The tag-feed canary asserted that one row had claps. Claps and responses ride on Apollo rows only (34-35 per run, 100% in 3 of 4 live runs, 30 of 35 in the fourth; 0 of 20 JSON-LD rows in every run), so it now judges the Apollo rows with floors that admit the measured worst day. Both verified by planting faults and requiring the named check to go red. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/canary.yml | 26 +++++++-- CHANGELOG.md | 19 +++++++ page_flow.py | 41 ++++++++++++++ playwright_scraper.py | 11 +--- puppeteer_scraper.py | 32 ++++++++--- smoke_test.py | 101 +++++++++++++++++++++++++++++++++++ 6 files changed, 211 insertions(+), 19 deletions(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 9881cb5..4fe7f01 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -206,9 +206,29 @@ jobs: if len(rows) < 15: fail.append("only %d rows; a measured tag feed gave 42-53" % len(rows)) - if not any(r.get("claps") is not None for r in rows): - fail.append("no row carries a clap count, and the modern " - "payload carries one on every story it describes") + # Claps and responses ride on the Apollo rows ONLY: the ~20 rows + # that come from JSON-LD never carry them, so the share over all + # rows measures the mix of sources and says nothing about the + # read. Judge each source on what it should hold. Measured over + # four live runs of /tag/python on 2026-10-05: Apollo rows 34-35 + # per run with claps and responses on 100% of them in three runs + # and 30 of 35 (86%) in the fourth, JSON-LD rows 20 with neither. + # Floors, with room for that fourth run. + apollo = [r for r in rows if r.get("data_source") == "apollo"] + if len(apollo) < 15: + fail.append("only %d rows came from the Apollo payload; " + "measured 34-35, and the clap check below would " + "pass on an empty set" % len(apollo)) + for field in ("claps", "responses"): + got = sum(1 for r in apollo if r.get(field) is not None) + if apollo and got / len(apollo) < 0.8: + fail.append("%s on only %d of %d Apollo rows; measured " + "86-100%%" % (field, got, len(apollo))) + if any(r.get("claps") is not None or r.get("responses") is not None + for r in rows if r.get("data_source") == "jsonld"): + fail.append("a JSON-LD row carried claps or responses — the " + "payload changed, and what the README says about " + "these columns by mode needs rewriting") # The documented asymmetry, asserted rather than assumed: a # tag-feed row has NO reading time because the payload does not # contain one. If this ever starts being populated, the README is diff --git a/CHANGELOG.md b/CHANGELOG.md index 32fc6de..8502031 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,25 @@ empty output file. ### Fixed +- **A refused `--cdp-endpoint` explained the wrong cause.** Every connect + failure was followed by the same paragraph about `profile_locked`, so an + expired login (HTTP 401, `deny_no_user`) sent the reader looking for a + run holding the pid. The text is now chosen by what the failure says: + expired credentials, a held profile, an unreachable host, or — for a bare + timeout — an honest "no status, try fresh credentials first". pyppeteer + also crashed with a traceback and exit 1 on the same failure; it now + exits 5 like Playwright, with the endpoint's password masked. Pinned by + offline checks that drive each engine's real connect path with the + driver stubbed; verified by planting four faults and requiring the named + check to go red. +- **The tag-feed canary asserted that ONE row had claps.** Claps and + responses ride on the Apollo rows only; the 20 rows that come from + JSON-LD never carry them, so a share over all rows tracks the source mix, + not the read. The canary now judges the Apollo rows (at least 15, claps + and responses on at least 80% of them) and fails if a JSON-LD row ever + carries either. Measured 2026-10-05 over four live runs of `/tag/python`: + 34-35 Apollo rows, 100% in three runs and 30 of 35 in the fourth, 0 of 20 + JSON-LD rows in every run. - **The Scraper API path sent `waitFor` in a form the live API rejects, and read the wrong field as the target's status.** Measured 2026-09-23 against `scraper.2captcha.com/tasks/sync`: `waitFor` sent as a diff --git a/page_flow.py b/page_flow.py index 1fa618c..c70a349 100644 --- a/page_flow.py +++ b/page_flow.py @@ -51,6 +51,7 @@ from __future__ import annotations import logging +import re from typing import Callable, Dict, List, Optional from product_parser import (SELECTORS, PAGE_CAP, CONCURRENCY_REASON, @@ -301,6 +302,46 @@ def is_unpainted(state: str, html: Optional[str]) -> bool: RETRY_NEEDS_FRESH_CONTEXT = True +def cdp_connect_advice(error_text: str) -> str: + """Say what a refused --cdp-endpoint connection most likely MEANS. + + The same sentence used to follow every connect failure, and it explained + `profile_locked`. A 401 is not that: the credentials are expired or + wrong (a profile's last about a day), and a reader told to look for a + run holding the pid goes looking in the wrong place. Classified by what + the failure SAYS, not by which driver raised it. Pure text in, text out, + and no secret ever goes into the answer. + + pyppeteer never surfaces the HTTP status of a rejected upgrade — only a + timeout ends its connect — so a bare timeout is named as ambiguous + rather than guessed at. + """ + text = (error_text or "").lower() + if re.search(r"\b401\b", text) or "deny_no_user" in text or "unauthorized" in text: + return ("The endpoint refused the credentials (HTTP 401). A Scraping " + "Browser profile's login expires after about a day, so fetch " + "fresh ones from the 2Captcha dashboard and check the " + "`ws://{login}-zone-scraping_browser-country-xx-pid-{id}:" + "{password}@cb.2captcha.com:9222` shape. This is not " + "`profile_locked`, and another pid will not help.") + if "profile_locked" in text or re.search(r"\b500\b", text): + return ("A Scraping Browser profile allows ONE live connection, so " + "`profile_locked` means something holds this `pid` — it stays " + "locked for a couple of seconds after a clean disconnect. " + "Two profiles were also observed entering that state and not " + "leaving it, one for over forty minutes; if waiting does not " + "clear it, use a different pid or reset the profile from the " + "2Captcha dashboard.") + if any(k in text for k in ("econnrefused", "getaddrinfo", "name or service", + "not known", "unreachable", "econnreset")): + return ("The endpoint could not be reached at all — a network or DNS " + "problem, not a rejected login. Check the host and port.") + return ("The connection did not complete and the failure carries no " + "status. Expired credentials (HTTP 401) and a locked profile " + "(`profile_locked`) both look like this from some drivers; try " + "fresh credentials first, then a different pid.") + + def block_advice(html: Optional[str], headless: bool, has_pool: bool) -> str: """What a reader should actually DO about this block. diff --git a/playwright_scraper.py b/playwright_scraper.py index 7aa67eb..6f1dd3e 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -492,16 +492,7 @@ def _connect_remote(pw, args): f"could not connect to --cdp-endpoint " f"{_mask_credentials(args.cdp_endpoint)}: " f"{_mask_credentials(str(e))}\n" - f"A Scraping Browser profile allows ONE live connection at a " - f"time, so `profile_locked` means something holds this `pid`.\n" - f"Worth knowing before you go looking for it on your side: two " - f"profiles were observed here entering that state and NOT leaving " - f"it — one for over forty minutes, one still locked after four " - f"minutes of no requests at all, having locked on its very first " - f"connection attempt. Waiting did not clear either. If that is " - f"what you are seeing, it is not another run of this tool holding " - f"it, and nothing on this side will free it: use a different pid, " - f"or reset the profile from the 2Captcha dashboard." + f"{page_flow.cdp_connect_advice(_mask_credentials(str(e)))}" ) from None context = browser.contexts[0] if browser.contexts else browser.new_context() page = context.new_page() diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index db99256..c1607b4 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -65,7 +65,7 @@ detect_bot_challenge, listing_kind, normalize_url, page_url, paginates_by_url, redirected_away, served_by_medium, source_of, unsupported_reason) -from output_writer import dedupe_by_key, finish_run +from output_writer import dedupe_by_key, finish_run, EXIT_API_ERROR import page_flow from proxy_pool import (from_args as proxy_pool_from_args, split_credentials, mask, ROTATE_MODES, ProxyError) @@ -90,6 +90,14 @@ CONNECT_TIMEOUT = 150 +class CdpConnectError(RuntimeError): + """A remote browser that would not accept the connection. + + A remote API failure (exit 5), not a crash in this code (exit 1): the + Playwright engine already reports it that way. + """ + + class _AsyncBridge: """Runs pyppeteer's coroutines on a private event loop, synchronously. @@ -233,11 +241,20 @@ def open(self): # form and authenticates on the WebSocket upgrade, so an # authenticated Scraping Browser endpoint works here — unlike # Selenium's debuggerAddress, which has nowhere to put a password. - self.browser = self.bridge.run( - connect(browserWSEndpoint=self.args.cdp_endpoint, - ignoreHTTPSErrors=True), - timeout=getattr(self.args, "cdp_connect_timeout", - CONNECT_TIMEOUT)) + try: + self.browser = self.bridge.run( + connect(browserWSEndpoint=self.args.cdp_endpoint, + ignoreHTTPSErrors=True), + timeout=getattr(self.args, "cdp_connect_timeout", + CONNECT_TIMEOUT)) + except Exception as e: # noqa: BLE001 — any failure to connect + # The library's text can carry the endpoint, password and + # all, so mask before it is put anywhere. + text = _mask_credentials(str(e)) + raise CdpConnectError( + f"could not connect to --cdp-endpoint " + f"{_mask_credentials(self.args.cdp_endpoint)}: {text}\n" + f"{page_flow.cdp_connect_advice(text)}") from None self.page = self.bridge.run(self.browser.newPage()) _watch_graphql(self) return self @@ -1110,3 +1127,6 @@ def parse_args(): except ProxyError as e: logger.error("%s", e) sys.exit(2) + except CdpConnectError as e: + logger.error("%s", e) + sys.exit(EXIT_API_ERROR) diff --git a/smoke_test.py b/smoke_test.py index d2a9b15..316eec2 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1642,6 +1642,106 @@ def test_credentials_never_reach_a_log(): return ok +def test_cdp_connect_failure_names_its_cause(): + group("A refused --cdp-endpoint says WHY, and a 401 is not profile_locked") + ok = True + secret = "hunter2" + # Concatenated, so no line here is a complete scheme://user:pass@host + # literal (see test_credentials_never_reach_a_log). + endpoint = "ws://user:" + secret + "@cb.2captcha.com:9222" + advice = page_flow.cdp_connect_advice + unauthorized = "WebSocket error: " + endpoint + " 401 Unauthorized (deny_no_user)" + ok &= check("a 401 is named as expired credentials", + "expires after" in advice(unauthorized)) + # Each signal alone, so removing one of the three cannot hide behind the + # other two in the line above. + ok &= check("a bare status 401 is enough", + "expires after" in advice("server returned HTTP 401")) + ok &= check("deny_no_user alone is enough", + "expires after" in advice("deny_no_user")) + ok &= check("a 401 is NOT explained as a locked profile", + "means something holds" not in advice(unauthorized)) + ok &= check("a 500 / profile_locked is explained as a held pid", + "means something holds" in advice("HTTP 500 profile_locked")) + ok &= check("an unreachable host is a network problem", + "network" in advice("getaddrinfo ENOTFOUND cb.example")) + ok &= check("a bare timeout is called ambiguous, not guessed", + "no status" in advice("Timeout 30000ms exceeded")) + # A port that merely contains the numerals must not read as a status. + ok &= check("numerals inside a port are not an HTTP status", + "expires" not in advice("connect to host:14019 failed")) + ok &= check("the advice never carries a secret", + all(secret not in advice(t) for t in (unauthorized, endpoint))) + + # Through each engine that CAN connect remotely: the message that + # reaches the log names the cause, and carries no password. + try: + import playwright_scraper as pw_engine + except ImportError: + pw_engine = None + if pw_engine is not None: + class _Chromium: + def connect_over_cdp(self, *a, **k): + raise pw_engine.PWError(unauthorized) + + class _Pw: + chromium = _Chromium() + + args = type("A", (), {"cdp_endpoint": endpoint, + "cdp_connect_timeout": 1})() + try: + pw_engine._connect_remote(_Pw(), args) + message = "" + except pw_engine.PWError as e: + message = str(e) + ok &= check("playwright: a 401 reaches the log as a 401", + "expires after" in message + and "means something holds" not in message) + ok &= check("playwright: no password in the connect error", + bool(message) and secret not in message) + + try: + import puppeteer_scraper as pp_engine + except ImportError: + pp_engine = None + if pp_engine is not None: + class _Bridge: + def run(self, coro, timeout=None): + # A library error repeats the endpoint, password and all. + raise TimeoutError("pyppeteer call did not return within 10s: " + + " ".join([endpoint] * 3)) + + def _connect(**kwargs): + return None + + real = pp_engine.connect + pp_engine.connect = _connect + try: + args = type("A", (), {"cdp_endpoint": endpoint, + "cdp_connect_timeout": 1})() + try: + pp_engine._Session(_Bridge(), args, None).open() + message = "" + except pp_engine.CdpConnectError as e: + message = str(e) + finally: + pp_engine.connect = real + ok &= check("pyppeteer: a failed connect is a CdpConnectError", + bool(message)) + ok &= check("pyppeteer: no password in the connect error", + bool(message) and secret not in message) + ok &= check("pyppeteer: a timeout with no status is called ambiguous", + "no status" in message) + ok &= check("pyppeteer: the host and port survive the masking", + "cb.2captcha.com:9222" in message) + source = _engine_source("puppeteer_scraper") or "" + ok &= check("pyppeteer: CdpConnectError exits 5, not a traceback", + "except CdpConnectError" in source + and "EXIT_API_ERROR" in + source.split("except CdpConnectError")[1][:120]) + return ok + + def test_driver_primitives_tolerate_a_navigation(): group("Every driver primitive survives the page moving under it") ok = True @@ -2635,6 +2735,7 @@ def main() -> int: ok &= test_fingerprint_cache_is_key_aware() ok &= test_proxy_pool() ok &= test_credentials_never_reach_a_log() + ok &= test_cdp_connect_failure_names_its_cause() ok &= test_driver_primitives_tolerate_a_navigation() ok &= test_concurrency_machinery(skips) ok &= test_engine_parity(skips)