Name the real cause of a refused --cdp-endpoint; judge claps per source in the tag canary

.github/workflows/canary.yml

@@ -206,9 +206,29 @@ jobs:
           if len(rows) < 15:
               fail.append("only %d rows; a measured tag feed gave 42-53"
                           % len(rows))
-          if not any(r.get("claps") is not None for r in rows):
-              fail.append("no row carries a clap count, and the modern "
-                          "payload carries one on every story it describes")
+          # Claps and responses ride on the Apollo rows ONLY: the ~20 rows
+          # that come from JSON-LD never carry them, so the share over all
+          # rows measures the mix of sources and says nothing about the
+          # read. Judge each source on what it should hold. Measured over
+          # four live runs of /tag/python on 2026-10-05: Apollo rows 34-35
+          # per run with claps and responses on 100% of them in three runs
+          # and 30 of 35 (86%) in the fourth, JSON-LD rows 20 with neither.
+          # Floors, with room for that fourth run.
+          apollo = [r for r in rows if r.get("data_source") == "apollo"]
+          if len(apollo) < 15:
+              fail.append("only %d rows came from the Apollo payload; "
+                          "measured 34-35, and the clap check below would "
+                          "pass on an empty set" % len(apollo))
+          for field in ("claps", "responses"):
+              got = sum(1 for r in apollo if r.get(field) is not None)
+              if apollo and got / len(apollo) < 0.8:
+                  fail.append("%s on only %d of %d Apollo rows; measured "
+                              "86-100%%" % (field, got, len(apollo)))
+          if any(r.get("claps") is not None or r.get("responses") is not None
+                 for r in rows if r.get("data_source") == "jsonld"):
+              fail.append("a JSON-LD row carried claps or responses — the "
+                          "payload changed, and what the README says about "
+                          "these columns by mode needs rewriting")
           # The documented asymmetry, asserted rather than assumed: a
           # tag-feed row has NO reading time because the payload does not
           # contain one. If this ever starts being populated, the README is

CHANGELOG.md

@@ -13,6 +13,25 @@ empty output file.
 
 ### Fixed
 
+- **A refused `--cdp-endpoint` explained the wrong cause.** Every connect
+  failure was followed by the same paragraph about `profile_locked`, so an
+  expired login (HTTP 401, `deny_no_user`) sent the reader looking for a
+  run holding the pid. The text is now chosen by what the failure says:
+  expired credentials, a held profile, an unreachable host, or — for a bare
+  timeout — an honest "no status, try fresh credentials first". pyppeteer
+  also crashed with a traceback and exit 1 on the same failure; it now
+  exits 5 like Playwright, with the endpoint's password masked. Pinned by
+  offline checks that drive each engine's real connect path with the
+  driver stubbed; verified by planting four faults and requiring the named
+  check to go red.
+- **The tag-feed canary asserted that ONE row had claps.** Claps and
+  responses ride on the Apollo rows only; the 20 rows that come from
+  JSON-LD never carry them, so a share over all rows tracks the source mix,
+  not the read. The canary now judges the Apollo rows (at least 15, claps
+  and responses on at least 80% of them) and fails if a JSON-LD row ever
+  carries either. Measured 2026-10-05 over four live runs of `/tag/python`:
+  34-35 Apollo rows, 100% in three runs and 30 of 35 in the fourth, 0 of 20
+  JSON-LD rows in every run.
 - **The Scraper API path sent `waitFor` in a form the live API rejects,
   and read the wrong field as the target's status.** Measured 2026-09-23
   against `scraper.2captcha.com/tasks/sync`: `waitFor` sent as a

page_flow.py

@@ -51,6 +51,7 @@
 from __future__ import annotations
 
 import logging
+import re
 from typing import Callable, Dict, List, Optional
 
 from product_parser import (SELECTORS, PAGE_CAP, CONCURRENCY_REASON,
@@ -301,6 +302,46 @@ def is_unpainted(state: str, html: Optional[str]) -> bool:
 RETRY_NEEDS_FRESH_CONTEXT = True
 
 
+def cdp_connect_advice(error_text: str) -> str:
+    """Say what a refused --cdp-endpoint connection most likely MEANS.
+
+    The same sentence used to follow every connect failure, and it explained
+    `profile_locked`. A 401 is not that: the credentials are expired or
+    wrong (a profile's last about a day), and a reader told to look for a
+    run holding the pid goes looking in the wrong place. Classified by what
+    the failure SAYS, not by which driver raised it. Pure text in, text out,
+    and no secret ever goes into the answer.
+
+    pyppeteer never surfaces the HTTP status of a rejected upgrade — only a
+    timeout ends its connect — so a bare timeout is named as ambiguous
+    rather than guessed at.
+    """
+    text = (error_text or "").lower()
+    if re.search(r"\b401\b", text) or "deny_no_user" in text or "unauthorized" in text:
+        return ("The endpoint refused the credentials (HTTP 401). A Scraping "
+                "Browser profile's login expires after about a day, so fetch "
+                "fresh ones from the 2Captcha dashboard and check the "
+                "`ws://{login}-zone-scraping_browser-country-xx-pid-{id}:"
+                "{password}@cb.2captcha.com:9222` shape. This is not "
+                "`profile_locked`, and another pid will not help.")
+    if "profile_locked" in text or re.search(r"\b500\b", text):
+        return ("A Scraping Browser profile allows ONE live connection, so "
+                "`profile_locked` means something holds this `pid` — it stays "
+                "locked for a couple of seconds after a clean disconnect. "
+                "Two profiles were also observed entering that state and not "
+                "leaving it, one for over forty minutes; if waiting does not "
+                "clear it, use a different pid or reset the profile from the "
+                "2Captcha dashboard.")
+    if any(k in text for k in ("econnrefused", "getaddrinfo", "name or service",
+                               "not known", "unreachable", "econnreset")):
+        return ("The endpoint could not be reached at all — a network or DNS "
+                "problem, not a rejected login. Check the host and port.")
+    return ("The connection did not complete and the failure carries no "
+            "status. Expired credentials (HTTP 401) and a locked profile "
+            "(`profile_locked`) both look like this from some drivers; try "
+            "fresh credentials first, then a different pid.")
+
+
 def block_advice(html: Optional[str], headless: bool, has_pool: bool) -> str:
     """What a reader should actually DO about this block.
 

playwright_scraper.py

@@ -492,16 +492,7 @@ def _connect_remote(pw, args):
             f"could not connect to --cdp-endpoint "
             f"{_mask_credentials(args.cdp_endpoint)}: "
             f"{_mask_credentials(str(e))}\n"
-            f"A Scraping Browser profile allows ONE live connection at a "
-            f"time, so `profile_locked` means something holds this `pid`.\n"
-            f"Worth knowing before you go looking for it on your side: two "
-            f"profiles were observed here entering that state and NOT leaving "
-            f"it — one for over forty minutes, one still locked after four "
-            f"minutes of no requests at all, having locked on its very first "
-            f"connection attempt. Waiting did not clear either. If that is "
-            f"what you are seeing, it is not another run of this tool holding "
-            f"it, and nothing on this side will free it: use a different pid, "
-            f"or reset the profile from the 2Captcha dashboard."
+            f"{page_flow.cdp_connect_advice(_mask_credentials(str(e)))}"
         ) from None
     context = browser.contexts[0] if browser.contexts else browser.new_context()
     page = context.new_page()

puppeteer_scraper.py

@@ -65,7 +65,7 @@
                             detect_bot_challenge, listing_kind, normalize_url,
                             page_url, paginates_by_url, redirected_away,
                             served_by_medium, source_of, unsupported_reason)
-from output_writer import dedupe_by_key, finish_run
+from output_writer import dedupe_by_key, finish_run, EXIT_API_ERROR
 import page_flow
 from proxy_pool import (from_args as proxy_pool_from_args, split_credentials,
                         mask, ROTATE_MODES, ProxyError)
@@ -90,6 +90,14 @@
 CONNECT_TIMEOUT = 150
 
 
+class CdpConnectError(RuntimeError):
+    """A remote browser that would not accept the connection.
+
+    A remote API failure (exit 5), not a crash in this code (exit 1): the
+    Playwright engine already reports it that way.
+    """
+
+
 class _AsyncBridge:
     """Runs pyppeteer's coroutines on a private event loop, synchronously.
 
@@ -233,11 +241,20 @@ def open(self):
             # form and authenticates on the WebSocket upgrade, so an
             # authenticated Scraping Browser endpoint works here — unlike
             # Selenium's debuggerAddress, which has nowhere to put a password.
-            self.browser = self.bridge.run(
-                connect(browserWSEndpoint=self.args.cdp_endpoint,
-                        ignoreHTTPSErrors=True),
-                timeout=getattr(self.args, "cdp_connect_timeout",
-                                CONNECT_TIMEOUT))
+            try:
+                self.browser = self.bridge.run(
+                    connect(browserWSEndpoint=self.args.cdp_endpoint,
+                            ignoreHTTPSErrors=True),
+                    timeout=getattr(self.args, "cdp_connect_timeout",
+                                    CONNECT_TIMEOUT))
+            except Exception as e:  # noqa: BLE001 — any failure to connect
+                # The library's text can carry the endpoint, password and
+                # all, so mask before it is put anywhere.
+                text = _mask_credentials(str(e))
+                raise CdpConnectError(
+                    f"could not connect to --cdp-endpoint "
+                    f"{_mask_credentials(self.args.cdp_endpoint)}: {text}\n"
+                    f"{page_flow.cdp_connect_advice(text)}") from None
             self.page = self.bridge.run(self.browser.newPage())
             _watch_graphql(self)
             return self
@@ -1110,3 +1127,6 @@ def parse_args():
     except ProxyError as e:
         logger.error("%s", e)
         sys.exit(2)
+    except CdpConnectError as e:
+        logger.error("%s", e)
+        sys.exit(EXIT_API_ERROR)

smoke_test.py

@@ -1642,6 +1642,106 @@ def test_credentials_never_reach_a_log():
     return ok
 
 
+def test_cdp_connect_failure_names_its_cause():
+    group("A refused --cdp-endpoint says WHY, and a 401 is not profile_locked")
+    ok = True
+    secret = "hunter2"
+    # Concatenated, so no line here is a complete scheme://user:pass@host
+    # literal (see test_credentials_never_reach_a_log).
+    endpoint = "ws://user:" + secret + "@cb.2captcha.com:9222"
+    advice = page_flow.cdp_connect_advice
+    unauthorized = "WebSocket error: " + endpoint + " 401 Unauthorized (deny_no_user)"
+    ok &= check("a 401 is named as expired credentials",
+                "expires after" in advice(unauthorized))
+    # Each signal alone, so removing one of the three cannot hide behind the
+    # other two in the line above.
+    ok &= check("a bare status 401 is enough",
+                "expires after" in advice("server returned HTTP 401"))
+    ok &= check("deny_no_user alone is enough",
+                "expires after" in advice("deny_no_user"))
+    ok &= check("a 401 is NOT explained as a locked profile",
+                "means something holds" not in advice(unauthorized))
+    ok &= check("a 500 / profile_locked is explained as a held pid",
+                "means something holds" in advice("HTTP 500 profile_locked"))
+    ok &= check("an unreachable host is a network problem",
+                "network" in advice("getaddrinfo ENOTFOUND cb.example"))
+    ok &= check("a bare timeout is called ambiguous, not guessed",
+                "no status" in advice("Timeout 30000ms exceeded"))
+    # A port that merely contains the numerals must not read as a status.
+    ok &= check("numerals inside a port are not an HTTP status",
+                "expires" not in advice("connect to host:14019 failed"))
+    ok &= check("the advice never carries a secret",
+                all(secret not in advice(t) for t in (unauthorized, endpoint)))
+
+    # Through each engine that CAN connect remotely: the message that
+    # reaches the log names the cause, and carries no password.
+    try:
+        import playwright_scraper as pw_engine
+    except ImportError:
+        pw_engine = None
+    if pw_engine is not None:
+        class _Chromium:
+            def connect_over_cdp(self, *a, **k):
+                raise pw_engine.PWError(unauthorized)
+
+        class _Pw:
+            chromium = _Chromium()
+
+        args = type("A", (), {"cdp_endpoint": endpoint,
+                              "cdp_connect_timeout": 1})()
+        try:
+            pw_engine._connect_remote(_Pw(), args)
+            message = ""
+        except pw_engine.PWError as e:
+            message = str(e)
+        ok &= check("playwright: a 401 reaches the log as a 401",
+                    "expires after" in message
+                    and "means something holds" not in message)
+        ok &= check("playwright: no password in the connect error",
+                    bool(message) and secret not in message)
+
+    try:
+        import puppeteer_scraper as pp_engine
+    except ImportError:
+        pp_engine = None
+    if pp_engine is not None:
+        class _Bridge:
+            def run(self, coro, timeout=None):
+                # A library error repeats the endpoint, password and all.
+                raise TimeoutError("pyppeteer call did not return within 10s: "
+                                   + " ".join([endpoint] * 3))
+
+        def _connect(**kwargs):
+            return None
+
+        real = pp_engine.connect
+        pp_engine.connect = _connect
+        try:
+            args = type("A", (), {"cdp_endpoint": endpoint,
+                                  "cdp_connect_timeout": 1})()
+            try:
+                pp_engine._Session(_Bridge(), args, None).open()
+                message = ""
+            except pp_engine.CdpConnectError as e:
+                message = str(e)
+        finally:
+            pp_engine.connect = real
+        ok &= check("pyppeteer: a failed connect is a CdpConnectError",
+                    bool(message))
+        ok &= check("pyppeteer: no password in the connect error",
+                    bool(message) and secret not in message)
+        ok &= check("pyppeteer: a timeout with no status is called ambiguous",
+                    "no status" in message)
+        ok &= check("pyppeteer: the host and port survive the masking",
+                    "cb.2captcha.com:9222" in message)
+        source = _engine_source("puppeteer_scraper") or ""
+        ok &= check("pyppeteer: CdpConnectError exits 5, not a traceback",
+                    "except CdpConnectError" in source
+                    and "EXIT_API_ERROR" in
+                    source.split("except CdpConnectError")[1][:120])
+    return ok
+
+
 def test_driver_primitives_tolerate_a_navigation():
     group("Every driver primitive survives the page moving under it")
     ok = True
@@ -2635,6 +2735,7 @@ def main() -> int:
     ok &= test_fingerprint_cache_is_key_aware()
     ok &= test_proxy_pool()
     ok &= test_credentials_never_reach_a_log()
+    ok &= test_cdp_connect_failure_names_its_cause()
     ok &= test_driver_primitives_tolerate_a_navigation()
     ok &= test_concurrency_machinery(skips)
     ok &= test_engine_parity(skips)