Family sync 2026-09-29: fingerprint script, pyppeteer proxy auth, CI pipefail, skips, gitignore

.github/ci_checks.py

@@ -92,12 +92,37 @@
              ".pytest_cache"}
 
 
+_VENV_CACHE = {}
+
+
+def _is_in_virtualenv(path):
+    """True if any ancestor of `path` under REPO is a virtualenv root.
+
+    A directory holding `pyvenv.cfg` IS a virtualenv, whatever it is called
+    (family template §23). The name set in SKIP_DIRS missed CI's own
+    `.venv-<engine>`, so the engine jobs scanned pip's and selenium's vendored
+    code and failed on it — hidden until pipefail made that job's exit status
+    real (2026-09-29).
+    """
+    for parent in path.parents:
+        cached = _VENV_CACHE.get(parent)
+        if cached is None:
+            cached = _VENV_CACHE[parent] = (parent / "pyvenv.cfg").is_file()
+        if cached:
+            return True
+        if parent == REPO:
+            break
+    return False
+
+
 def scanned_files():
     for path in sorted(REPO.rglob("*")):
         if not path.is_file() or path.suffix not in SCANNED_SUFFIXES:
             continue
         if any(part in SKIP_DIRS for part in path.parts):
             continue
+        if _is_in_virtualenv(path):
+            continue
         yield path
 
 

.github/workflows/tests.yml

@@ -92,9 +92,29 @@ jobs:
           # Proves each lock is self-consistent on its own.
           pip check
 
+      - name: A browser for this engine
+        # The suite launches the engine for real in places (the fingerprint-
+        # refusal check starts Chromium before it fetches). Without a browser
+        # that check had been red — hidden until pipefail (2026-09-29).
+        # Playwright brings its own Chromium; the other two use the runner's
+        # Google Chrome.
+        run: |
+          . .venv-${{ matrix.engine }}/bin/activate
+          if [ "${{ matrix.engine }}" = playwright ]; then
+            python -m playwright install --with-deps chromium
+          else
+            echo "PYPPETEER_EXECUTABLE_PATH=$(command -v google-chrome)" >> "$GITHUB_ENV"
+            google-chrome --version
+          fi
+
       - name: The suite, with THIS engine importable
         # The other two are absent by design, so the suite will report skips
         # for them; what must NOT happen is this engine's group skipping.
+        # `shell: bash` turns on pipefail: GitHub runs an UNSPECIFIED shell as
+        # `bash -e` without it, so `smoke_test.py | tee` took tee's exit status
+        # and a failing suite went green (found by an audit of a sibling repo,
+        # 2026-09-29).
+        shell: bash
         run: |
           set -e
           . .venv-${{ matrix.engine }}/bin/activate
@@ -103,6 +123,11 @@ jobs:
             puppeteer)  module=puppeteer_scraper ;;
             selenium)   module=selenium_scraper ;;
           esac
+          # The import itself, first: it fails BY NAME, because every engine
+          # imports its driver at module level, whatever wording the suite
+          # uses for a skip (family template §26 — the grep below fired in
+          # only 2 of 31 sibling suites when this was measured).
+          python -c "import $module"
           python smoke_test.py | tee smoke.log
           if grep -q "SKIPPED.*$module" smoke.log; then
             echo "FAILED: $module reported skipped although ${{ matrix.engine }} is installed in this job — likely a real import error."

.gitignore

@@ -8,7 +8,23 @@ lg_products.*
 *.meta.json
 *_debug.html
 *_debug.png
+# --dump-html X over several pages writes X.page2, X.page3 …, which no
+# suffix rule above covers (family template §26).
+*.page[0-9]*
+
+# ANY working directory a live run writes into, not one chosen name: a
+# sibling committed a 1.5 MB raw dump through `live/` because only
+# `live_results/` was listed (family template §24).
+live/
 live_results/
+out/
+output/
+runs/
+results/
+tmp/
+scratch/
+dumps/
+captures/
 
 __pycache__/
 *.py[cod]
@@ -25,6 +41,11 @@ env/
 .idea/
 .vscode/
 *.log
-.env
 *.key
 secrets*
+
+# Every spelling of a working .env, not just `.env`: a `.env.bak` or
+# `.env.local` is exactly how a live key gets one `git add -A` from a commit
+# (family template §22). The negation keeps the documented example tracked.
+.env*
+!.env.example

CHANGELOG.md

@@ -5,6 +5,61 @@ All notable changes to this project are documented here. The format follows
 [SemVer](https://semver.org/) as closely as a CLI toolkit can: a PATCH release
 means fixes, not that every flag is frozen.
 
+## [Unreleased]
+
+### Fixed
+
+Found by a family-wide pass against the template on 2026-09-29; each one
+made a documented feature do less than it said while the run reported
+success.
+
+- **`CONTRIBUTING.md` promised checks the suite did not make**: that a
+  workflow installing anything but a hashed lock, a lock drifting from its
+  `.txt`, or an action referenced by tag fails the suite. It now does.
+- **`--fingerprint` never patched anything.** The init script it installs
+  was syntactically broken JavaScript (`( => {` — the parameter list and the
+  IIFE call were missing), and a syntax error in an init script is silent:
+  the page ran unpatched. Fixed, and the suite now runs `node --check` on
+  that script too — 0.2.2's "every shipped script" did not include it.
+- **pyppeteer could not authenticate a proxy on current Chrome.**
+  `page.authenticate()` uses `Network.setRequestInterception`, which
+  current Chrome no longer has ("wasn't found", exit 5 before the first
+  request — measured 2026-09-29 in rosreestr-scraper, which shares this
+  engine code). The engine now answers the proxy's 407 through the CDP
+  Fetch domain; verified against a local Basic-auth proxy.
+- **A failing suite could turn the engine CI job green.** The step ran
+  `smoke_test.py | tee` under GitHub's default shell, which has no
+  pipefail, so the job took tee's exit status. It now runs with
+  `shell: bash`, imports the engine by name first, and the suite fails if
+  any piped step lacks pipefail.
+- **The engine CI jobs had been failing, unseen.** Once pipefail made the
+  step's exit status real, all three engine jobs went red on the secret
+  scan: it walked into CI's own `.venv-<engine>` and read pip's and
+  selenium's vendored code as credentials. The scan now skips any directory
+  holding `pyvenv.cfg`, whatever it is called (template §23), and still
+  reads ordinary untracked files — both planted in a check.
+- **The Playwright engine job never had a browser**, so the suite's
+  fingerprint-refusal check — which starts Chromium before it fetches —
+  crashed there with exit 1 instead of the expected 5. Also hidden until
+  pipefail; the engine jobs now install the browser they drive.
+- **Skipped checks were counted as passed.** A check that could not run
+  printed "SKIPPED" and was added to the pass count. Skips are now their own
+  count, listed at the end.
+- **`env_config.py --help` read `.env`.** `ci_checks.py` runs every CLI's
+  `--help`; a help screen now opens no credentials file.
+- **`.gitignore` let a renamed `.env` through** (`.env.bak`, `.env.local`),
+  and a run directory (`live/`) or a paged dump (`X.page3`). Now ignored by
+  shape, with a check.
+
+### Changed
+
+- **Connecting to the Scraping Browser API** follows one policy in every
+  engine (family template §26): three attempts 3 s apart for a locked
+  profile or an outage, none for an expired login (401), and a message that
+  names the status's own cause. pyppeteer's per-attempt timeout is 10 s (it
+  reports a refused handshake only as a timeout), and the orphaned-task
+  traceback that followed a correct exit is filtered.
+
 ## [0.2.2] — 2026-09-28
 
 ### Fixed

env_config.py

@@ -304,6 +304,13 @@ def apply(args, keys=None, quiet=False):
 if __name__ == "__main__":
     # `python3 env_config.py` — report what is configured, without printing
     # any secret. Useful as a first step when a key "isn't being picked up".
+    import sys
+    if any(arg in ("-h", "--help") for arg in sys.argv[1:]):
+        # Answered WITHOUT reading .env: ci_checks runs every CLI's --help,
+        # and a help screen has no business opening a credentials file.
+        print("usage: python3 env_config.py\n\nReports which variables from "
+              ".env / the environment were picked up, never their values.")
+        sys.exit(0)
     logging.basicConfig(level=logging.INFO, format="%(message)s")
     where = load_env()
     print(f".env file:      {where or 'not found (this is fine — env vars still work)'}")

fingerprint_client.py

@@ -280,11 +280,11 @@ def playwright_init_script(fp: dict) -> str:
         "webglRenderer": found["webgl_renderer"]["value"],
     })
     return """
-( => {
+(() => {
   const fp = %s;
   const def = (obj, prop, value) => {
     if (value === null || value === undefined) return;
-    try { Object.defineProperty(obj, prop, {get:  => value, configurable: true}); }
+    try { Object.defineProperty(obj, prop, {get: () => value, configurable: true}); }
     catch (e) { /* already non-configurable: leave it rather than throw */ }
   };
   def(Navigator.prototype, 'platform', fp.platform);
@@ -303,7 +303,7 @@ def playwright_init_script(fp: dict) -> str:
       return original.apply(this, arguments);
     };
   }
-});
+})();
 """ % payload
 
 

playwright_scraper.py

@@ -66,7 +66,8 @@
 from output_writer import dedupe_by_sku, finish_run
 from product_parser import (SELECTORS, category_from_url, page_url,
                             parse_products, unsupported_locale_reason)
-from proxy_pool import (ROTATE_MODES, ProxyError, ProxyPool,
+from proxy_pool import (CDP_CONNECT_ATTEMPTS, CDP_CONNECT_PAUSE_S, CDP_CONNECT_TIMEOUT_S,
+                        ROTATE_MODES, cdp_refusal_advice, cdp_retryable, ProxyError, ProxyPool,
                         check_exit_or_raise,
                         from_args as proxy_pool_from_args, mask,
                         redact_secret_patterns, to_playwright)
@@ -201,12 +202,26 @@ def _connect_remote(pw, args):
     # underneath it. Measured 2026-09-19 against a real endpoint that answered
     # 401: the password appeared five times in one traceback. An exception
     # message is a log.
-    try:
-        browser = pw.chromium.connect_over_cdp(args.cdp_endpoint, timeout=30000)
-    except Exception as e:  # noqa: BLE001 — re-raised immediately, redacted
-        raise RuntimeError(
-            f"Could not connect to --cdp-endpoint: "
-            f"{redact_secret_patterns(str(e))}") from None
+    #
+    # Retried per proxy_pool's CDP policy: a profile stays locked for ~2 s
+    # after a clean disconnect (family template §26), so a run started right
+    # after another met HTTP 500. A 401 (expired credentials) is not retried.
+    for attempt in range(1, CDP_CONNECT_ATTEMPTS + 1):
+        try:
+            browser = pw.chromium.connect_over_cdp(args.cdp_endpoint, timeout=30000)
+            break
+        except Exception as e:  # noqa: BLE001 — re-raised immediately, redacted
+            # Whole message, on one line: the endpoint's host:port is what
+            # says WHICH exit failed, and it sits past the first line.
+            text = " ".join(redact_secret_patterns(str(e)).split())[:600]
+            if attempt < CDP_CONNECT_ATTEMPTS and cdp_retryable(text):
+                logger.warning("Scraping Browser connect failed (%s); retrying in %.0fs "
+                               "[%d/%d].", text, CDP_CONNECT_PAUSE_S, attempt,
+                               CDP_CONNECT_ATTEMPTS)
+                time.sleep(CDP_CONNECT_PAUSE_S)
+                continue
+            raise RuntimeError(f"Could not connect to --cdp-endpoint: {text} — "
+                               f"{cdp_refusal_advice(text)}") from None
     # Reuse the remote browser's existing context so its fingerprint, session
     # and exit stay intact.
     context = browser.contexts[0] if browser.contexts else browser.new_context()

proxy_pool.py

@@ -371,3 +371,64 @@ def from_args(args) -> Optional[ProxyPool]:
         return ProxyPool([entry], rotate="per-run")
 
     return None
+
+
+# ---------------------------------------------------------------------------
+# Connecting to the Scraping Browser API — one policy for every engine
+# ---------------------------------------------------------------------------
+# Measured in the family (template §26, 2026-09-24): a profile stays
+# `profile_locked` for 1.6-1.9 s after a CLEAN disconnect, so a second run
+# started straight after the first was refused with HTTP 500. Three attempts
+# 3 s apart ride that out, while a profile genuinely held by another run
+# still fails in about 9 s. And pyppeteer's connect never surfaces the 500
+# at all — only a timeout ends it — so its per-attempt timeout is short: a
+# successful connect took 0.8-0.95 s. The 500's body names the cause when it
+# is a lock: "profile_locked" (seen 2026-09-29 in rosreestr-scraper's canary,
+# where two jobs shared one profile — no retry window covers that).
+CDP_CONNECT_ATTEMPTS = 3
+CDP_CONNECT_PAUSE_S = 3.0
+CDP_CONNECT_TIMEOUT_S = 10.0
+
+
+def _has_status(text: str, *codes: str) -> bool:
+    """A status as a NUMBER on its own — never a digit run inside a port, a
+    duration or a request id (a bare "500" matched "15000ms")."""
+    return any(re.search(rf"(?<!\d){c}(?!\d)", text) for c in codes)
+
+
+def cdp_refusal_advice(error_text: str) -> str:
+    """What a failed Scraping Browser connect most likely means, by its status.
+
+    The old message explained a 500 whatever the status was, and every
+    expired endpoint in the family answers 401.
+    """
+    text = error_text or ""
+    if "profile_locked" in text:
+        # The service's own word for it, in the 500's body (seen 2026-09-29
+        # when two canary jobs shared one profile).
+        return ("the profile is in use by another connection (profile_locked): "
+                "one live connection per pid. Wait for the other run to finish, or "
+                "give each concurrent run its own pid.")
+    if _has_status(text, "401"):
+        return ("HTTP 401: the profile's credentials were refused — they expire "
+                "(about a day). Take fresh ones from the 2Captcha dashboard.")
+    if _has_status(text, "403"):
+        return "HTTP 403: this account may not use that profile or zone."
+    if _has_status(text, "500", "502", "503"):
+        return ("HTTP 5xx on the WebSocket upgrade: usually the profile is still "
+                "locked by another (or just-finished) connection — one live "
+                "connection per pid — or a service-side outage. Wait, or use "
+                "another pid.")
+    if "imeout" in text or not text.strip():
+        return ("no answer in time. pyppeteer reports a REFUSED handshake this "
+                "way too, so this may be the same 5xx; try the Playwright engine "
+                "to see the status.")
+    return "see the error above."
+
+
+def cdp_retryable(error_text: str) -> bool:
+    """Worth another attempt: a lock or an outage, never a refused login."""
+    text = error_text or ""
+    if _has_status(text, "401", "403"):
+        return False
+    return _has_status(text, "500", "502", "503") or "imeout" in text or not text.strip()