Atomic output, CSV formula guard, one paid solve per page, canary names a dead proxy

.env.example

@@ -71,7 +71,7 @@ MEDIAMARKT_CDP_ENDPOINT=ws://{login}-zone-scraping_browser-country-de-pid-{profi
 MEDIAMARKT_PROXY=http://{user}:{password}@eu.proxy.2captcha.com:2334
 
 # --- Default target -------------------------------------------------------
-# Saves retyping --url. Any of the eleven country sites; the hostname decides
+# Saves retyping --url. Any of the ten country sites; the hostname decides
 # which, so there is no --country flag that could disagree with the URL.
 #
 # Careful with category URLs: some are HUB pages that list sub-categories and

.github/canary_preflight.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""
+Check the canary's proxy BEFORE the browser run, and name what is wrong.
+
+Why this exists: from 2026-09-10 the daily canary failed on every run with
+the same three lines, `Timeout loading ... (attempt 1/3)`, `2/3`, `Gave up`,
+then exit 5. The scraper cannot say more than that: Chromium reports a proxy
+that accepts a connection and never delivers anything as a plain navigation
+timeout. This script asks the same proxy the same question with short
+timeouts and turns the answer into one of a few named causes, so a red run
+says "the proxy secret is dead" instead of "unexpected exit code".
+
+It reads the proxy from MEDIAMARKT_PROXY and never prints it: only the host
+and port (which exit, not the secret) and the two-letter region code in the
+login, if there is one.
+
+Exit codes:
+    0  the proxy delivered a page from the site (the browser run goes ahead)
+    1  the proxy is unusable, and the ::error:: line says how
+    2  MEDIAMARKT_PROXY is not set
+
+A 403 from the site through the proxy is reported as a warning and exits 0:
+the browser run is the real test of whether the exit is accepted, and a
+plain HTTP client is a weaker witness than a browser.
+"""
+
+import os
+import re
+import socket
+import sys
+from urllib.parse import urlparse
+
+TARGET = os.environ.get(
+    "CANARY_URL",
+    "https://www.mediamarkt.de/de/category/k%C3%BChlen-gefrieren-32.html")
+CONNECT_TIMEOUT = float(os.environ.get("PREFLIGHT_CONNECT_TIMEOUT", "15"))
+READ_TIMEOUT = float(os.environ.get("PREFLIGHT_READ_TIMEOUT", "45"))
+UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+      "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
+
+
+def error(msg):
+    print(f"::error::{msg}")
+
+
+def where(proxy):
+    """host:port of the proxy, which is safe to print."""
+    p = urlparse(proxy)
+    return f"{p.hostname}:{p.port}"
+
+
+def region_in_login(proxy):
+    """The exit country the login asks for, e.g. 'de', or None."""
+    user = urlparse(proxy).username or ""
+    m = re.search(r"(?:region|country)-([a-zA-Z]{2})\b", user)
+    return m.group(1).lower() if m else None
+
+
+def main():
+    proxy = (os.environ.get("MEDIAMARKT_PROXY") or "").strip()
+    if not proxy:
+        error("MEDIAMARKT_PROXY is not set. Add a residential German exit as "
+              "that repository secret.")
+        return 2
+    if "://" not in proxy:
+        proxy = "http://" + proxy
+    p = urlparse(proxy)
+    if not p.hostname or not p.port:
+        error("MEDIAMARKT_PROXY is not a URL of the form "
+              "scheme://user:password@host:port (value not printed).")
+        return 1
+
+    region = region_in_login(proxy)
+    print(f"proxy {where(proxy)}; credentials "
+          f"{'present' if p.username else 'ABSENT'}; region in login: "
+          f"{region or 'none'}")
+    if region and region != "de":
+        print(f"::warning::The proxy login asks for region '{region}'. This "
+              f"canary scrapes mediamarkt.de, which refuses most non-German "
+              f"exits; the login should ask for 'de'.")
+
+    # 1. Is anything listening at all?
+    try:
+        socket.create_connection((p.hostname, p.port), timeout=CONNECT_TIMEOUT).close()
+    except OSError as e:
+        error(f"Cannot open a TCP connection to the proxy {where(proxy)} "
+              f"({type(e).__name__}). The host or port in MEDIAMARKT_PROXY is "
+              f"wrong, or the gateway is down.")
+        return 1
+
+    import requests
+
+    # 2. Does it carry a request to the site?
+    try:
+        r = requests.get(TARGET, proxies={"http": proxy, "https": proxy},
+                         headers={"User-Agent": UA,
+                                  "Accept-Language": "de-DE,de;q=0.9"},
+                         timeout=(CONNECT_TIMEOUT, READ_TIMEOUT),
+                         allow_redirects=True)
+    except requests.exceptions.ProxyError as e:
+        text = str(e)
+        if "407" in text:
+            error(f"The proxy {where(proxy)} rejected the credentials (HTTP "
+                  f"407). The login or password in MEDIAMARKT_PROXY is wrong "
+                  f"or has expired: replace the secret.")
+        else:
+            error(f"The proxy {where(proxy)} accepted the connection but would "
+                  f"not open a tunnel to the site ({_short(text)}). This is "
+                  f"what an expired, suspended or out-of-traffic proxy "
+                  f"account looks like: replace the MEDIAMARKT_PROXY secret.")
+        return 1
+    except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout) as e:
+        error(f"The proxy {where(proxy)} accepted a TCP connection but the "
+              f"page did not arrive within {CONNECT_TIMEOUT:.0f}+"
+              f"{READ_TIMEOUT:.0f} s ({type(e).__name__}). The browser run "
+              f"would time out the same way (exit 5). A gateway that answers "
+              f"and then delivers nothing is what an expired or out-of-traffic "
+              f"proxy account looks like: replace the MEDIAMARKT_PROXY secret "
+              f"with a working residential German exit.")
+        return 1
+    except requests.exceptions.RequestException as e:
+        error(f"Requesting the site through the proxy {where(proxy)} failed: "
+              f"{type(e).__name__}: {_short(str(e))}")
+        return 1
+
+    final_host = urlparse(r.url).hostname or ""
+    print(f"site answered HTTP {r.status_code}, {len(r.content)} bytes, "
+          f"final host {final_host}")
+    if r.status_code == 403:
+        print("::warning::MediaMarkt refused this plain HTTP request through the "
+              "proxy (403). If the browser run below is refused too (exit 3), "
+              "the exit is not a residential German address.")
+    elif r.status_code >= 400:
+        print(f"::warning::The site answered HTTP {r.status_code} through the "
+              f"proxy.")
+    elif "mediamarkt.de" not in final_host:
+        print(f"::warning::The request was redirected to {final_host}: the exit "
+              f"is not in Germany, and the canary's source check will fail.")
+    return 0
+
+
+def _short(text):
+    """An exception message without anything that could be a credential."""
+    text = re.sub(r"[^\s/:@]+:[^\s/@]+@", "***@", text)
+    return text[:200]
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/ci_checks.py

@@ -76,7 +76,27 @@
 HEX32 = re.compile(r"\b[0-9a-f]{32}\b")
 HEX32_ALLOWED = ("sha", "hash", "nonce", "example", "md5", "digest", "checksum")
 
-SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example")
+SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example",
+                    # JSON too: fixtures and run outputs are where a captured
+                    # page's front-end key would arrive.
+                    ".json")
+
+# Directory names skipped wherever they appear INSIDE the repo. Judged on the
+# path relative to the repo root, never on the absolute path: a clone that
+# happens to live under a directory called `venv` or `.venv` otherwise skips
+# every file and still prints "nothing credential-shaped".
+SKIPPED_DIRS = {".git", "__pycache__", ".venv", "venv"}
+
+
+def _is_env_backup(path):
+    """`.env.bak`, `.env.local`, `.env.save`, ... but not `.env` itself.
+
+    A renamed copy of `.env` is the classic way a key ends up one `git add -A`
+    away from a commit, and its suffix (`.bak`, `.local`) is not one the suffix
+    list would ever include. `.env` itself is the user's own config, ignored by
+    git, and is supposed to hold a key; `.env.example` is matched by suffix.
+    """
+    return path.name.startswith(".env.") and path.name != ".env.example"
 
 
 # A directory holding `pyvenv.cfg` is a virtualenv, whatever it is called.
@@ -87,7 +107,7 @@
 # into pip's vendored code and flags a 32-hex string in `_elffile.py` as
 # key-shaped. Correct about the string, wrong about the file — and it is the
 # FIRST thing a new user sees from `python3 smoke_test.py`. A guard people
-# have to argue with is one they learn to suppress (CLAUDE.md §22).
+# have to argue with is one they learn to suppress.
 #
 # Deliberately NOT a narrowing of what gets scanned: an untracked file is
 # still read, because a key pasted into a scratch file beside the scripts is
@@ -116,10 +136,11 @@ def _is_in_virtualenv(path):
 
 def scanned_files():
     for path in sorted(REPO.rglob("*")):
-        if not path.is_file() or path.suffix not in SCANNED_SUFFIXES:
+        if not path.is_file():
+            continue
+        if path.suffix not in SCANNED_SUFFIXES and not _is_env_backup(path):
             continue
-        if any(part in {".git", "__pycache__", ".venv", "venv"}
-               for part in path.parts):
+        if any(part in SKIPPED_DIRS for part in path.relative_to(REPO).parts):
             continue
         if _is_in_virtualenv(path):
             continue
@@ -211,6 +232,11 @@ def secret_check():
                 failed.append(f"{rel}:{lineno} contains {match[:6]}… — a "
                               f"32-char hex string, the shape of a 2captcha key")
 
+    if not scanned:
+        # Zero files is not "clean", it is "looked at nothing". Without this a
+        # broken exclusion reports exactly what a clean repo reports.
+        failed.append("0 files scanned — the scan looked at nothing, so it "
+                      "cannot say the repo is clean")
     if not failed:
         print(f"ok       {scanned} files scanned, nothing credential-shaped")
     return failed

.github/workflows/canary.yml

@@ -17,12 +17,22 @@ name: canary
 # single day — and a check that is always red teaches everyone to ignore
 # checks, which is worse than having no check at all.
 #
-# So the live steps below are SKIPPED, not failed, when the secret is absent.
-# The job goes green with a notice saying why. Set the secret and the same
-# workflow starts doing real work with no edit:
+# So the live steps need the secret, and WITHOUT it the job FAILS, naming
+# the missing secret. It used to skip green with a notice, and a green badge
+# over a canary that tested nothing is worse than a red one: it says the site
+# was checked when it was not. Set the secret and the same workflow starts
+# doing real work with no edit:
 #
 #     gh secret set MEDIAMARKT_PROXY --repo 2scraper/mediamarkt-scraper
 #
+# A secret that is SET but dead fails too, and says so. From 2026-09-10 every
+# scheduled run timed out loading page 1 through the proxy and ended "exit 5";
+# the run itself cannot tell a dead proxy from a slow site. The preflight step
+# (.github/canary_preflight.py) asks the proxy directly with short timeouts and
+# names the cause: unreachable gateway, rejected credentials (407), a tunnel
+# that never opens, or a connection that delivers nothing (the shape of an
+# expired or out-of-traffic proxy account). Only then does the browser run.
+#
 # It must be a RESIDENTIAL exit, and for this URL a German one — the site
 # geo-redirects a cookie-less visit on the exit IP, and a run redirected to
 # another country site fails the `source` check below on purpose.
@@ -51,29 +61,53 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Explain a skipped run
+      - name: Require the proxy secret
         if: env.HAVE_PROXY != 'true'
         run: |
-          echo "::notice::Skipping the live run: no MEDIAMARKT_PROXY secret is set."
+          echo "::error::Missing secret MEDIAMARKT_PROXY: the canary cannot test the site without it."
           echo "MediaMarkt answers every datacentre address with HTTP 403, and a"
-          echo "GitHub runner is one, so a run from here would be red every day"
-          echo "whatever the state of the code. Set a residential German exit as"
-          echo "the MEDIAMARKT_PROXY repository secret to turn this canary on:"
+          echo "GitHub runner is one. Set a residential GERMAN exit as the"
+          echo "MEDIAMARKT_PROXY repository secret:"
           echo
           echo "    gh secret set MEDIAMARKT_PROXY --repo 2scraper/mediamarkt-scraper"
-          echo
-          echo "The offline suite (tests.yml) covers everything that does not"
-          echo "need the network, and it runs on every push."
+          exit 1
+
+      - name: Can this runner reach the site without a proxy?
+        if: always()
+        # Informational, never fails the job. It separates "the site is down"
+        # from "the site refuses this address": a 403 here is the expected
+        # datacentre refusal, a 000 means the site did not answer at all.
+        run: |
+          code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 30 \
+            -A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" \
+            "https://www.mediamarkt.de/de/category/k%C3%BChlen-gefrieren-32.html" || true)
+          case "$code" in
+            403) echo "::notice::Bare runner: HTTP 403, the site's datacentre refusal. The site is up; the proxy is what has to get past it." ;;
+            200) echo "::notice::Bare runner: HTTP 200. The site served a datacentre address today, so a failure below is about the proxy, not the site." ;;
+            000) echo "::warning::Bare runner: no HTTP answer from mediamarkt.de within 30 s. The site or the network may be down." ;;
+            *)   echo "::notice::Bare runner: HTTP $code." ;;
+          esac
 
       - uses: actions/setup-python@v5
         if: env.HAVE_PROXY == 'true'
         with:
           python-version: "3.12"
 
-      - name: Install Playwright engine
+      - name: Check the proxy before the browser run
         if: env.HAVE_PROXY == 'true'
+        # Fails the job with an ::error:: that names the cause when the proxy
+        # cannot deliver a page, and then the browser run is not attempted:
+        # it would only time out three times and report exit 5.
+        env:
+          MEDIAMARKT_PROXY: ${{ secrets.MEDIAMARKT_PROXY }}
         run: |
           python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          python .github/canary_preflight.py
+
+      - name: Install Playwright engine
+        if: env.HAVE_PROXY == 'true'
+        run: |
           pip install -r requirements.txt -r requirements-playwright.txt
           playwright install --with-deps chromium
 

.gitignore

@@ -24,6 +24,20 @@ live_results.zip
 *_debug.html
 *_debug.png
 *.html.page*
+# --dump-html X --pages N writes X.page1..X.pageN whatever X is called, so a
+# dump named canary_dump.json becomes canary_dump.json.page3, which neither
+# `*.json` nor `*.html.page*` covers.
+*.page[0-9]*
+# A bare dump at the repo root (canary_run.html, dump.html). No page of this
+# project is an HTML file at the root.
+/*.html
+# Working directories by KIND, not by one chosen name: a raw capture reached
+# a sibling repo's first commit through `live/`, a directory nobody had listed.
+live/
+out/
+output/
+dumps/
+scratch/
 
 # --- captures kept for parser work ---------------------------------------
 # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py);
@@ -73,7 +87,6 @@ secrets*
 # recoverable only because that repo was private at the time. On 2026-09-17
 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC
 # repository, one `git add -A` from publication. Craigslist was fixed then
-# and the fix never reached anyone else, which is §16's rule unapplied: when
-# you fix one, check the siblings.
+# and the fix never reached anyone else: when you fix one, check the siblings.
 .env*
 !.env.example