Canary: use a given endpoint on a fresh Russian session; fix the run check crash

.github/ci_checks.py

@@ -175,7 +175,7 @@ def run_check(prefix="sample_output"):
         failed.append(f"{len(inverted)} row(s) have an original_price at or below their price")
     if not failed:
         print(f"ok       {len(rows)} rows, status={meta.get('status')}, "
-              f"{priced} priced, {named} named sellers, "
+              f"{priced} priced, "
               f"stop_reason={meta.get('stop_reason')!r}")
     return failed
 

.github/workflows/canary.yml

@@ -38,6 +38,9 @@ jobs:
       HAS_TWOCAPTCHA_KEY: ${{ secrets.TWOCAPTCHA_KEY != '' }}
       HAS_AVITO_PROXY: ${{ secrets.AVITO_PROXY != '' }}
       HAS_ACCOUNT_ID: ${{ vars.AVITO_BROWSER_ACCOUNT_ID != '' }}
+      # A ready endpoint from the owner (secret AVITO_CDP_ENDPOINT) is used as
+      # is; only without one does the job build its own from the account.
+      HAS_CDP: ${{ secrets.AVITO_CDP_ENDPOINT != '' }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -51,15 +54,54 @@ jobs:
           missing=0
           [ "$HAS_TWOCAPTCHA_KEY" = "true" ] || { echo "::error::secret TWOCAPTCHA_KEY is not set — the canary cannot reach avito.ru"; missing=1; }
           [ "$HAS_AVITO_PROXY" = "true" ] || { echo "::error::secret AVITO_PROXY (the base Russian gateway login) is not set"; missing=1; }
-          [ "$HAS_ACCOUNT_ID" = "true" ] || { echo "::error::repository variable AVITO_BROWSER_ACCOUNT_ID is not set"; missing=1; }
+          [ "$HAS_CDP" = "true" ] || [ "$HAS_ACCOUNT_ID" = "true" ] || { echo "::error::neither secret AVITO_CDP_ENDPOINT nor repository variable AVITO_BROWSER_ACCOUNT_ID is set"; missing=1; }
           exit $missing
 
       - name: Install
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt -r requirements-playwright.txt
 
+      - name: Use the given endpoint, on a fresh Russian session
+        if: env.HAS_CDP == 'true'
+        # The owner's endpoint is re-pointed at a Russian exit: `country-ru`,
+        # and a `-proxy-<base64url>` segment carrying a session freshly minted
+        # on AVITO_PROXY. The solver gets that SAME session, so a solved
+        # captcha leaves from the browser's IP. Both are written to a 0600
+        # .env from the environment and never printed.
+        env:
+          AVITO_CDP_ENDPOINT: ${{ secrets.AVITO_CDP_ENDPOINT }}
+          AVITO_PROXY: ${{ secrets.AVITO_PROXY }}
+        run: |
+          install -m 600 /dev/null .env
+          python3 - <<'PY2'
+          import base64, os, re, sys
+          from urllib.parse import urlsplit, urlunsplit, quote, unquote
+          sys.path.insert(0, "tools"); sys.path.insert(0, ".")
+          import browser_profile_client as bpc
+          proxy_url = bpc.minted_proxy(60)  # fresh -session-/-sessTime-60 on AVITO_PROXY
+          ep = urlsplit(os.environ["AVITO_CDP_ENDPOINT"].strip())
+          user = unquote(ep.username or "")
+          if not user or "-zone-" not in user:
+              sys.exit("!! AVITO_CDP_ENDPOINT has no Scraping Browser login segment")
+          user = re.sub(r"-proxy-[A-Za-z0-9_=-]+$", "", user)
+          if re.search(r"-country-[a-z]{2}", user):
+              user = re.sub(r"-country-[a-z]{2}", "-country-ru", user)
+          else:
+              user = re.sub(r"(-zone-[a-z_]+)", r"\1-country-ru", user, count=1)
+          enc = base64.urlsafe_b64encode(proxy_url.encode()).decode().rstrip("=")
+          user += "-proxy-" + enc
+          netloc = quote(user, safe="-_.~") + ":" + quote(unquote(ep.password or ""), safe="-_.~") + "@" + ep.hostname + (f":{ep.port}" if ep.port else "")
+          uri = urlunsplit((ep.scheme, netloc, ep.path, ep.query, ep.fragment))
+          with open(".env", "w", encoding="utf-8") as f:
+              f.write("AVITO_CDP_ENDPOINT=" + uri + "\n")
+              f.write("AVITO_PROXY=" + proxy_url + "\n")
+          print("endpoint         :", bpc.mask_url(uri), "(country-ru, custom proxy segment)")
+          print("solver proxy     :", bpc.mask_url(proxy_url), "(same session)")
+          PY2
+
       - name: Build a fresh endpoint and its matching solver proxy
+        if: env.HAS_CDP != 'true'
         # Credentials through the ENVIRONMENT, never argv. The tool writes
         # AVITO_CDP_ENDPOINT and the minted AVITO_PROXY into a 0600 .env and
         # prints both masked.

CHANGELOG.md

@@ -18,6 +18,18 @@ entry in a blockquote.
 
 ### Fixed
 
+* **The canary's run check crashed on a good run.** Its success message still
+  named a seller count whose computation had been removed, so the first
+  complete live scrape (150 rows, 2026-10-16) ended in a `NameError` and a red
+  job. The message no longer uses it, and the suite now runs the check on a
+  synthetic good run and on a broken one.
+* **The canary can use a ready Scraping Browser endpoint.** With the
+  `AVITO_CDP_ENDPOINT` secret set, the job uses that endpoint instead of
+  building one from `AVITO_BROWSER_ACCOUNT_ID`, re-pointed at a Russian exit:
+  `country-ru` and a `-proxy-` segment carrying a session freshly minted on
+  `AVITO_PROXY`, which the captcha solver then uses too. Measured 2026-10-16:
+  148 rows over 3 pages, complete; the same endpoint without the Russian
+  proxy did not clear the firewall's proof-of-work.
 * **`tools/browser_profile_client.py` hid the Browser API's error code.** An
   HTTP error from the API was reported as the bare status ("404 Client Error:
   Not Found"), dropping the body that names the cause, e.g.

smoke_test.py

@@ -2145,6 +2145,45 @@ def test_canary_cannot_go_green_without_scraping():
     check("and no longer a seller-name share", "rows name a seller" not in checks)
 
 
+def test_the_canary_run_check_passes_a_good_run():
+    # The canary's run check only executes after a successful live scrape, so
+    # a crash in its success path stays hidden until the first green run: on
+    # 2026-10-16 it died with a NameError on a 150-row run that had worked.
+    # Run it here on a synthetic good run, and on a broken one.
+    import importlib.util
+    import tempfile
+    from output_writer import SOURCE_DEFAULT
+    spec = importlib.util.spec_from_file_location(
+        "ci_checks_under_test", os.path.join(HERE, ".github", "ci_checks.py"))
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    rows = [{"sku": str(1000 + i), "url": "https://www.avito.ru/moskva/x/%d" % i,
+             "title": "item %d" % i, "price": 100 + i, "original_price": None,
+             "region_slug": "moskva", "page": 1 + i // 50, "position": i % 50}
+            for i in range(150)]
+    meta = {"status": "complete", "pages_completed": 3, "source": SOURCE_DEFAULT,
+            "stop_reason": "max_pages"}
+    with tempfile.TemporaryDirectory() as tmp:
+        prefix = os.path.join(tmp, "canary")
+        with open(prefix + ".json", "w", encoding="utf-8") as f:
+            json.dump(rows, f)
+        with open(prefix + ".meta.json", "w", encoding="utf-8") as f:
+            json.dump(meta, f)
+        try:
+            failed = mod.run_check(prefix)
+        except Exception as exc:  # noqa: BLE001
+            failed = ["raised %s: %s" % (type(exc).__name__, exc)]
+        check("the canary run check passes a good 150-row run", failed == [],
+              "(got %r)" % (failed,))
+        rows[0]["price"] = None
+        rows[1]["sku"] = ""
+        with open(prefix + ".json", "w", encoding="utf-8") as f:
+            json.dump(rows, f)
+        failed = mod.run_check(prefix)
+        check("and fails one with a row missing its sku",
+              any("no sku" in x for x in failed), "(got %r)" % (failed,))
+
+
 def main():
     tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
     for test in tests: