From 4b069dc8f549fe8c7d17f45de47804f9558d5d0e Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:42:39 +0000 Subject: [PATCH 1/4] Canary: use a given AVITO_CDP_ENDPOINT before building one Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CKsd7q25N19WzUZjrAdzx8 --- .github/workflows/canary.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 9ec70b8..3c6c3e1 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -38,6 +38,9 @@ jobs: HAS_TWOCAPTCHA_KEY: ${{ secrets.TWOCAPTCHA_KEY != '' }} HAS_AVITO_PROXY: ${{ secrets.AVITO_PROXY != '' }} HAS_ACCOUNT_ID: ${{ vars.AVITO_BROWSER_ACCOUNT_ID != '' }} + # A ready endpoint from the owner (secret AVITO_CDP_ENDPOINT) is used as + # is; only without one does the job build its own from the account. + HAS_CDP: ${{ secrets.AVITO_CDP_ENDPOINT != '' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -51,7 +54,7 @@ jobs: missing=0 [ "$HAS_TWOCAPTCHA_KEY" = "true" ] || { echo "::error::secret TWOCAPTCHA_KEY is not set — the canary cannot reach avito.ru"; missing=1; } [ "$HAS_AVITO_PROXY" = "true" ] || { echo "::error::secret AVITO_PROXY (the base Russian gateway login) is not set"; missing=1; } - [ "$HAS_ACCOUNT_ID" = "true" ] || { echo "::error::repository variable AVITO_BROWSER_ACCOUNT_ID is not set"; missing=1; } + [ "$HAS_CDP" = "true" ] || [ "$HAS_ACCOUNT_ID" = "true" ] || { echo "::error::neither secret AVITO_CDP_ENDPOINT nor repository variable AVITO_BROWSER_ACCOUNT_ID is set"; missing=1; } exit $missing - name: Install @@ -59,7 +62,26 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt -r requirements-playwright.txt + - name: Use the given endpoint + if: env.HAS_CDP == 'true' + # Written to a 0600 .env from the environment; never echoed. The + # solver gets AVITO_PROXY as stored: it must be the same exit as the + # endpoint's, or a solved captcha is rejected. + env: + AVITO_CDP_ENDPOINT: ${{ secrets.AVITO_CDP_ENDPOINT }} + AVITO_PROXY: ${{ secrets.AVITO_PROXY }} + run: | + install -m 600 /dev/null .env + python3 - <<'PY2' + import os + with open(".env", "w", encoding="utf-8") as f: + f.write("AVITO_CDP_ENDPOINT=" + os.environ["AVITO_CDP_ENDPOINT"].strip() + "\n") + f.write("AVITO_PROXY=" + os.environ["AVITO_PROXY"].strip() + "\n") + PY2 + echo "using the endpoint from secret AVITO_CDP_ENDPOINT" + - name: Build a fresh endpoint and its matching solver proxy + if: env.HAS_CDP != 'true' # Credentials through the ENVIRONMENT, never argv. The tool writes # AVITO_CDP_ENDPOINT and the minted AVITO_PROXY into a 0600 .env and # prints both masked. From 6491915c0eeea9835636086d7b36f7559bc1293c Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:51:13 +0000 Subject: [PATCH 2/4] Canary: re-point a given endpoint at a fresh Russian session shared with the solver Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CKsd7q25N19WzUZjrAdzx8 --- .github/workflows/canary.yml | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 3c6c3e1..5abd4ca 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -62,23 +62,43 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt -r requirements-playwright.txt - - name: Use the given endpoint + - name: Use the given endpoint, on a fresh Russian session if: env.HAS_CDP == 'true' - # Written to a 0600 .env from the environment; never echoed. The - # solver gets AVITO_PROXY as stored: it must be the same exit as the - # endpoint's, or a solved captcha is rejected. + # The owner's endpoint is re-pointed at a Russian exit: `country-ru`, + # and a `-proxy-` segment carrying a session freshly minted + # on AVITO_PROXY. The solver gets that SAME session, so a solved + # captcha leaves from the browser's IP. Both are written to a 0600 + # .env from the environment and never printed. env: AVITO_CDP_ENDPOINT: ${{ secrets.AVITO_CDP_ENDPOINT }} AVITO_PROXY: ${{ secrets.AVITO_PROXY }} run: | install -m 600 /dev/null .env python3 - <<'PY2' - import os + import base64, os, re, sys + from urllib.parse import urlsplit, urlunsplit, quote, unquote + sys.path.insert(0, "tools"); sys.path.insert(0, ".") + import browser_profile_client as bpc + proxy_url = bpc.minted_proxy(60) # fresh -session-/-sessTime-60 on AVITO_PROXY + ep = urlsplit(os.environ["AVITO_CDP_ENDPOINT"].strip()) + user = unquote(ep.username or "") + if not user or "-zone-" not in user: + sys.exit("!! AVITO_CDP_ENDPOINT has no Scraping Browser login segment") + user = re.sub(r"-proxy-[A-Za-z0-9_=-]+$", "", user) + if re.search(r"-country-[a-z]{2}", user): + user = re.sub(r"-country-[a-z]{2}", "-country-ru", user) + else: + user = user.replace("-zone-scraping_browser", "-zone-scraping_browser-country-ru", 1) + enc = base64.urlsafe_b64encode(proxy_url.encode()).decode().rstrip("=") + user += "-proxy-" + enc + netloc = quote(user, safe="-_.~") + ":" + quote(unquote(ep.password or ""), safe="-_.~") + "@" + ep.hostname + (f":{ep.port}" if ep.port else "") + uri = urlunsplit((ep.scheme, netloc, ep.path, ep.query, ep.fragment)) with open(".env", "w", encoding="utf-8") as f: - f.write("AVITO_CDP_ENDPOINT=" + os.environ["AVITO_CDP_ENDPOINT"].strip() + "\n") - f.write("AVITO_PROXY=" + os.environ["AVITO_PROXY"].strip() + "\n") + f.write("AVITO_CDP_ENDPOINT=" + uri + "\n") + f.write("AVITO_PROXY=" + proxy_url + "\n") + print("endpoint :", bpc.mask_url(uri), "(country-ru, custom proxy segment)") + print("solver proxy :", bpc.mask_url(proxy_url), "(same session)") PY2 - echo "using the endpoint from secret AVITO_CDP_ENDPOINT" - name: Build a fresh endpoint and its matching solver proxy if: env.HAS_CDP != 'true' From 20d2d52539a66fc556ae3c3f0fe4cfa8079cb040 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:53:58 +0000 Subject: [PATCH 3/4] Fix the canary run check crashing on a good run; guard it offline Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CKsd7q25N19WzUZjrAdzx8 --- .github/ci_checks.py | 2 +- .github/workflows/canary.yml | 2 +- smoke_test.py | 39 ++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/ci_checks.py b/.github/ci_checks.py index e6fb5ab..f033cfc 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -175,7 +175,7 @@ def run_check(prefix="sample_output"): failed.append(f"{len(inverted)} row(s) have an original_price at or below their price") if not failed: print(f"ok {len(rows)} rows, status={meta.get('status')}, " - f"{priced} priced, {named} named sellers, " + f"{priced} priced, " f"stop_reason={meta.get('stop_reason')!r}") return failed diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 5abd4ca..cec479e 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -88,7 +88,7 @@ jobs: if re.search(r"-country-[a-z]{2}", user): user = re.sub(r"-country-[a-z]{2}", "-country-ru", user) else: - user = user.replace("-zone-scraping_browser", "-zone-scraping_browser-country-ru", 1) + user = re.sub(r"(-zone-[a-z_]+)", r"\1-country-ru", user, count=1) enc = base64.urlsafe_b64encode(proxy_url.encode()).decode().rstrip("=") user += "-proxy-" + enc netloc = quote(user, safe="-_.~") + ":" + quote(unquote(ep.password or ""), safe="-_.~") + "@" + ep.hostname + (f":{ep.port}" if ep.port else "") diff --git a/smoke_test.py b/smoke_test.py index 973e19e..a9436c2 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2145,6 +2145,45 @@ def test_canary_cannot_go_green_without_scraping(): check("and no longer a seller-name share", "rows name a seller" not in checks) +def test_the_canary_run_check_passes_a_good_run(): + # The canary's run check only executes after a successful live scrape, so + # a crash in its success path stays hidden until the first green run: on + # 2026-10-16 it died with a NameError on a 150-row run that had worked. + # Run it here on a synthetic good run, and on a broken one. + import importlib.util + import tempfile + from output_writer import SOURCE_DEFAULT + spec = importlib.util.spec_from_file_location( + "ci_checks_under_test", os.path.join(HERE, ".github", "ci_checks.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + rows = [{"sku": str(1000 + i), "url": "https://www.avito.ru/moskva/x/%d" % i, + "title": "item %d" % i, "price": 100 + i, "original_price": None, + "region_slug": "moskva", "page": 1 + i // 50, "position": i % 50} + for i in range(150)] + meta = {"status": "complete", "pages_completed": 3, "source": SOURCE_DEFAULT, + "stop_reason": "max_pages"} + with tempfile.TemporaryDirectory() as tmp: + prefix = os.path.join(tmp, "canary") + with open(prefix + ".json", "w", encoding="utf-8") as f: + json.dump(rows, f) + with open(prefix + ".meta.json", "w", encoding="utf-8") as f: + json.dump(meta, f) + try: + failed = mod.run_check(prefix) + except Exception as exc: # noqa: BLE001 + failed = ["raised %s: %s" % (type(exc).__name__, exc)] + check("the canary run check passes a good 150-row run", failed == [], + "(got %r)" % (failed,)) + rows[0]["price"] = None + rows[1]["sku"] = "" + with open(prefix + ".json", "w", encoding="utf-8") as f: + json.dump(rows, f) + failed = mod.run_check(prefix) + check("and fails one with a row missing its sku", + any("no sku" in x for x in failed), "(got %r)" % (failed,)) + + def main(): tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] for test in tests: From cec16fce251429f617115a0d29e2d7bdee035af9 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:55:08 +0000 Subject: [PATCH 4/4] CHANGELOG: canary endpoint and run-check fixes Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CKsd7q25N19WzUZjrAdzx8 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca00f04..0a0719f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,18 @@ entry in a blockquote. ### Fixed +* **The canary's run check crashed on a good run.** Its success message still + named a seller count whose computation had been removed, so the first + complete live scrape (150 rows, 2026-10-16) ended in a `NameError` and a red + job. The message no longer uses it, and the suite now runs the check on a + synthetic good run and on a broken one. +* **The canary can use a ready Scraping Browser endpoint.** With the + `AVITO_CDP_ENDPOINT` secret set, the job uses that endpoint instead of + building one from `AVITO_BROWSER_ACCOUNT_ID`, re-pointed at a Russian exit: + `country-ru` and a `-proxy-` segment carrying a session freshly minted on + `AVITO_PROXY`, which the captcha solver then uses too. Measured 2026-10-16: + 148 rows over 3 pages, complete; the same endpoint without the Russian + proxy did not clear the firewall's proof-of-work. * **`tools/browser_profile_client.py` hid the Browser API's error code.** An HTTP error from the API was reported as the bare status ("404 Client Error: Not Found"), dropping the body that names the cause, e.g.