diff --git a/.github/ci_checks.py b/.github/ci_checks.py index e6fb5ab..f033cfc 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -175,7 +175,7 @@ def run_check(prefix="sample_output"): failed.append(f"{len(inverted)} row(s) have an original_price at or below their price") if not failed: print(f"ok {len(rows)} rows, status={meta.get('status')}, " - f"{priced} priced, {named} named sellers, " + f"{priced} priced, " f"stop_reason={meta.get('stop_reason')!r}") return failed diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 9ec70b8..cec479e 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -38,6 +38,9 @@ jobs: HAS_TWOCAPTCHA_KEY: ${{ secrets.TWOCAPTCHA_KEY != '' }} HAS_AVITO_PROXY: ${{ secrets.AVITO_PROXY != '' }} HAS_ACCOUNT_ID: ${{ vars.AVITO_BROWSER_ACCOUNT_ID != '' }} + # A ready endpoint from the owner (secret AVITO_CDP_ENDPOINT) is used as + # is; only without one does the job build its own from the account. + HAS_CDP: ${{ secrets.AVITO_CDP_ENDPOINT != '' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -51,7 +54,7 @@ jobs: missing=0 [ "$HAS_TWOCAPTCHA_KEY" = "true" ] || { echo "::error::secret TWOCAPTCHA_KEY is not set — the canary cannot reach avito.ru"; missing=1; } [ "$HAS_AVITO_PROXY" = "true" ] || { echo "::error::secret AVITO_PROXY (the base Russian gateway login) is not set"; missing=1; } - [ "$HAS_ACCOUNT_ID" = "true" ] || { echo "::error::repository variable AVITO_BROWSER_ACCOUNT_ID is not set"; missing=1; } + [ "$HAS_CDP" = "true" ] || [ "$HAS_ACCOUNT_ID" = "true" ] || { echo "::error::neither secret AVITO_CDP_ENDPOINT nor repository variable AVITO_BROWSER_ACCOUNT_ID is set"; missing=1; } exit $missing - name: Install @@ -59,7 +62,46 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt -r requirements-playwright.txt + - name: Use the given endpoint, on a fresh Russian session + if: env.HAS_CDP == 'true' + # The owner's endpoint is re-pointed at a Russian exit: `country-ru`, + # and a `-proxy-` segment carrying a session freshly minted + # on AVITO_PROXY. The solver gets that SAME session, so a solved + # captcha leaves from the browser's IP. Both are written to a 0600 + # .env from the environment and never printed. + env: + AVITO_CDP_ENDPOINT: ${{ secrets.AVITO_CDP_ENDPOINT }} + AVITO_PROXY: ${{ secrets.AVITO_PROXY }} + run: | + install -m 600 /dev/null .env + python3 - <<'PY2' + import base64, os, re, sys + from urllib.parse import urlsplit, urlunsplit, quote, unquote + sys.path.insert(0, "tools"); sys.path.insert(0, ".") + import browser_profile_client as bpc + proxy_url = bpc.minted_proxy(60) # fresh -session-/-sessTime-60 on AVITO_PROXY + ep = urlsplit(os.environ["AVITO_CDP_ENDPOINT"].strip()) + user = unquote(ep.username or "") + if not user or "-zone-" not in user: + sys.exit("!! AVITO_CDP_ENDPOINT has no Scraping Browser login segment") + user = re.sub(r"-proxy-[A-Za-z0-9_=-]+$", "", user) + if re.search(r"-country-[a-z]{2}", user): + user = re.sub(r"-country-[a-z]{2}", "-country-ru", user) + else: + user = re.sub(r"(-zone-[a-z_]+)", r"\1-country-ru", user, count=1) + enc = base64.urlsafe_b64encode(proxy_url.encode()).decode().rstrip("=") + user += "-proxy-" + enc + netloc = quote(user, safe="-_.~") + ":" + quote(unquote(ep.password or ""), safe="-_.~") + "@" + ep.hostname + (f":{ep.port}" if ep.port else "") + uri = urlunsplit((ep.scheme, netloc, ep.path, ep.query, ep.fragment)) + with open(".env", "w", encoding="utf-8") as f: + f.write("AVITO_CDP_ENDPOINT=" + uri + "\n") + f.write("AVITO_PROXY=" + proxy_url + "\n") + print("endpoint :", bpc.mask_url(uri), "(country-ru, custom proxy segment)") + print("solver proxy :", bpc.mask_url(proxy_url), "(same session)") + PY2 + - name: Build a fresh endpoint and its matching solver proxy + if: env.HAS_CDP != 'true' # Credentials through the ENVIRONMENT, never argv. The tool writes # AVITO_CDP_ENDPOINT and the minted AVITO_PROXY into a 0600 .env and # prints both masked. diff --git a/CHANGELOG.md b/CHANGELOG.md index ca00f04..0a0719f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,18 @@ entry in a blockquote. ### Fixed +* **The canary's run check crashed on a good run.** Its success message still + named a seller count whose computation had been removed, so the first + complete live scrape (150 rows, 2026-10-16) ended in a `NameError` and a red + job. The message no longer uses it, and the suite now runs the check on a + synthetic good run and on a broken one. +* **The canary can use a ready Scraping Browser endpoint.** With the + `AVITO_CDP_ENDPOINT` secret set, the job uses that endpoint instead of + building one from `AVITO_BROWSER_ACCOUNT_ID`, re-pointed at a Russian exit: + `country-ru` and a `-proxy-` segment carrying a session freshly minted on + `AVITO_PROXY`, which the captcha solver then uses too. Measured 2026-10-16: + 148 rows over 3 pages, complete; the same endpoint without the Russian + proxy did not clear the firewall's proof-of-work. * **`tools/browser_profile_client.py` hid the Browser API's error code.** An HTTP error from the API was reported as the bare status ("404 Client Error: Not Found"), dropping the body that names the cause, e.g. diff --git a/smoke_test.py b/smoke_test.py index 973e19e..a9436c2 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2145,6 +2145,45 @@ def test_canary_cannot_go_green_without_scraping(): check("and no longer a seller-name share", "rows name a seller" not in checks) +def test_the_canary_run_check_passes_a_good_run(): + # The canary's run check only executes after a successful live scrape, so + # a crash in its success path stays hidden until the first green run: on + # 2026-10-16 it died with a NameError on a 150-row run that had worked. + # Run it here on a synthetic good run, and on a broken one. + import importlib.util + import tempfile + from output_writer import SOURCE_DEFAULT + spec = importlib.util.spec_from_file_location( + "ci_checks_under_test", os.path.join(HERE, ".github", "ci_checks.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + rows = [{"sku": str(1000 + i), "url": "https://www.avito.ru/moskva/x/%d" % i, + "title": "item %d" % i, "price": 100 + i, "original_price": None, + "region_slug": "moskva", "page": 1 + i // 50, "position": i % 50} + for i in range(150)] + meta = {"status": "complete", "pages_completed": 3, "source": SOURCE_DEFAULT, + "stop_reason": "max_pages"} + with tempfile.TemporaryDirectory() as tmp: + prefix = os.path.join(tmp, "canary") + with open(prefix + ".json", "w", encoding="utf-8") as f: + json.dump(rows, f) + with open(prefix + ".meta.json", "w", encoding="utf-8") as f: + json.dump(meta, f) + try: + failed = mod.run_check(prefix) + except Exception as exc: # noqa: BLE001 + failed = ["raised %s: %s" % (type(exc).__name__, exc)] + check("the canary run check passes a good 150-row run", failed == [], + "(got %r)" % (failed,)) + rows[0]["price"] = None + rows[1]["sku"] = "" + with open(prefix + ".json", "w", encoding="utf-8") as f: + json.dump(rows, f) + failed = mod.run_check(prefix) + check("and fails one with a row missing its sku", + any("no sku" in x for x in failed), "(got %r)" % (failed,)) + + def main(): tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)] for test in tests: