From c236d98896514e7bc56b7086810bd9a5324ffe0a Mon Sep 17 00:00:00 2001 From: petr Date: Thu, 24 Sep 2026 14:18:48 +0000 Subject: [PATCH] AWS WAF cookie domain from the site's own list; engine-smoke checks the import - page_flow.cookie_domain reads window.awsWafCookieDomainList from the challenge page: binance lists binance.com (-> .binance.com, as measured), transfermarkt lists nothing (-> the page host). 'The registrable domain' was binance's list, not a rule. Live: solved, token on .binance.com, page served (pyppeteer, 29 s). - tests.yml engine-smoke imports the installed engine instead of grepping the suite's wording, as 25 sibling repos now do. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests.yml | 19 ++++++++---------- page_flow.py | 39 ++++++++++++++++++++++++++++++------- playwright_scraper.py | 2 +- puppeteer_scraper.py | 2 +- selenium_scraper.py | 2 +- smoke_test.py | 12 ++++++++++-- 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fe43acd..24894d6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -141,20 +141,17 @@ jobs: run: .venv/bin/pip check - name: Offline suite, with ${{ matrix.engine }} actually importable - # The suite prints "SKIP _scraper — engine library absent" for - # each engine it could not import. In this job that engine IS - # installed, so a skip naming it means a real import error rather - # than an expected absence — the other two engines are absent here by - # design and their skips are fine. - # - # The pattern is the suite's OWN wording. The copy this repo inherited - # grepped for "could not be imported", a phrase the suite never - # prints, so this step could not fail. Checked against a real log of - # the suite run without the engine, where it DOES match. + # The engine this job installed must IMPORT. Checked directly, and + # not by grepping the suite's log: the family's copy grepped for a + # phrase its suites never printed, so it could not fail (measured + # 2026-09-24: 2 of 31 logs carried it), and a grep for this suite's + # current wording would break the day that wording changes. Every + # engine imports its driver at module level (smoke_test.py asserts + # it), so a broken import fails here by name. run: | set -o pipefail .venv/bin/python smoke_test.py | tee smoke.log - if grep -q "SKIP ${{ matrix.engine }}_scraper " smoke.log; then + if ! .venv/bin/python -c "import ${{ matrix.engine }}_scraper"; then echo "FAILED: ${{ matrix.engine }}_scraper failed to import even though" echo " ${{ matrix.engine }} is installed in this job." exit 1 diff --git a/page_flow.py b/page_flow.py index 66097dd..1d381a8 100644 --- a/page_flow.py +++ b/page_flow.py @@ -29,6 +29,7 @@ """ import logging +import re from dataclasses import dataclass, field from typing import Callable, List, Optional, Tuple @@ -321,15 +322,39 @@ def finish(args, query: Query, outcomes: List, stop_reason: str, "query": query_summary(query)}) -def cookie_domain(host: Optional[str]) -> str: - """The domain an `aws-waf-token` cookie is set on: the registrable one. +_WAF_COOKIE_DOMAINS_RE = re.compile(r"awsWafCookieDomainList\s*=\s*\[([^\]]*)\]") - `.binance.com` for www.binance.com and p2p.binance.com alike. The site's - own WAF integration keeps the token there, and a copy scoped to - www.binance.com alone did not clear the CAPTCHA (2026-09-24). + +def cookie_domain(host: Optional[str], html: Optional[str] = None) -> str: + """The domain an `aws-waf-token` cookie is set on: the one the SITE says. + + AWS WAF's own integration names it on the challenge page, in + `window.awsWafCookieDomainList`, and the token cookie goes on the listed + domain that covers the page's host, or on the host itself when the list + is empty. Measured 2026-09-24: + + binance.com ['binance.com','binance.bh', ...] -> .binance.com + transfermarkt.com [] -> the host + + On binance, a token scoped to www.binance.com alone left the page on + "Human Verification", and the same token on .binance.com was served. So + "the registrable domain" was binance's list, not a rule, and a site with + an empty list wants the host. """ - parts = (host or "www.binance.com").lower().split(".") - return "." + ".".join(parts[-2:]) if len(parts) >= 2 else (host or "") + host = (host or "www.binance.com").lower() + listed = None + if html: + m = _WAF_COOKIE_DOMAINS_RE.search(html) + if m: + listed = [d.strip().strip("'\"").lower().lstrip(".") + for d in m.group(1).split(",") if d.strip().strip("'\"")] + if listed is None: + # No page to read it from: this site's list, as measured. + listed = ["binance.com"] + for d in listed: + if host == d or host.endswith("." + d): + return "." + d + return host # --------------------------------------------------------------------------- diff --git a/playwright_scraper.py b/playwright_scraper.py index 0003e57..94ad992 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -398,7 +398,7 @@ def handle_captcha_if_present(page, args, proxy: Optional[str] = None) -> bool: logger.error("Solving the AWS WAF CAPTCHA failed (%s) — continuing.", _mask_credentials(str(e))) return False - domain = page_flow.cookie_domain(urlparse(page.url).hostname) + domain = page_flow.cookie_domain(urlparse(page.url).hostname, html) # AWS WAF reads its answer back from a COOKIE, not a form field. Set on # the context so the reload and every fetch() after it carry it, and on # the REGISTRABLE domain, which is where the site's own WAF integration diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 129da07..ff3330e 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -443,7 +443,7 @@ def handle_captcha_if_present(ops, args, proxy: Optional[str] = None) -> bool: logger.error("Solving the AWS WAF CAPTCHA failed (%s) — continuing.", _mask_credentials(str(e))) return False - domain = page_flow.cookie_domain(urlparse(ops.page.url).hostname) + domain = page_flow.cookie_domain(urlparse(ops.page.url).hostname, html) ops.bridge.run(ops.page.setCookie({"name": AWS_WAF_COOKIE, "value": token, "domain": domain, "path": "/"})) logger.info("Set %s for %s — reloading to let the WAF re-check.", diff --git a/selenium_scraper.py b/selenium_scraper.py index e1bb903..990d067 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -335,7 +335,7 @@ def handle_captcha_if_present(ops, args, proxy: Optional[str] = None) -> bool: logger.error("Solving the AWS WAF CAPTCHA failed (%s) — continuing.", _mask_credentials(str(e))) return False - domain = page_flow.cookie_domain(urlparse(url).hostname) + domain = page_flow.cookie_domain(urlparse(url).hostname, html) try: ops.driver.add_cookie({"name": AWS_WAF_COOKIE, "value": token, "domain": domain, "path": "/"}) diff --git a/smoke_test.py b/smoke_test.py index fd592f9..e31b99f 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -453,9 +453,17 @@ def check_state_policy(): "expired" in F.cdp_connect_hint("WebSocket error: 401 Unauthorized") and "pid" not in F.cdp_connect_hint("401 Unauthorized")) check("...and a 500 as a held pid", "pid" in F.cdp_connect_hint("HTTP 500")) - equal("the aws-waf-token goes on the REGISTRABLE domain", - (F.cookie_domain("www.binance.com"), F.cookie_domain("p2p.binance.com")), + waf = fx("waf_captcha_chromium") + check("the captured CAPTCHA page names its cookie domains (not vacuous)", + "awsWafCookieDomainList" in waf) + equal("the aws-waf-token goes on the domain the SITE lists", + (F.cookie_domain("www.binance.com", waf), F.cookie_domain("p2p.binance.com", waf)), (".binance.com", ".binance.com")) + equal("an EMPTY list means the page host (transfermarkt's, 2026-09-24)", + F.cookie_domain("www.transfermarkt.com", "awsWafCookieDomainList = [];"), + "www.transfermarkt.com") + equal("a host the list does not cover gets the host", + F.cookie_domain("www.example.org", waf), "www.example.org") def check_policy_constants_have_a_consumer():