AWS WAF cookie domain from the site's own list; engine-smoke checks the import

.github/workflows/tests.yml

@@ -141,20 +141,17 @@ jobs:
         run: .venv/bin/pip check
 
       - name: Offline suite, with ${{ matrix.engine }} actually importable
-        # The suite prints "SKIP <engine>_scraper — engine library absent" for
-        # each engine it could not import. In this job that engine IS
-        # installed, so a skip naming it means a real import error rather
-        # than an expected absence — the other two engines are absent here by
-        # design and their skips are fine.
-        #
-        # The pattern is the suite's OWN wording. The copy this repo inherited
-        # grepped for "could not be imported", a phrase the suite never
-        # prints, so this step could not fail. Checked against a real log of
-        # the suite run without the engine, where it DOES match.
+        # The engine this job installed must IMPORT. Checked directly, and
+        # not by grepping the suite's log: the family's copy grepped for a
+        # phrase its suites never printed, so it could not fail (measured
+        # 2026-09-24: 2 of 31 logs carried it), and a grep for this suite's
+        # current wording would break the day that wording changes. Every
+        # engine imports its driver at module level (smoke_test.py asserts
+        # it), so a broken import fails here by name.
         run: |
           set -o pipefail
           .venv/bin/python smoke_test.py | tee smoke.log
-          if grep -q "SKIP ${{ matrix.engine }}_scraper " smoke.log; then
+          if ! .venv/bin/python -c "import ${{ matrix.engine }}_scraper"; then
             echo "FAILED: ${{ matrix.engine }}_scraper failed to import even though"
             echo "        ${{ matrix.engine }} is installed in this job."
             exit 1

page_flow.py

@@ -29,6 +29,7 @@
 """
 
 import logging
+import re
 from dataclasses import dataclass, field
 from typing import Callable, List, Optional, Tuple
 
@@ -321,15 +322,39 @@ def finish(args, query: Query, outcomes: List, stop_reason: str,
                "query": query_summary(query)})
 
 
-def cookie_domain(host: Optional[str]) -> str:
-    """The domain an `aws-waf-token` cookie is set on: the registrable one.
+_WAF_COOKIE_DOMAINS_RE = re.compile(r"awsWafCookieDomainList\s*=\s*\[([^\]]*)\]")
 
-    `.binance.com` for www.binance.com and p2p.binance.com alike. The site's
-    own WAF integration keeps the token there, and a copy scoped to
-    www.binance.com alone did not clear the CAPTCHA (2026-09-24).
+
+def cookie_domain(host: Optional[str], html: Optional[str] = None) -> str:
+    """The domain an `aws-waf-token` cookie is set on: the one the SITE says.
+
+    AWS WAF's own integration names it on the challenge page, in
+    `window.awsWafCookieDomainList`, and the token cookie goes on the listed
+    domain that covers the page's host, or on the host itself when the list
+    is empty. Measured 2026-09-24:
+
+        binance.com         ['binance.com','binance.bh', ...]  -> .binance.com
+        transfermarkt.com   []                                 -> the host
+
+    On binance, a token scoped to www.binance.com alone left the page on
+    "Human Verification", and the same token on .binance.com was served. So
+    "the registrable domain" was binance's list, not a rule, and a site with
+    an empty list wants the host.
     """
-    parts = (host or "www.binance.com").lower().split(".")
-    return "." + ".".join(parts[-2:]) if len(parts) >= 2 else (host or "")
+    host = (host or "www.binance.com").lower()
+    listed = None
+    if html:
+        m = _WAF_COOKIE_DOMAINS_RE.search(html)
+        if m:
+            listed = [d.strip().strip("'\"").lower().lstrip(".")
+                      for d in m.group(1).split(",") if d.strip().strip("'\"")]
+    if listed is None:
+        # No page to read it from: this site's list, as measured.
+        listed = ["binance.com"]
+    for d in listed:
+        if host == d or host.endswith("." + d):
+            return "." + d
+    return host
 
 
 # ---------------------------------------------------------------------------

playwright_scraper.py

@@ -398,7 +398,7 @@ def handle_captcha_if_present(page, args, proxy: Optional[str] = None) -> bool:
         logger.error("Solving the AWS WAF CAPTCHA failed (%s) — continuing.",
                      _mask_credentials(str(e)))
         return False
-    domain = page_flow.cookie_domain(urlparse(page.url).hostname)
+    domain = page_flow.cookie_domain(urlparse(page.url).hostname, html)
     # AWS WAF reads its answer back from a COOKIE, not a form field. Set on
     # the context so the reload and every fetch() after it carry it, and on
     # the REGISTRABLE domain, which is where the site's own WAF integration

puppeteer_scraper.py

@@ -443,7 +443,7 @@ def handle_captcha_if_present(ops, args, proxy: Optional[str] = None) -> bool:
         logger.error("Solving the AWS WAF CAPTCHA failed (%s) — continuing.",
                      _mask_credentials(str(e)))
         return False
-    domain = page_flow.cookie_domain(urlparse(ops.page.url).hostname)
+    domain = page_flow.cookie_domain(urlparse(ops.page.url).hostname, html)
     ops.bridge.run(ops.page.setCookie({"name": AWS_WAF_COOKIE, "value": token,
                                        "domain": domain, "path": "/"}))
     logger.info("Set %s for %s — reloading to let the WAF re-check.",

selenium_scraper.py

@@ -335,7 +335,7 @@ def handle_captcha_if_present(ops, args, proxy: Optional[str] = None) -> bool:
         logger.error("Solving the AWS WAF CAPTCHA failed (%s) — continuing.",
                      _mask_credentials(str(e)))
         return False
-    domain = page_flow.cookie_domain(urlparse(url).hostname)
+    domain = page_flow.cookie_domain(urlparse(url).hostname, html)
     try:
         ops.driver.add_cookie({"name": AWS_WAF_COOKIE, "value": token,
                                "domain": domain, "path": "/"})

smoke_test.py

@@ -453,9 +453,17 @@ def check_state_policy():
           "expired" in F.cdp_connect_hint("WebSocket error: 401 Unauthorized")
           and "pid" not in F.cdp_connect_hint("401 Unauthorized"))
     check("...and a 500 as a held pid", "pid" in F.cdp_connect_hint("HTTP 500"))
-    equal("the aws-waf-token goes on the REGISTRABLE domain",
-          (F.cookie_domain("www.binance.com"), F.cookie_domain("p2p.binance.com")),
+    waf = fx("waf_captcha_chromium")
+    check("the captured CAPTCHA page names its cookie domains (not vacuous)",
+          "awsWafCookieDomainList" in waf)
+    equal("the aws-waf-token goes on the domain the SITE lists",
+          (F.cookie_domain("www.binance.com", waf), F.cookie_domain("p2p.binance.com", waf)),
           (".binance.com", ".binance.com"))
+    equal("an EMPTY list means the page host (transfermarkt's, 2026-09-24)",
+          F.cookie_domain("www.transfermarkt.com", "awsWafCookieDomainList = [];"),
+          "www.transfermarkt.com")
+    equal("a host the list does not cover gets the host",
+          F.cookie_domain("www.example.org", waf), "www.example.org")
 
 
 def check_policy_constants_have_a_consumer():