Scraper API: send waitFor as an object, read the target status from http_code

CHANGELOG.md

@@ -11,6 +11,21 @@ with it, so nobody discovers it from a bill or from a diff.
 
 ---
 
+## [Unreleased]
+
+### Fixed
+
+- **The Scraper API path (`scraper_api_client.py`) failed whenever a wait flag
+  was given, and never saw the target's status.** Measured 2026-09-23 against
+  the live `/tasks/sync` endpoint: `waitFor` sent as a JSON-encoded string
+  (what this client sent) is answered HTTP 422 "params.waitFor must be an
+  object" and is still billed ($0.0005); sent as an object it is answered
+  HTTP 200. It is now an object. And the response's `status` is the API's own
+  verdict ("success"), while the target site's HTTP code is `http_code` — the
+  client handed `status` onward, so a target 403/503 never reached the page
+  classifier. It now reads `http_code`, falling back to `status` only if that
+  is an integer. After the fix, one live call (`--wait-text mug` on the canary's search URL, no `--cdp-url`) was accepted — no 422 — but ran into the API's own sync-wait limit (HTTP 408, task still pending at the default 60 s), so no target status came back from that call to show.
+
 ## [0.2.3] — 2026-09-16
 
 ### Added

scraper_api_client.py

@@ -127,21 +127,25 @@ def _redact_debug_header(value: str) -> str:
                                _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value))
 
 
-def _build_wait_for(args) -> Optional[str]:
-    """`waitFor` must be a JSON STRING (double-encoded), per the API docs.
-    Passing a nested object is silently wrong.
+def _build_wait_for(args) -> Optional[dict]:
+    """`waitFor` is an OBJECT. Measured 2026-09-23 against the live
+    /tasks/sync endpoint: the JSON-encoded string form this client used to
+    send was answered with HTTP 422 ("params.waitFor must be an object")
+    and was still billed ($0.0005); the same request with an object
+    answered HTTP 200. The earlier note here, that the API wanted a
+    double-encoded string, no longer describes the API.
 
     Default (no flag): wait for the DOM. On a challenge-protected page
     that resolves instantly against the challenge page itself — which is
     exactly the trap documented in this module's docstring, so
     --wait-text/--wait-element exist to wait on something only the real
     page can contain."""
     if args.wait_text:
-        return json.dumps({"text": args.wait_text})
+        return {"text": args.wait_text}
     if args.wait_element:
-        return json.dumps({"element": args.wait_element, "checkVisible": True})
+        return {"element": args.wait_element, "checkVisible": True}
     if args.wait_state:
-        return json.dumps({"state": args.wait_state})
+        return {"state": args.wait_state}
     return None
 
 
@@ -150,14 +154,14 @@ def fetch_html(args) -> str:
         "task_type": "scrape",
         "url": args.url,
         "data_format": "raw",   # we want HTML; product_parser does the rest
-        "format": "json",       # so we get {"status", "headers", "body"}
+        "format": "json",       # {"status": verdict, "http_code": target status, "headers", "body"}
         "timeout": min(args.timeout, MAX_API_TIMEOUT),
     }
 
     wait_for = _build_wait_for(args)
     if wait_for:
         payload["waitFor"] = wait_for
-        logger.info("waitFor: %s", wait_for)
+        logger.info("waitFor: %s", json.dumps(wait_for))
 
     if args.cdp_url:
         payload["cdpurl"] = args.cdp_url
@@ -192,8 +196,15 @@ def fetch_html(args) -> str:
 
     body = resp.json()
     html = body.get("body") or ""
-    upstream_status = body.get("status")
-    logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html))
+    # The TARGET's HTTP status is `http_code`. `status` is the API's own
+    # verdict string ("success"), measured 2026-09-23 -- passing it on
+    # handed the page classifier a string, so a target 403/503 was never
+    # seen. Fall back to `status` only if it is itself an integer.
+    upstream_status = body.get("http_code")
+    if not isinstance(upstream_status, int):
+        legacy = body.get("status")
+        upstream_status = legacy if isinstance(legacy, int) and not isinstance(legacy, bool) else None
+    logger.info("Upstream page HTTP status %s, %d bytes of HTML.", upstream_status, len(html))
     # The STATUS is returned alongside the HTML, not thrown away. It used to
     # be, and that cost this engine the family's central distinction: a 403
     # from Etsy carries a DataDome shell whose `t` field says whether

smoke_test.py

@@ -4328,6 +4328,49 @@ def test_x_debug_header_is_redacted():
     return ok
 
 
+def test_scraper_api_sends_waitfor_as_an_object_and_reads_http_code():
+    """Measured 2026-09-23 against the live Scraper API: a JSON-encoded
+    STRING waitFor is answered HTTP 422 and still billed, an object is
+    answered 200; and the target's status is `http_code`, while `status` is
+    the API's own verdict ("success"). Driven through the real fetch_html
+    with requests.post stubbed -- no network."""
+    group("Scraper API payload and target status")
+    import types
+    import scraper_api_client as sac
+    sent = {}
+
+    class _Resp:
+        status_code = 200
+        headers = {}
+        text = ""
+
+        def json(self):
+            return {"status": "success", "http_code": 403, "headers": {},
+                    "body": "<html></html>"}
+
+    def _post(url, **kw):
+        sent.update(kw.get("json") or {})
+        return _Resp()
+
+    args = types.SimpleNamespace(
+        url='https://www.etsy.com/search?q=handmade+mug', key="k" * 8,
+        timeout=60, cdp_url=None, wait_text='mug', wait_element=None,
+        wait_state=None)
+    real_post = sac.requests.post
+    sac.requests.post = _post
+    try:
+        _html, status = sac.fetch_html(args)
+    finally:
+        sac.requests.post = real_post
+    ok = check("Scraper API: --wait-text sends waitFor as an OBJECT, not a JSON "
+               "string (422 + billed, 2026-09-23) -- got %r" % (sent.get("waitFor"),),
+               sent.get("waitFor") == {"text": 'mug'})
+    ok &= check("Scraper API: the target status handed onward is http_code (403), "
+                "not the API's 'success' -- got %r" % (status,),
+                status == 403 and isinstance(status, int))
+    return ok
+
+
 def main() -> int:
     ok = True
     # Checks that could not run because an optional engine library is absent.
@@ -4374,6 +4417,7 @@ def main() -> int:
     ok &= test_shared_calls_bind_against_the_real_signature()
     ok &= test_sample_output()
     ok &= test_x_debug_header_is_redacted()
+    ok &= test_scraper_api_sends_waitfor_as_an_object_and_reads_http_code()
 
     print()
     if _failures: