Scraper API: send waitFor as an object, read the target status from http_code

CHANGELOG.md

@@ -39,6 +39,23 @@ from a row count, it leads the section in a blockquote.
   no fixtures. CI has built it on every PR from the start; this is the first
   time it was built and RUN by hand (§16).
 
+### Fixed
+
+- **The Scraper API path sent `waitFor` in a form the live API rejects,
+  and read the wrong field as the target's status.** Measured 2026-09-23
+  against `scraper.2captcha.com/tasks/sync`: `waitFor` sent as a
+  JSON-encoded string (what this client built for every `--wait-*` flag)
+  is answered HTTP 422 "params.waitFor must be an object" and is still
+  billed ($0.0005); the same request with an object gets HTTP 200. It is
+  now an object. And the response's `status` is the API's own verdict
+  ("success"), not the target site's HTTP code, which is `http_code` —
+  so a target 403 or 503 never reached the page classifier. The target
+  status is now read from `http_code` (falling back to `status` only if
+  that is an integer). Pinned by an offline check that drives the real
+  client with `requests.post` stubbed; verified by control (red with the
+  old client).
+
+
 ## [0.1.1] — 2026-09-16
 
 An audit of v0.1.0 against the family's own checklist, done by MUTATION —

scraper_api_client.py

@@ -123,21 +123,22 @@ def _redact_debug_header(value: str) -> str:
                                _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value))
 
 
-def _build_wait_for(args) -> Optional[str]:
-    """`waitFor` must be a JSON STRING (double-encoded), per the API docs.
-    Passing a nested object is silently wrong.
+def _build_wait_for(args) -> Optional[dict]:
+    """`waitFor` is sent as an OBJECT. Measured 2026-09-23: the
+    JSON-encoded string form is answered with HTTP 422 and still billed
+    ($0.0005); the object form gets HTTP 200.
 
     Default (no flag): wait for the DOM. On a challenge-protected page
     that resolves instantly against the challenge page itself — which is
     exactly the trap documented in this module's docstring, so
     --wait-text/--wait-element exist to wait on something only the real
     page can contain."""
     if args.wait_text:
-        return json.dumps({"text": args.wait_text})
+        return {"text": args.wait_text}
     if args.wait_element:
-        return json.dumps({"element": args.wait_element, "checkVisible": True})
+        return {"element": args.wait_element, "checkVisible": True}
     if args.wait_state:
-        return json.dumps({"state": args.wait_state})
+        return {"state": args.wait_state}
     return None
 
 
@@ -146,14 +147,14 @@ def fetch_html(args) -> str:
         "task_type": "scrape",
         "url": args.url,
         "data_format": "raw",   # we want HTML; product_parser does the rest
-        "format": "json",       # so we get {"status", "headers", "body"}
+        "format": "json",       # so we get {"status", "http_code", "headers", "body"}
         "timeout": min(args.timeout, MAX_API_TIMEOUT),
     }
 
     wait_for = _build_wait_for(args)
     if wait_for:
         payload["waitFor"] = wait_for
-        logger.info("waitFor: %s", wait_for)
+        logger.info("waitFor: %s", json.dumps(wait_for))
 
     if args.cdp_url:
         payload["cdpurl"] = args.cdp_url
@@ -188,8 +189,16 @@ def fetch_html(args) -> str:
 
     body = resp.json()
     html = body.get("body") or ""
-    upstream_status = body.get("status")
-    logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html))
+    # The TARGET's HTTP code is `http_code`. `status` is the API's own
+    # verdict string ("success", measured 2026-09-23), and passing it on
+    # meant the page classifier never saw a 403. Fall back to `status` only
+    # if some response shape carries an int there.
+    upstream_status = body.get("http_code")
+    if not isinstance(upstream_status, int):
+        legacy = body.get("status")
+        upstream_status = legacy if isinstance(legacy, int) else None
+    logger.info("Upstream page HTTP %s (API status %r), %d bytes of HTML.",
+                upstream_status, body.get("status"), len(html))
     # The STATUS is returned alongside the HTML, not thrown away. It used to
     # be, and that cost this engine the family's central distinction. On this
     # site a refusal carries no markup at all — nothing a challenge check

smoke_test.py

@@ -2614,6 +2614,50 @@ def test_x_debug_header_is_redacted():
                       'logger.info("x-debug: %s", _redact_debug_header(debug))' in src)
     return ok
 
+def test_scraper_api_payload_and_status():
+    """Measured 2026-09-23 against the live Scraper API: `waitFor` sent as
+    a JSON-encoded string is answered HTTP 422 and still billed, and the
+    response's `status` is the API's own "success" while the target's code
+    is `http_code`. Drive the real fetch_html with requests.post stubbed:
+    no network, no key spent."""
+    try:
+        import scraper_api_client as sac
+    except ImportError:
+        return False
+
+    captured = {}
+
+    class _Resp:
+        status_code = 200
+        headers = {}
+        text = ""
+
+        def json(self):
+            return {"status": "success", "http_code": 403,
+                    "headers": {}, "body": "<html></html>"}
+
+    def fake_post(url, **kw):
+        captured.update(kw.get("json") or {})
+        return _Resp()
+
+    real_post, real_argv = sac.requests.post, sys.argv
+    sac.requests.post = fake_post
+    sys.argv = ["scraper_api_client.py", "--key", "k" * 8,
+                "--url", 'https://polymarket.com/predictions/crypto',
+                "--wait-text", 'Crypto']
+    try:
+        _html, status = sac.fetch_html(sac.parse_args())
+    finally:
+        sac.requests.post, sys.argv = real_post, real_argv
+
+    ok = True
+    ok &= check("scraper API: waitFor is sent as an object, not a JSON string",
+                captured.get("waitFor") == {"text": 'Crypto'})
+    ok &= check("scraper API: the target status comes from http_code (403), "
+                "not the API's own 'success'", status == 403)
+    return ok
+
+
 
 def main() -> int:
     ok = True
@@ -2657,6 +2701,7 @@ def main() -> int:
     ok &= test_required_files_are_committed()
     ok &= test_readme_claims()
     ok &= test_x_debug_header_is_redacted()
+    ok &= test_scraper_api_payload_and_status()
 
     passed = _total_checks - len(_failures)
     if passed < CLAIMED_CHECK_FLOOR: