Scraper API: send waitFor as an object, read the target status from http_code

CHANGELOG.md

@@ -10,6 +10,20 @@ does it leads the release notes rather than being discovered from a bill.
 
 ## [Unreleased]
 
+### Fixed
+- **The Scraper API engine failed on every `--wait-text` / `--wait-element` /
+  `--wait-state` call, and was billed for it.** It sent `waitFor` as a
+  JSON-encoded string; measured 2026-09-23 the live API answers that with
+  HTTP 422 "params.waitFor must be an object" and still charges $0.0005,
+  while the same request with an object is answered 200. It is now sent as
+  an object. **And the target site's status was never seen:** the client
+  read the response's `status`, which is the API's own verdict string
+  ("success"), instead of `http_code`, the target's HTTP status — so a
+  target 403/503 reached the page classifier as "success". It now reads
+  `http_code` (falling back to `status` only if that is an int). A
+  regression check drives the real `fetch_html` with `requests.post`
+  stubbed.
+
 ## [0.1.1] — 2026-09-17
 
 An audit against this family's own checklist, section by section, executing

scraper_api_client.py

@@ -110,21 +110,26 @@ def _redact_debug_header(value: str) -> str:
                                _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value))
 
 
-def _build_wait_for(args) -> Optional[str]:
-    """`waitFor` must be a JSON STRING (double-encoded), per the API docs.
-    Passing a nested object is silently wrong.
+def _build_wait_for(args) -> Optional[dict]:
+    """`waitFor` is sent as a JSON OBJECT.
+
+    Measured 2026-09-23 against /tasks/sync: the JSON-encoded STRING form
+    this client used to send (on the strength of the API docs of the time)
+    is now answered HTTP 422, "params.waitFor must be an object" -- and the
+    task is still billed ($0.0005). The same request with an object is
+    answered HTTP 200.
 
     Default (no flag): wait for the DOM. On a challenge-protected page
     that resolves instantly against the challenge page itself — which is
     exactly the trap documented in this module's docstring, so
     --wait-text/--wait-element exist to wait on something only the real
     page can contain."""
     if args.wait_text:
-        return json.dumps({"text": args.wait_text})
+        return {"text": args.wait_text}
     if args.wait_element:
-        return json.dumps({"element": args.wait_element, "checkVisible": True})
+        return {"element": args.wait_element, "checkVisible": True}
     if args.wait_state:
-        return json.dumps({"state": args.wait_state})
+        return {"state": args.wait_state}
     return None
 
 
@@ -140,7 +145,7 @@ def fetch_html(args) -> str:
     wait_for = _build_wait_for(args)
     if wait_for:
         payload["waitFor"] = wait_for
-        logger.info("waitFor: %s", wait_for)
+        logger.info("waitFor: %s", json.dumps(wait_for))
 
     if args.cdp_url:
         payload["cdpurl"] = args.cdp_url
@@ -175,8 +180,16 @@ def fetch_html(args) -> str:
 
     body = resp.json()
     html = body.get("body") or ""
-    upstream_status = body.get("status")
-    logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html))
+    # The TARGET's HTTP status is `http_code` (an int). `status` is the
+    # API's own verdict STRING ("success"), which this line used to read --
+    # so a target 403 or 503 reached the classifier as "success" and was
+    # never seen (measured 2026-09-23). `status` is kept as a fallback only
+    # when it really is an int.
+    upstream_status = body.get("http_code")
+    if not isinstance(upstream_status, int):
+        raw = body.get("status")
+        upstream_status = raw if isinstance(raw, int) and not isinstance(raw, bool) else None
+    logger.info("Upstream page status (http_code) %s, %d bytes of HTML.", upstream_status, len(html))
     # The STATUS is returned alongside the HTML, not thrown away. It used to
     # be, and that cost this engine the family's central distinction. On this
     # site a refusal carries no markup at all — nothing a challenge check

smoke_test.py

@@ -1988,6 +1988,48 @@ def check_x_debug_header_is_redacted():
 
 
 
+def check_scraper_api_waitfor_is_object_and_status_is_http_code():
+    """Measured 2026-09-23 against the live Scraper API: a `waitFor` sent as
+    a JSON-encoded STRING is answered HTTP 422 ("params.waitFor must be an
+    object") and still billed; an object is answered 200. And the target's
+    status is `http_code` -- `status` is the API's own "success", which must
+    never be what reaches the page classifier. Driven through the real
+    fetch_html with requests.post stubbed, so no network and no key."""
+    import argparse
+    import scraper_api_client as sac
+    captured = {}
+
+    class _Resp:
+        status_code = 200
+        headers = {}
+        text = ""
+
+        def json(self):
+            return {"status": "success", "http_code": 403, "body": "<html></html>"}
+
+    def _post(url, headers=None, json=None, timeout=None, **_kw):  # noqa: A002
+        captured["payload"] = json
+        return _Resp()
+
+    real_post = sac.requests.post
+    sac.requests.post = _post
+    try:
+        args = argparse.Namespace(url='https://wellfound.com/role/python-developer', key="k", timeout=60,
+                                  cdp_url=None, wait_text='Python',
+                                  wait_element=None, wait_state=None)
+        result = sac.fetch_html(args)
+    finally:
+        sac.requests.post = real_post
+    status = result[1] if isinstance(result, tuple) else None
+    wf = (captured.get("payload") or {}).get("waitFor")
+    check(f"Scraper API: --wait-text must send waitFor as an OBJECT (a string "
+          f"is HTTP 422 and still billed), got {wf!r}",
+          isinstance(wf, dict) and wf.get("text") == 'Python')
+    check(f"Scraper API: the status handed onward must be the target's "
+          f"http_code 403 (int), not the API's own verdict, got {status!r}",
+          status == 403 and isinstance(status, int))
+
+
 CHECKS = [v for k, v in sorted(globals().items()) if k.startswith("check_")]