Scraper API: send waitFor as an object, read the target status from http_code

CHANGELOG.md

@@ -8,6 +8,22 @@ as a CLI toolkit can. A **patch** release means fixes; it does not promise
 that every flag and default is frozen. Where a patch changes behaviour an
 existing user would notice, the release notes lead with it.
 
+## [Unreleased]
+
+### Fixed
+
+- **Scraper API: `waitFor` is now sent as a JSON object.** Measured
+  2026-09-23 against `scraper.2captcha.com/tasks/sync`: the JSON-encoded
+  string this client sent (the form older docs described) is refused with
+  HTTP 422 "params.waitFor must be an object" -- and the task is still
+  billed ($0.0005) -- so every run with `--wait-text` or its sibling wait flags
+  failed with exit 5. The object form answers HTTP 200.
+- **Scraper API: the target page's status is read from `http_code`.** The
+  response's `status` field is the API's own verdict string (`"success"`),
+  not the target site's HTTP code, so a target 403/503 was never seen by
+  this client. `http_code` (an int) is read now, with `status` kept as a
+  fallback only when it is an int.
+
 ## [0.1.1] — 2026-09-18
 
 A pass back over CLAUDE.md before calling the repo finished. Everything here

scraper_api_client.py

@@ -115,21 +115,39 @@ def _redact_debug_header(value: str) -> str:
                                _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value))
 
 
-def _build_wait_for(args) -> Optional[str]:
-    """`waitFor` must be a JSON STRING (double-encoded), per the API docs.
-    Passing a nested object is silently wrong.
+def _build_wait_for(args) -> Optional[dict]:
+    """`waitFor` is sent as a JSON OBJECT. Measured 2026-09-23: the
+    JSON-encoded-string form this client used before is refused with HTTP
+    422 ("params.waitFor must be an object") and still billed; the object
+    form answers 200.
 
     Default (no flag): wait for the DOM. On a challenge-protected page
     that resolves instantly against the challenge page itself — which is
     exactly the trap documented in this module's docstring, so
     --wait-text/--wait-element exist to wait on something only the real
     page can contain."""
     if args.wait_text:
-        return json.dumps({"text": args.wait_text})
+        return {"text": args.wait_text}
     if args.wait_element:
-        return json.dumps({"element": args.wait_element, "checkVisible": True})
+        return {"element": args.wait_element, "checkVisible": True}
     if args.wait_state:
-        return json.dumps({"state": args.wait_state})
+        return {"state": args.wait_state}
+    return None
+
+
+def _target_status(body: dict) -> Optional[int]:
+    """The TARGET page's HTTP status, or None.
+
+    Measured 2026-09-23: the response carries the target's code as
+    `http_code` (an int) and the API's own verdict as `status` ("success").
+    Reading `status` handed the string "success" onward and a target 403
+    was never seen. `status` is kept as a fallback only when it is an int,
+    for the older response shape. bool is excluded: it is an int to Python.
+    """
+    for key in ("http_code", "status"):
+        v = body.get(key)
+        if isinstance(v, int) and not isinstance(v, bool):
+            return v
     return None
 
 
@@ -138,14 +156,14 @@ def fetch_html(args) -> str:
         "task_type": "scrape",
         "url": args.url,
         "data_format": "raw",   # we want HTML; product_parser does the rest
-        "format": "json",       # so we get {"status", "headers", "body"}
+        "format": "json",       # so we get {"status", "http_code", "headers", "body"}
         "timeout": min(args.timeout, MAX_API_TIMEOUT),
     }
 
     wait_for = _build_wait_for(args)
     if wait_for:
         payload["waitFor"] = wait_for
-        logger.info("waitFor: %s", wait_for)
+        logger.info("waitFor: %s", json.dumps(wait_for))
 
     if args.cdp_url:
         payload["cdpurl"] = args.cdp_url
@@ -180,7 +198,7 @@ def fetch_html(args) -> str:
 
     body = resp.json()
     html = body.get("body") or ""
-    upstream_status = body.get("status")
+    upstream_status = _target_status(body)
     logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html))
     # The STATUS is returned alongside the HTML, not thrown away. It used to
     # be, and that cost this engine the family's central distinction. On this

smoke_test.py

@@ -2544,6 +2544,62 @@ def check_x_debug_header_is_redacted():
                  'logger.info("x-debug: %s", _redact_debug_header(debug))' in src)
 
 
+def check_scraper_api_waitfor_object_and_http_code():
+    """Measured 2026-09-23 against the live Scraper API: `waitFor` sent as
+    a JSON-encoded string is refused with HTTP 422 and still billed, and
+    the response's `status` is the API's verdict ("success") while the
+    target's own code is `http_code`. Drives the real fetch_html with
+    requests.post replaced, so no network and no money."""
+    import argparse
+    import logging
+    import scraper_api_client as sac
+    captured = {}
+
+    class _Resp:
+        status_code = 200
+        headers = {}
+        text = ""
+
+        def json(self):
+            return {"status": "success", "http_code": 403, "body": "<html></html>"}
+
+    def _fake_post(url, **kw):
+        captured["json"] = kw.get("json")
+        return _Resp()
+
+    logged = []
+
+    class _H(logging.Handler):
+        def emit(self, record):
+            if str(record.msg).startswith("Upstream page status"):
+                logged.append(record.args[0])
+
+    h = _H()
+    real_post = sac.requests.post
+    sac.requests.post = _fake_post
+    sac.logger.addHandler(h)
+    result = None
+    try:
+        args = argparse.Namespace(url='https://maison.kose.co.jp/site/cosmedecorte/c/c15/', key="k", timeout=60, cdp_url=None,
+                                  wait_text='円', wait_element=None,
+                                  wait_state=None)
+        result = sac.fetch_html(args)
+    finally:
+        sac.requests.post = real_post
+        sac.logger.removeHandler(h)
+    # Where fetch_html returns (html, status) the status handed onward is
+    # the second element; where it returns only the html, the log line is
+    # the only place the status goes.
+    handed = result[1] if isinstance(result, tuple) else (logged[0] if logged else None)
+    wf = (captured.get("json") or {}).get("waitFor")
+    check("Scraper API: --wait-text sends waitFor as an OBJECT, not a JSON string "
+                "(a string is HTTP 422 and still billed, measured 2026-09-23)",
+                isinstance(wf, dict) and wf.get("text") == '円')
+    check("Scraper API: the target status handed onward is http_code (403, an int), "
+                "not the API's own 'success' verdict",
+                type(handed) is int and handed == 403 and logged == [403])
+
+
 
 CHECKS = [v for k, v in sorted(globals().items()) if k.startswith("check_")]