diff --git a/CHANGELOG.md b/CHANGELOG.md index a8b7395..52f2c58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,22 @@ as a CLI toolkit can. A **patch** release means fixes; it does not promise that every flag and default is frozen. Where a patch changes behaviour an existing user would notice, the release notes lead with it. +## [Unreleased] + +### Fixed + +- **Scraper API: `waitFor` is now sent as a JSON object.** Measured + 2026-09-23 against `scraper.2captcha.com/tasks/sync`: the JSON-encoded + string this client sent (the form older docs described) is refused with + HTTP 422 "params.waitFor must be an object" -- and the task is still + billed ($0.0005) -- so every run with `--wait-text` or its sibling wait flags + failed with exit 5. The object form answers HTTP 200. +- **Scraper API: the target page's status is read from `http_code`.** The + response's `status` field is the API's own verdict string (`"success"`), + not the target site's HTTP code, so a target 403/503 was never seen by + this client. `http_code` (an int) is read now, with `status` kept as a + fallback only when it is an int. + ## [0.1.1] — 2026-09-18 A pass back over CLAUDE.md before calling the repo finished. Everything here diff --git a/scraper_api_client.py b/scraper_api_client.py index ac4a471..3d12a19 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -115,9 +115,11 @@ def _redact_debug_header(value: str) -> str: _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value)) -def _build_wait_for(args) -> Optional[str]: - """`waitFor` must be a JSON STRING (double-encoded), per the API docs. - Passing a nested object is silently wrong. +def _build_wait_for(args) -> Optional[dict]: + """`waitFor` is sent as a JSON OBJECT. Measured 2026-09-23: the + JSON-encoded-string form this client used before is refused with HTTP + 422 ("params.waitFor must be an object") and still billed; the object + form answers 200. Default (no flag): wait for the DOM. On a challenge-protected page that resolves instantly against the challenge page itself — which is @@ -125,11 +127,27 @@ def _build_wait_for(args) -> Optional[str]: --wait-text/--wait-element exist to wait on something only the real page can contain.""" if args.wait_text: - return json.dumps({"text": args.wait_text}) + return {"text": args.wait_text} if args.wait_element: - return json.dumps({"element": args.wait_element, "checkVisible": True}) + return {"element": args.wait_element, "checkVisible": True} if args.wait_state: - return json.dumps({"state": args.wait_state}) + return {"state": args.wait_state} + return None + + +def _target_status(body: dict) -> Optional[int]: + """The TARGET page's HTTP status, or None. + + Measured 2026-09-23: the response carries the target's code as + `http_code` (an int) and the API's own verdict as `status` ("success"). + Reading `status` handed the string "success" onward and a target 403 + was never seen. `status` is kept as a fallback only when it is an int, + for the older response shape. bool is excluded: it is an int to Python. + """ + for key in ("http_code", "status"): + v = body.get(key) + if isinstance(v, int) and not isinstance(v, bool): + return v return None @@ -138,14 +156,14 @@ def fetch_html(args) -> str: "task_type": "scrape", "url": args.url, "data_format": "raw", # we want HTML; product_parser does the rest - "format": "json", # so we get {"status", "headers", "body"} + "format": "json", # so we get {"status", "http_code", "headers", "body"} "timeout": min(args.timeout, MAX_API_TIMEOUT), } wait_for = _build_wait_for(args) if wait_for: payload["waitFor"] = wait_for - logger.info("waitFor: %s", wait_for) + logger.info("waitFor: %s", json.dumps(wait_for)) if args.cdp_url: payload["cdpurl"] = args.cdp_url @@ -180,7 +198,7 @@ def fetch_html(args) -> str: body = resp.json() html = body.get("body") or "" - upstream_status = body.get("status") + upstream_status = _target_status(body) logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html)) # The STATUS is returned alongside the HTML, not thrown away. It used to # be, and that cost this engine the family's central distinction. On this diff --git a/smoke_test.py b/smoke_test.py index b788a94..8693c36 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2544,6 +2544,62 @@ def check_x_debug_header_is_redacted(): 'logger.info("x-debug: %s", _redact_debug_header(debug))' in src) +def check_scraper_api_waitfor_object_and_http_code(): + """Measured 2026-09-23 against the live Scraper API: `waitFor` sent as + a JSON-encoded string is refused with HTTP 422 and still billed, and + the response's `status` is the API's verdict ("success") while the + target's own code is `http_code`. Drives the real fetch_html with + requests.post replaced, so no network and no money.""" + import argparse + import logging + import scraper_api_client as sac + captured = {} + + class _Resp: + status_code = 200 + headers = {} + text = "" + + def json(self): + return {"status": "success", "http_code": 403, "body": ""} + + def _fake_post(url, **kw): + captured["json"] = kw.get("json") + return _Resp() + + logged = [] + + class _H(logging.Handler): + def emit(self, record): + if str(record.msg).startswith("Upstream page status"): + logged.append(record.args[0]) + + h = _H() + real_post = sac.requests.post + sac.requests.post = _fake_post + sac.logger.addHandler(h) + result = None + try: + args = argparse.Namespace(url='https://maison.kose.co.jp/site/cosmedecorte/c/c15/', key="k", timeout=60, cdp_url=None, + wait_text='円', wait_element=None, + wait_state=None) + result = sac.fetch_html(args) + finally: + sac.requests.post = real_post + sac.logger.removeHandler(h) + # Where fetch_html returns (html, status) the status handed onward is + # the second element; where it returns only the html, the log line is + # the only place the status goes. + handed = result[1] if isinstance(result, tuple) else (logged[0] if logged else None) + wf = (captured.get("json") or {}).get("waitFor") + check("Scraper API: --wait-text sends waitFor as an OBJECT, not a JSON string " + "(a string is HTTP 422 and still billed, measured 2026-09-23)", + isinstance(wf, dict) and wf.get("text") == '円') + check("Scraper API: the target status handed onward is http_code (403, an int), " + "not the API's own 'success' verdict", + type(handed) is int and handed == 403 and logged == [403]) + + CHECKS = [v for k, v in sorted(globals().items()) if k.startswith("check_")]