diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ef34e0..10fa659 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,23 @@ from a row count, it leads the section in a blockquote. no fixtures. CI has built it on every PR from the start; this is the first time it was built and RUN by hand (§16). +### Fixed + +- **The Scraper API path sent `waitFor` in a form the live API rejects, + and read the wrong field as the target's status.** Measured 2026-09-23 + against `scraper.2captcha.com/tasks/sync`: `waitFor` sent as a + JSON-encoded string (what this client built for every `--wait-*` flag) + is answered HTTP 422 "params.waitFor must be an object" and is still + billed ($0.0005); the same request with an object gets HTTP 200. It is + now an object. And the response's `status` is the API's own verdict + ("success"), not the target site's HTTP code, which is `http_code` — + so a target 403 or 503 never reached the page classifier. The target + status is now read from `http_code` (falling back to `status` only if + that is an integer). Pinned by an offline check that drives the real + client with `requests.post` stubbed; verified by control (red with the + old client). + + ## [0.1.1] — 2026-09-16 An audit of v0.1.0 against the family's own checklist, done by MUTATION — diff --git a/scraper_api_client.py b/scraper_api_client.py index 3d5d1ca..a4063b3 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -123,9 +123,10 @@ def _redact_debug_header(value: str) -> str: _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value)) -def _build_wait_for(args) -> Optional[str]: - """`waitFor` must be a JSON STRING (double-encoded), per the API docs. - Passing a nested object is silently wrong. +def _build_wait_for(args) -> Optional[dict]: + """`waitFor` is sent as an OBJECT. Measured 2026-09-23: the + JSON-encoded string form is answered with HTTP 422 and still billed + ($0.0005); the object form gets HTTP 200. Default (no flag): wait for the DOM. On a challenge-protected page that resolves instantly against the challenge page itself — which is @@ -133,11 +134,11 @@ def _build_wait_for(args) -> Optional[str]: --wait-text/--wait-element exist to wait on something only the real page can contain.""" if args.wait_text: - return json.dumps({"text": args.wait_text}) + return {"text": args.wait_text} if args.wait_element: - return json.dumps({"element": args.wait_element, "checkVisible": True}) + return {"element": args.wait_element, "checkVisible": True} if args.wait_state: - return json.dumps({"state": args.wait_state}) + return {"state": args.wait_state} return None @@ -146,14 +147,14 @@ def fetch_html(args) -> str: "task_type": "scrape", "url": args.url, "data_format": "raw", # we want HTML; product_parser does the rest - "format": "json", # so we get {"status", "headers", "body"} + "format": "json", # so we get {"status", "http_code", "headers", "body"} "timeout": min(args.timeout, MAX_API_TIMEOUT), } wait_for = _build_wait_for(args) if wait_for: payload["waitFor"] = wait_for - logger.info("waitFor: %s", wait_for) + logger.info("waitFor: %s", json.dumps(wait_for)) if args.cdp_url: payload["cdpurl"] = args.cdp_url @@ -188,8 +189,16 @@ def fetch_html(args) -> str: body = resp.json() html = body.get("body") or "" - upstream_status = body.get("status") - logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html)) + # The TARGET's HTTP code is `http_code`. `status` is the API's own + # verdict string ("success", measured 2026-09-23), and passing it on + # meant the page classifier never saw a 403. Fall back to `status` only + # if some response shape carries an int there. + upstream_status = body.get("http_code") + if not isinstance(upstream_status, int): + legacy = body.get("status") + upstream_status = legacy if isinstance(legacy, int) else None + logger.info("Upstream page HTTP %s (API status %r), %d bytes of HTML.", + upstream_status, body.get("status"), len(html)) # The STATUS is returned alongside the HTML, not thrown away. It used to # be, and that cost this engine the family's central distinction. On this # site a refusal carries no markup at all — nothing a challenge check diff --git a/smoke_test.py b/smoke_test.py index c3b0435..cecfd54 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2614,6 +2614,50 @@ def test_x_debug_header_is_redacted(): 'logger.info("x-debug: %s", _redact_debug_header(debug))' in src) return ok +def test_scraper_api_payload_and_status(): + """Measured 2026-09-23 against the live Scraper API: `waitFor` sent as + a JSON-encoded string is answered HTTP 422 and still billed, and the + response's `status` is the API's own "success" while the target's code + is `http_code`. Drive the real fetch_html with requests.post stubbed: + no network, no key spent.""" + try: + import scraper_api_client as sac + except ImportError: + return False + + captured = {} + + class _Resp: + status_code = 200 + headers = {} + text = "" + + def json(self): + return {"status": "success", "http_code": 403, + "headers": {}, "body": ""} + + def fake_post(url, **kw): + captured.update(kw.get("json") or {}) + return _Resp() + + real_post, real_argv = sac.requests.post, sys.argv + sac.requests.post = fake_post + sys.argv = ["scraper_api_client.py", "--key", "k" * 8, + "--url", 'https://polymarket.com/predictions/crypto', + "--wait-text", 'Crypto'] + try: + _html, status = sac.fetch_html(sac.parse_args()) + finally: + sac.requests.post, sys.argv = real_post, real_argv + + ok = True + ok &= check("scraper API: waitFor is sent as an object, not a JSON string", + captured.get("waitFor") == {"text": 'Crypto'}) + ok &= check("scraper API: the target status comes from http_code (403), " + "not the API's own 'success'", status == 403) + return ok + + def main() -> int: ok = True @@ -2657,6 +2701,7 @@ def main() -> int: ok &= test_required_files_are_committed() ok &= test_readme_claims() ok &= test_x_debug_header_is_redacted() + ok &= test_scraper_api_payload_and_status() passed = _total_checks - len(_failures) if passed < CLAIMED_CHECK_FLOOR: