From 416a3e4e3970b9d71c5c87079c8f8373677f7280 Mon Sep 17 00:00:00 2001 From: Petr Date: Wed, 23 Sep 2026 15:17:00 +0000 Subject: [PATCH] Scraper API: send waitFor as an object, read the target status from http_code Measured 2026-09-23 against the live /tasks/sync endpoint: waitFor sent as a JSON-encoded string is answered HTTP 422 "params.waitFor must be an object" and still billed ($0.0005); as an object it is answered 200. The response's `status` is the API's own verdict ("success"); the target's HTTP code is `http_code`, which is now what reaches the page classifier (fallback to `status` only when it is an int). Adds one offline regression check, controlled: with the old client the suite goes red on exactly that check. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 15 +++++++++++++++ scraper_api_client.py | 31 ++++++++++++++++++++---------- smoke_test.py | 44 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faffc62..c9494fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,21 @@ as a CLI toolkit can. A **patch** release means fixes — it does not mean every flag and every default is frozen. Where a patch changes behaviour an existing user would notice, the release notes say so first. +## [Unreleased] + +### Fixed + +- **The Scraper API path (`scraper_api_client.py`) failed whenever a wait flag + was given, and never saw the target's status.** Measured 2026-09-23 against + the live `/tasks/sync` endpoint: `waitFor` sent as a JSON-encoded string + (what this client sent) is answered HTTP 422 "params.waitFor must be an + object" and is still billed ($0.0005); sent as an object it is answered + HTTP 200. It is now an object. And the response's `status` is the API's own + verdict ("success"), while the target site's HTTP code is `http_code` — the + client handed `status` onward, so a target 403/503 never reached the page + classifier. It now reads `http_code`, falling back to `status` only if that + is an integer. After the fix, one live call (`--wait-text AED` on the canary's used-cars listing, no `--cdp-url`) answered HTTP 200 (no 422) with an integer upstream status, 200; the body was a 1,156-byte Incapsula interstitial, correctly reported as blocked (exit 3), as the module docstring already says for the API's own exit. + ## [0.1.1] — 2026-09-16 ### Fixed diff --git a/scraper_api_client.py b/scraper_api_client.py index 7230ebe..c4ba0b7 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -120,9 +120,13 @@ def _redact_debug_header(value: str) -> str: _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value)) -def _build_wait_for(args) -> Optional[str]: - """`waitFor` must be a JSON STRING (double-encoded), per the API docs. - Passing a nested object is silently wrong. +def _build_wait_for(args) -> Optional[dict]: + """`waitFor` is an OBJECT. Measured 2026-09-23 against the live + /tasks/sync endpoint: the JSON-encoded string form this client used to + send was answered with HTTP 422 ("params.waitFor must be an object") + and was still billed ($0.0005); the same request with an object + answered HTTP 200. The earlier note here, that the API wanted a + double-encoded string, no longer describes the API. Default (no flag): wait for the DOM. On a challenge-protected page that resolves instantly against the challenge page itself — which is @@ -130,11 +134,11 @@ def _build_wait_for(args) -> Optional[str]: --wait-text/--wait-element exist to wait on something only the real page can contain.""" if args.wait_text: - return json.dumps({"text": args.wait_text}) + return {"text": args.wait_text} if args.wait_element: - return json.dumps({"element": args.wait_element, "checkVisible": True}) + return {"element": args.wait_element, "checkVisible": True} if args.wait_state: - return json.dumps({"state": args.wait_state}) + return {"state": args.wait_state} return None @@ -143,14 +147,14 @@ def fetch_html(args) -> str: "task_type": "scrape", "url": args.url, "data_format": "raw", # we want HTML; product_parser does the rest - "format": "json", # so we get {"status", "headers", "body"} + "format": "json", # {"status": verdict, "http_code": target status, "headers", "body"} "timeout": min(args.timeout, MAX_API_TIMEOUT), } wait_for = _build_wait_for(args) if wait_for: payload["waitFor"] = wait_for - logger.info("waitFor: %s", wait_for) + logger.info("waitFor: %s", json.dumps(wait_for)) if args.cdp_url: payload["cdpurl"] = args.cdp_url @@ -185,8 +189,15 @@ def fetch_html(args) -> str: body = resp.json() html = body.get("body") or "" - upstream_status = body.get("status") - logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html)) + # The TARGET's HTTP status is `http_code`. `status` is the API's own + # verdict string ("success"), measured 2026-09-23 -- passing it on + # handed the page classifier a string, so a target 403/503 was never + # seen. Fall back to `status` only if it is itself an integer. + upstream_status = body.get("http_code") + if not isinstance(upstream_status, int): + legacy = body.get("status") + upstream_status = legacy if isinstance(legacy, int) and not isinstance(legacy, bool) else None + logger.info("Upstream page HTTP status %s, %d bytes of HTML.", upstream_status, len(html)) # The STATUS is returned alongside the HTML, not thrown away. It used to # be, and that cost this engine the family's central distinction. On this # site a refusal carries no markup at all — nothing a challenge check diff --git a/smoke_test.py b/smoke_test.py index 3c5e711..7918614 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2969,6 +2969,49 @@ def test_parse_is_gated_on_the_policy(): return ok +def test_scraper_api_sends_waitfor_as_an_object_and_reads_http_code(): + """Measured 2026-09-23 against the live Scraper API: a JSON-encoded + STRING waitFor is answered HTTP 422 and still billed, an object is + answered 200; and the target's status is `http_code`, while `status` is + the API's own verdict ("success"). Driven through the real fetch_html + with requests.post stubbed -- no network.""" + group("Scraper API payload and target status") + import types + import scraper_api_client as sac + sent = {} + + class _Resp: + status_code = 200 + headers = {} + text = "" + + def json(self): + return {"status": "success", "http_code": 403, "headers": {}, + "body": ""} + + def _post(url, **kw): + sent.update(kw.get("json") or {}) + return _Resp() + + args = types.SimpleNamespace( + url='https://uae.dubizzle.com/motors/used-cars/', key="k" * 8, + timeout=60, cdp_url=None, wait_text='AED', wait_element=None, + wait_state=None) + real_post = sac.requests.post + sac.requests.post = _post + try: + _html, status = sac.fetch_html(args) + finally: + sac.requests.post = real_post + ok = check("Scraper API: --wait-text sends waitFor as an OBJECT, not a JSON " + "string (422 + billed, 2026-09-23) -- got %r" % (sent.get("waitFor"),), + sent.get("waitFor") == {"text": 'AED'}) + ok &= check("Scraper API: the target status handed onward is http_code (403), " + "not the API's 'success' -- got %r" % (status,), + status == 403 and isinstance(status, int)) + return ok + + def main() -> int: ok = True # Checks that could not run because an optional engine library is absent. @@ -3010,6 +3053,7 @@ def main() -> int: ok &= test_public_names_have_consumers() ok &= test_log_format_strings_match_their_args() ok &= test_x_debug_header_is_redacted() + ok &= test_scraper_api_sends_waitfor_as_an_object_and_reads_http_code() print() if _failures: