diff --git a/CHANGELOG.md b/CHANGELOG.md index 816558a..17e2fe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,21 @@ with it, so nobody discovers it from a bill or from a diff. --- +## [Unreleased] + +### Fixed + +- **The Scraper API path (`scraper_api_client.py`) failed whenever a wait flag + was given, and never saw the target's status.** Measured 2026-09-23 against + the live `/tasks/sync` endpoint: `waitFor` sent as a JSON-encoded string + (what this client sent) is answered HTTP 422 "params.waitFor must be an + object" and is still billed ($0.0005); sent as an object it is answered + HTTP 200. It is now an object. And the response's `status` is the API's own + verdict ("success"), while the target site's HTTP code is `http_code` — the + client handed `status` onward, so a target 403/503 never reached the page + classifier. It now reads `http_code`, falling back to `status` only if that + is an integer. After the fix, one live call (`--wait-text mug` on the canary's search URL, no `--cdp-url`) was accepted — no 422 — but ran into the API's own sync-wait limit (HTTP 408, task still pending at the default 60 s), so no target status came back from that call to show. + ## [0.2.3] — 2026-09-16 ### Added diff --git a/scraper_api_client.py b/scraper_api_client.py index 7a9c06e..375c18c 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -127,9 +127,13 @@ def _redact_debug_header(value: str) -> str: _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value)) -def _build_wait_for(args) -> Optional[str]: - """`waitFor` must be a JSON STRING (double-encoded), per the API docs. - Passing a nested object is silently wrong. +def _build_wait_for(args) -> Optional[dict]: + """`waitFor` is an OBJECT. Measured 2026-09-23 against the live + /tasks/sync endpoint: the JSON-encoded string form this client used to + send was answered with HTTP 422 ("params.waitFor must be an object") + and was still billed ($0.0005); the same request with an object + answered HTTP 200. The earlier note here, that the API wanted a + double-encoded string, no longer describes the API. Default (no flag): wait for the DOM. On a challenge-protected page that resolves instantly against the challenge page itself — which is @@ -137,11 +141,11 @@ def _build_wait_for(args) -> Optional[str]: --wait-text/--wait-element exist to wait on something only the real page can contain.""" if args.wait_text: - return json.dumps({"text": args.wait_text}) + return {"text": args.wait_text} if args.wait_element: - return json.dumps({"element": args.wait_element, "checkVisible": True}) + return {"element": args.wait_element, "checkVisible": True} if args.wait_state: - return json.dumps({"state": args.wait_state}) + return {"state": args.wait_state} return None @@ -150,14 +154,14 @@ def fetch_html(args) -> str: "task_type": "scrape", "url": args.url, "data_format": "raw", # we want HTML; product_parser does the rest - "format": "json", # so we get {"status", "headers", "body"} + "format": "json", # {"status": verdict, "http_code": target status, "headers", "body"} "timeout": min(args.timeout, MAX_API_TIMEOUT), } wait_for = _build_wait_for(args) if wait_for: payload["waitFor"] = wait_for - logger.info("waitFor: %s", wait_for) + logger.info("waitFor: %s", json.dumps(wait_for)) if args.cdp_url: payload["cdpurl"] = args.cdp_url @@ -192,8 +196,15 @@ def fetch_html(args) -> str: body = resp.json() html = body.get("body") or "" - upstream_status = body.get("status") - logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html)) + # The TARGET's HTTP status is `http_code`. `status` is the API's own + # verdict string ("success"), measured 2026-09-23 -- passing it on + # handed the page classifier a string, so a target 403/503 was never + # seen. Fall back to `status` only if it is itself an integer. + upstream_status = body.get("http_code") + if not isinstance(upstream_status, int): + legacy = body.get("status") + upstream_status = legacy if isinstance(legacy, int) and not isinstance(legacy, bool) else None + logger.info("Upstream page HTTP status %s, %d bytes of HTML.", upstream_status, len(html)) # The STATUS is returned alongside the HTML, not thrown away. It used to # be, and that cost this engine the family's central distinction: a 403 # from Etsy carries a DataDome shell whose `t` field says whether diff --git a/smoke_test.py b/smoke_test.py index e6b6a08..b0fa923 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -4328,6 +4328,49 @@ def test_x_debug_header_is_redacted(): return ok +def test_scraper_api_sends_waitfor_as_an_object_and_reads_http_code(): + """Measured 2026-09-23 against the live Scraper API: a JSON-encoded + STRING waitFor is answered HTTP 422 and still billed, an object is + answered 200; and the target's status is `http_code`, while `status` is + the API's own verdict ("success"). Driven through the real fetch_html + with requests.post stubbed -- no network.""" + group("Scraper API payload and target status") + import types + import scraper_api_client as sac + sent = {} + + class _Resp: + status_code = 200 + headers = {} + text = "" + + def json(self): + return {"status": "success", "http_code": 403, "headers": {}, + "body": ""} + + def _post(url, **kw): + sent.update(kw.get("json") or {}) + return _Resp() + + args = types.SimpleNamespace( + url='https://www.etsy.com/search?q=handmade+mug', key="k" * 8, + timeout=60, cdp_url=None, wait_text='mug', wait_element=None, + wait_state=None) + real_post = sac.requests.post + sac.requests.post = _post + try: + _html, status = sac.fetch_html(args) + finally: + sac.requests.post = real_post + ok = check("Scraper API: --wait-text sends waitFor as an OBJECT, not a JSON " + "string (422 + billed, 2026-09-23) -- got %r" % (sent.get("waitFor"),), + sent.get("waitFor") == {"text": 'mug'}) + ok &= check("Scraper API: the target status handed onward is http_code (403), " + "not the API's 'success' -- got %r" % (status,), + status == 403 and isinstance(status, int)) + return ok + + def main() -> int: ok = True # Checks that could not run because an optional engine library is absent. @@ -4374,6 +4417,7 @@ def main() -> int: ok &= test_shared_calls_bind_against_the_real_signature() ok &= test_sample_output() ok &= test_x_debug_header_is_redacted() + ok &= test_scraper_api_sends_waitfor_as_an_object_and_reads_http_code() print() if _failures: