diff --git a/CHANGELOG.md b/CHANGELOG.md index d53060f..fba5d04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,20 @@ does it leads the release notes rather than being discovered from a bill. ## [Unreleased] +### Fixed +- **The Scraper API engine failed on every `--wait-text` / `--wait-element` / + `--wait-state` call, and was billed for it.** It sent `waitFor` as a + JSON-encoded string; measured 2026-09-23 the live API answers that with + HTTP 422 "params.waitFor must be an object" and still charges $0.0005, + while the same request with an object is answered 200. It is now sent as + an object. **And the target site's status was never seen:** the client + read the response's `status`, which is the API's own verdict string + ("success"), instead of `http_code`, the target's HTTP status — so a + target 403/503 reached the page classifier as "success". It now reads + `http_code` (falling back to `status` only if that is an int). A + regression check drives the real `fetch_html` with `requests.post` + stubbed. + ## [0.1.1] — 2026-09-17 An audit against this family's own checklist, section by section, executing diff --git a/scraper_api_client.py b/scraper_api_client.py index 1ca3369..fb0a30e 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -110,9 +110,14 @@ def _redact_debug_header(value: str) -> str: _CREDS_IN_TEXT_RE.sub(r"\1***:***@", value)) -def _build_wait_for(args) -> Optional[str]: - """`waitFor` must be a JSON STRING (double-encoded), per the API docs. - Passing a nested object is silently wrong. +def _build_wait_for(args) -> Optional[dict]: + """`waitFor` is sent as a JSON OBJECT. + + Measured 2026-09-23 against /tasks/sync: the JSON-encoded STRING form + this client used to send (on the strength of the API docs of the time) + is now answered HTTP 422, "params.waitFor must be an object" -- and the + task is still billed ($0.0005). The same request with an object is + answered HTTP 200. Default (no flag): wait for the DOM. On a challenge-protected page that resolves instantly against the challenge page itself — which is @@ -120,11 +125,11 @@ def _build_wait_for(args) -> Optional[str]: --wait-text/--wait-element exist to wait on something only the real page can contain.""" if args.wait_text: - return json.dumps({"text": args.wait_text}) + return {"text": args.wait_text} if args.wait_element: - return json.dumps({"element": args.wait_element, "checkVisible": True}) + return {"element": args.wait_element, "checkVisible": True} if args.wait_state: - return json.dumps({"state": args.wait_state}) + return {"state": args.wait_state} return None @@ -140,7 +145,7 @@ def fetch_html(args) -> str: wait_for = _build_wait_for(args) if wait_for: payload["waitFor"] = wait_for - logger.info("waitFor: %s", wait_for) + logger.info("waitFor: %s", json.dumps(wait_for)) if args.cdp_url: payload["cdpurl"] = args.cdp_url @@ -175,8 +180,16 @@ def fetch_html(args) -> str: body = resp.json() html = body.get("body") or "" - upstream_status = body.get("status") - logger.info("Upstream page status %s, %d bytes of HTML.", upstream_status, len(html)) + # The TARGET's HTTP status is `http_code` (an int). `status` is the + # API's own verdict STRING ("success"), which this line used to read -- + # so a target 403 or 503 reached the classifier as "success" and was + # never seen (measured 2026-09-23). `status` is kept as a fallback only + # when it really is an int. + upstream_status = body.get("http_code") + if not isinstance(upstream_status, int): + raw = body.get("status") + upstream_status = raw if isinstance(raw, int) and not isinstance(raw, bool) else None + logger.info("Upstream page status (http_code) %s, %d bytes of HTML.", upstream_status, len(html)) # The STATUS is returned alongside the HTML, not thrown away. It used to # be, and that cost this engine the family's central distinction. On this # site a refusal carries no markup at all — nothing a challenge check diff --git a/smoke_test.py b/smoke_test.py index 7a310d0..785cd36 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1988,6 +1988,48 @@ def check_x_debug_header_is_redacted(): +def check_scraper_api_waitfor_is_object_and_status_is_http_code(): + """Measured 2026-09-23 against the live Scraper API: a `waitFor` sent as + a JSON-encoded STRING is answered HTTP 422 ("params.waitFor must be an + object") and still billed; an object is answered 200. And the target's + status is `http_code` -- `status` is the API's own "success", which must + never be what reaches the page classifier. Driven through the real + fetch_html with requests.post stubbed, so no network and no key.""" + import argparse + import scraper_api_client as sac + captured = {} + + class _Resp: + status_code = 200 + headers = {} + text = "" + + def json(self): + return {"status": "success", "http_code": 403, "body": ""} + + def _post(url, headers=None, json=None, timeout=None, **_kw): # noqa: A002 + captured["payload"] = json + return _Resp() + + real_post = sac.requests.post + sac.requests.post = _post + try: + args = argparse.Namespace(url='https://wellfound.com/role/python-developer', key="k", timeout=60, + cdp_url=None, wait_text='Python', + wait_element=None, wait_state=None) + result = sac.fetch_html(args) + finally: + sac.requests.post = real_post + status = result[1] if isinstance(result, tuple) else None + wf = (captured.get("payload") or {}).get("waitFor") + check(f"Scraper API: --wait-text must send waitFor as an OBJECT (a string " + f"is HTTP 422 and still billed), got {wf!r}", + isinstance(wf, dict) and wf.get("text") == 'Python') + check(f"Scraper API: the status handed onward must be the target's " + f"http_code 403 (int), not the API's own verdict, got {status!r}", + status == 403 and isinstance(status, int)) + + CHECKS = [v for k, v in sorted(globals().items()) if k.startswith("check_")]