diff --git a/skills/web-use/SKILL.md b/skills/web-use/SKILL.md index a8c4166..efee6d0 100644 --- a/skills/web-use/SKILL.md +++ b/skills/web-use/SKILL.md @@ -1,6 +1,6 @@ --- name: web-use -description: "Use for any task that touches a web page or web data. Routes between web_search/web_fetch, Browserless/TinyFish/protected extraction, OpenClaw browser, and live browser context for login, 2FA, CAPTCHA, current tabs, carts, or checkout. Domain skills keep site-specific policy." +description: "Use for any task that touches a web page or web data. Routes between web_search/web_fetch, Browserless/TinyFish/protected extraction, OpenClaw browser, and live browser context for login, 2FA, CAPTCHA, current tabs, or account state. Domain skills keep site-specific policy." --- # Web Use @@ -23,7 +23,7 @@ Keep those questions separate even when one workflow needs both. | Read a simple public URL | `web_fetch` | | Inspect a JS-rendered page visually | OpenClaw `browser` | | Extract protected or bot-gated data | `references/extraction-backends.md` | -| Use a current tab, login, 2FA, CAPTCHA, extension, cart, or checkout | `references/context-device.md` | +| Use a current tab, login, 2FA, CAPTCHA, extension, or account state | `references/context-device.md` | | Site-specific paid API or domain policy | Relevant domain skill | Do not use a browser when search or fetch is enough. Do not use Browserless, @@ -87,7 +87,7 @@ Use browser context routing when the task needs one of these: - preserving or using an existing logged-in tab/session - a visible browser on the user's device - a browser extension lane -- CAPTCHA, 2FA, manual review, cart, checkout, or account state +- CAPTCHA, 2FA, manual review, or account state - user-facing UX/design for browser intents Plain-English labels: @@ -130,9 +130,10 @@ and phrasing guide. Use these from this skill directory when repeatable execution helps: -- `scripts/browserless_extract.py` - Browserless `content`, `unblock`, or `stealth-bql` +- `scripts/browserless_extract.py` - Browserless `content`, `unblock`, or `stealth-bql` using stdlib HTTP; reads `BROWSERLESS_TOKEN` or legacy `BROWSERLESS_API_KEY` +- `scripts/browserless_media_requests.py` - Browserless `/function` network media discovery for rendered pages where media URLs appear only after playback/rendering - `scripts/browserless_session.py` - opt-in persistent Browserless session helper with redacted output and 0600 session files -- `scripts/tinyfish_browser_extract.py` - TinyFish Browser API / CDP extraction helper +- `scripts/tinyfish_browser_extract.py` - optional TinyFish Browser API / CDP extraction helper See `references/backends.md.example` for credential and command templates. @@ -141,3 +142,7 @@ See `references/backends.md.example` for credential and command templates. - `references/context-device.md` - browser context/device/session matrix - `references/extraction-backends.md` - backend ladder and safety notes - `references/backends.md.example` - public-safe credential and command template + +## Baseline Checks + +Run `bash scripts/test.sh` after editing this skill. diff --git a/skills/web-use/references/backends.md.example b/skills/web-use/references/backends.md.example index 3d8a2c7..bb16585 100644 --- a/skills/web-use/references/backends.md.example +++ b/skills/web-use/references/backends.md.example @@ -13,7 +13,7 @@ Copy this file to `backends.md` and fill in the env var names or 1Password refer | Browserless (`/session`) | opt-in persistent protected extraction session | No | use sparingly only when cookies/localStorage/sessionStorage/cache must persist across repeated same-site BQL/CDP calls | | TinyFish Browser API (CDP session) | remote stealth browser primitive for brittle or multi-step flows | Secondary / situational | works on protected pages when used as a browser session | | Site-specific structured data API | clean structured fields for one site | No | domain skill owns credential, cost policy, and when it is worth a call | -| OpenClaw browser | interactive human-visible browsing | No | use for login, 2FA, manual review, checkout | +| OpenClaw browser | interactive human-visible browsing | No | use for login, 2FA, manual review, or account state | --- @@ -33,6 +33,9 @@ export BROWSERLESS_TOKEN="..." export TINYFISH_API_KEY="..." ``` +`BROWSERLESS_API_KEY` is accepted as a legacy fallback by the local helper +scripts, but new config should use `BROWSERLESS_TOKEN`. + --- ## Policy @@ -96,6 +99,18 @@ python3 skills/web-use/scripts/browserless_session.py stop \ Do not paste the session file contents, `browserQL`, `connect`, or `stop` URLs into chat or logs. They include the Browserless token and should be treated as bearer credentials. +### Browserless media-request helper + +```bash +BROWSERLESS_TOKEN="" \ + python3 skills/web-use/scripts/browserless_media_requests.py \ + +``` + +Use this when a page exposes media only after rendering or playback, such as a +public social post/Reel where `yt-dlp` failed. Returned CDN URLs are temporary; +store source URL, method, fetch time, and summarized media metadata by default. + ### TinyFish helper ```bash diff --git a/skills/web-use/references/extraction-backends.md b/skills/web-use/references/extraction-backends.md index fa630de..b10ddfd 100644 --- a/skills/web-use/references/extraction-backends.md +++ b/skills/web-use/references/extraction-backends.md @@ -72,7 +72,7 @@ session: - 2FA - CAPTCHA/manual solve - visual confirmation -- final shopping steps +- user-confirmed account actions ## Protected Page Escalation @@ -87,6 +87,7 @@ When doing repeatable operational work, prefer the bundled helpers before rewriting one-off curl/CDP glue: - `../scripts/browserless_extract.py` +- `../scripts/browserless_media_requests.py` - `../scripts/browserless_session.py` - `../scripts/tinyfish_browser_extract.py` diff --git a/skills/web-use/scripts/browserless_extract.py b/skills/web-use/scripts/browserless_extract.py index ebca44b..9a277f9 100755 --- a/skills/web-use/scripts/browserless_extract.py +++ b/skills/web-use/scripts/browserless_extract.py @@ -2,12 +2,12 @@ """Fetch a page through Browserless and return normalized extraction JSON. Examples: - BROWSERLESS_TOKEN="" \ - python3 skills/web-use/scripts/browserless_extract.py \ + BROWSERLESS_TOKEN="" \ + python3 scripts/browserless_extract.py \ https://example.com/protected-page - BROWSERLESS_TOKEN="" \ - python3 skills/web-use/scripts/browserless_extract.py \ + BROWSERLESS_TOKEN="" \ + python3 scripts/browserless_extract.py \ https://example.com/protected-page \ --mode unblock """ @@ -21,14 +21,41 @@ import sys from html import unescape from typing import Any - -import requests +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen DEFAULT_HOST = "https://production-sfo.browserless.io" DEFAULT_TIMEOUT = 120 DEFAULT_SNIPPET_CHARS = 3000 +class BrowserlessRequestError(RuntimeError): + """Raised when Browserless returns a non-2xx response.""" + + +def post_browserless(url: str, payload: dict[str, Any], timeout: int) -> tuple[int, str]: + body = json.dumps(payload).encode("utf-8") + request = Request( + url, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=timeout) as response: + return response.status, response.read().decode("utf-8", errors="replace") + except HTTPError as exc: + error_body = exc.read().decode("utf-8", errors="replace") + raise BrowserlessRequestError(f"HTTP {exc.code}: {error_body[:1000]}") from exc + except URLError as exc: + raise BrowserlessRequestError(str(exc.reason)) from exc + + +def browserless_url(host: str, path: str, token: str) -> str: + return f"{host}{path}?{urlencode({'token': token})}" + + def clean_text(text: str) -> str: return re.sub(r"\s+", " ", unescape(text or "")).strip() @@ -77,21 +104,18 @@ def sanitize_error(error: Exception, token: str | None) -> str: def request_content(token: str, url: str, host: str, timeout: int) -> dict[str, Any]: - response = requests.post( - f"{host}/content", - params={"token": token}, - json={"url": url}, - timeout=timeout, + status, html = post_browserless( + browserless_url(host, "/content", token), + {"url": url}, + timeout, ) - response.raise_for_status() - html = response.text title = extract_title_from_html(html) body = strip_html(html) return { "provider": "browserless", "mode": "content", "url": url, - "status_code": response.status_code, + "status_code": status, "title": title, "body": body, **signals(title, body), @@ -99,14 +123,12 @@ def request_content(token: str, url: str, host: str, timeout: int) -> dict[str, def request_unblock(token: str, url: str, host: str, timeout: int) -> dict[str, Any]: - response = requests.post( - f"{host}/unblock", - params={"token": token}, - json={"url": url, "browserWSEndpoint": False}, - timeout=timeout, + status, response_text = post_browserless( + browserless_url(host, "/unblock", token), + {"url": url, "browserWSEndpoint": False}, + timeout, ) - response.raise_for_status() - payload = response.json() + payload = json.loads(response_text) html = payload.get("content") or "" title = extract_title_from_html(html) body = strip_html(html) @@ -114,7 +136,7 @@ def request_unblock(token: str, url: str, host: str, timeout: int) -> dict[str, "provider": "browserless", "mode": "unblock", "url": url, - "status_code": response.status_code, + "status_code": status, "title": title, "body": body, "browser_ws_endpoint": payload.get("browserWSEndpoint"), @@ -132,14 +154,12 @@ def request_stealth_bql(token: str, url: str, host: str, timeout: int, solve: bo 'body: text(selector: "body") { text } ' "}" ) - response = requests.post( - f"{host}/stealth/bql", - params={"token": token}, - json={"query": query, "variables": {"url": url}}, - timeout=timeout, + _, response_text = post_browserless( + browserless_url(host, "/stealth/bql", token), + {"query": query, "variables": {"url": url}}, + timeout, ) - response.raise_for_status() - payload = response.json() + payload = json.loads(response_text) if payload.get("errors"): raise RuntimeError(json.dumps(payload["errors"], indent=2)) data = payload.get("data") or {} @@ -158,10 +178,7 @@ def request_stealth_bql(token: str, url: str, host: str, timeout: int, solve: bo def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("url", help="Target URL") parser.add_argument( "--mode", @@ -170,7 +187,11 @@ def parse_args() -> argparse.Namespace: help="Browserless surface to use", ) parser.add_argument("--host", default=DEFAULT_HOST, help="Browserless host") - parser.add_argument("--token", default=os.environ.get("BROWSERLESS_TOKEN"), help="Browserless API token") + parser.add_argument( + "--token", + default=os.environ.get("BROWSERLESS_TOKEN") or os.environ.get("BROWSERLESS_API_KEY"), + help="Browserless API token", + ) parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, help="Request timeout in seconds") parser.add_argument( "--snippet-chars", diff --git a/skills/web-use/scripts/browserless_media_requests.py b/skills/web-use/scripts/browserless_media_requests.py new file mode 100755 index 0000000..2d64065 --- /dev/null +++ b/skills/web-use/scripts/browserless_media_requests.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Discover media network requests from a rendered page via Browserless /function. + +This is intentionally generic web-use plumbing. It does not decide what kind of +social/video workflow should happen next; callers such as the video skill consume +the normalized JSON. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +DEFAULT_HOST = "https://production-sfo.browserless.io" +DEFAULT_TIMEOUT = 120 +DEFAULT_WAIT_MS = 6000 +DEFAULT_GOTO_TIMEOUT_MS = 90000 + + +def post_json(url: str, payload: dict[str, Any], timeout: int) -> tuple[int, str]: + body = json.dumps(payload).encode("utf-8") + request = Request( + url, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=timeout) as response: + return response.status, response.read().decode("utf-8", errors="replace") + except HTTPError as exc: + error_body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Browserless HTTP {exc.code}: {error_body[:1000]}") from exc + except URLError as exc: + raise RuntimeError(f"Browserless request failed: {exc.reason}") from exc + + +def browserless_url(host: str, path: str, token: str) -> str: + return f"{host}{path}?{urlencode({'token': token})}" + + +FUNCTION_CODE = r""" +export default async function({ page, context }) { + const seen = new Map(); + const mediaTypes = /video|audio|mpegurl|mp4|webm|m3u8/i; + const mediaUrl = /\.(mp4|m4v|webm|mov|m3u8|mp3|m4a)(\?|$)/i; + const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + + const remember = (url, source, headers = {}) => { + if (!url || seen.has(url)) return; + const contentType = headers['content-type'] || headers['Content-Type'] || ''; + if (!mediaTypes.test(contentType) && !mediaUrl.test(url)) return; + seen.set(url, { url, source, content_type: contentType || null }); + }; + + page.on('response', async response => { + try { + remember(response.url(), 'response', response.headers()); + } catch (err) {} + }); + + await page.goto(context.url, { waitUntil: 'networkidle2', timeout: context.gotoTimeoutMs || 90000 }); + + if (context.play) { + await page.evaluate(async () => { + for (const video of Array.from(document.querySelectorAll('video'))) { + try { video.muted = true; await video.play(); } catch (err) {} + } + for (const button of Array.from(document.querySelectorAll('button, [role="button"]')).slice(0, 20)) { + const label = `${button.ariaLabel || ''} ${button.textContent || ''}`.toLowerCase(); + if (/play|watch|reel|video/.test(label)) { + try { button.click(); } catch (err) {} + } + } + }); + } + + await sleep(context.waitMs || 6000); + + const domMedia = await page.evaluate(() => Array.from(document.querySelectorAll('video, audio, source')) + .map(el => el.currentSrc || el.src) + .filter(Boolean)); + for (const url of domMedia) remember(url, 'dom'); + + const title = await page.title().catch(() => ''); + return { + ok: true, + url: context.url, + title, + candidates: Array.from(seen.values()), + candidate_count: seen.size + }; +} +""" + + +def parse_browserless_function_response(text: str) -> dict[str, Any]: + payload = json.loads(text) + if isinstance(payload, dict) and "data" in payload and isinstance(payload["data"], dict): + return payload["data"] + if isinstance(payload, dict): + return payload + raise RuntimeError("Unexpected Browserless /function response shape") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("url", help="Target page URL") + parser.add_argument("--host", default=DEFAULT_HOST, help="Browserless host") + parser.add_argument( + "--token", + default=os.environ.get("BROWSERLESS_TOKEN") or os.environ.get("BROWSERLESS_API_KEY"), + help="Browserless token; defaults to BROWSERLESS_TOKEN or BROWSERLESS_API_KEY", + ) + parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, help="HTTP timeout in seconds") + parser.add_argument("--wait-ms", type=int, default=DEFAULT_WAIT_MS, help="Post-load wait time in ms") + parser.add_argument("--goto-timeout-ms", type=int, default=DEFAULT_GOTO_TIMEOUT_MS, help="Browser goto timeout in ms") + parser.add_argument("--play", action="store_true", help="Try muted playback/clicks before collecting media") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if not args.token: + print( + json.dumps( + { + "ok": False, + "error": "Missing Browserless token. Set BROWSERLESS_TOKEN or BROWSERLESS_API_KEY.", + }, + indent=2, + ), + file=sys.stderr, + ) + return 2 + + try: + _, response_text = post_json( + browserless_url(args.host, "/function", args.token), + { + "code": FUNCTION_CODE, + "context": { + "url": args.url, + "waitMs": args.wait_ms, + "gotoTimeoutMs": args.goto_timeout_ms, + "play": args.play, + }, + }, + args.timeout, + ) + result = parse_browserless_function_response(response_text) + result.setdefault("ok", True) + result.setdefault("provider", "browserless") + result.setdefault("mode", "function-media-requests") + print(json.dumps(result, indent=2)) + return 0 + except Exception as exc: # noqa: BLE001 + print(json.dumps({"ok": False, "error": str(exc), "provider": "browserless", "mode": "function-media-requests", "url": args.url}, indent=2)) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/web-use/scripts/browserless_session.py b/skills/web-use/scripts/browserless_session.py index e7c2c5d..f67d85e 100755 --- a/skills/web-use/scripts/browserless_session.py +++ b/skills/web-use/scripts/browserless_session.py @@ -41,8 +41,9 @@ import re import sys from typing import Any - -import requests +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen DEFAULT_HOST = "https://production-sfo.browserless.io" DEFAULT_TIMEOUT = 120 @@ -50,6 +51,44 @@ STD_STREAMS = {"-", "/dev/stdout", "/dev/stderr", "/dev/fd/1", "/dev/fd/2"} +class BrowserlessRequestError(RuntimeError): + """Raised when Browserless returns a failed HTTP response.""" + + +def with_query(url: str, params: dict[str, str] | None = None) -> str: + if not params: + return url + separator = "&" if "?" in url else "?" + return f"{url}{separator}{urlencode(params)}" + + +def request_json( + method: str, + url: str, + payload: dict[str, Any] | None, + timeout: int, +) -> tuple[int, Any]: + body = json.dumps(payload).encode("utf-8") if payload is not None else None + request = Request( + url, + data=body, + headers={"Content-Type": "application/json"} if body is not None else {}, + method=method, + ) + try: + with urlopen(request, timeout=timeout) as response: + text = response.read().decode("utf-8", errors="replace") + try: + return response.status, json.loads(text) if text else {} + except json.JSONDecodeError: + return response.status, {"raw": text} + except HTTPError as exc: + error_body = exc.read().decode("utf-8", errors="replace") + raise BrowserlessRequestError(f"HTTP {exc.code}: {error_body[:1000]}") from exc + except URLError as exc: + raise BrowserlessRequestError(str(exc.reason)) from exc + + def token_from_url(url: Any) -> str | None: if not isinstance(url, str): return None @@ -189,14 +228,12 @@ def cmd_create(args: argparse.Namespace) -> int: raise ValueError("--process-keep-alive-ms must be <= --ttl-ms.") body["processKeepAlive"] = args.process_keep_alive_ms - response = requests.post( - f"{args.host}/session", - params=params, - json=body, - timeout=args.timeout, + _, session = request_json( + "POST", + with_query(f"{args.host}/session", params), + body, + args.timeout, ) - response.raise_for_status() - session = response.json() if not isinstance(session, dict): raise ValueError("Browserless /session did not return a JSON object.") @@ -208,7 +245,7 @@ def cmd_create(args: argparse.Namespace) -> int: cleanup = "not attempted" if stop_url: try: - requests.delete(stop_url, timeout=args.timeout) + request_json("DELETE", stop_url, None, args.timeout) cleanup = "stopped" except Exception: # noqa: BLE001 cleanup = "stop failed" @@ -270,19 +307,15 @@ def cmd_query(args: argparse.Namespace) -> int: if not isinstance(variables, dict): raise ValueError("--variables-json must be a JSON object.") - response = requests.post( + status_code, payload = request_json( + "POST", browserql, - json={"query": query_text, "variables": variables}, - timeout=args.timeout, + {"query": query_text, "variables": variables}, + args.timeout, ) - status_code = response.status_code - try: - payload: Any = response.json() - except ValueError: - payload = {"raw": response.text} errors = payload.get("errors") if isinstance(payload, dict) else None - ok = response.ok and not errors + ok = 200 <= status_code < 300 and not errors emit( { "ok": ok, @@ -323,11 +356,15 @@ def cmd_stop(args: argparse.Namespace) -> int: "Session file has no stop URL; nothing to delete remotely." ) - response = requests.delete(stop_url, timeout=args.timeout) - status_code = response.status_code - stopped = response.ok + try: + status_code, _ = request_json("DELETE", stop_url, None, args.timeout) + stopped = 200 <= status_code < 300 + except BrowserlessRequestError as exc: + status_match = re.search(r"HTTP (\d+):", str(exc)) + status_code = int(status_match.group(1)) if status_match else None + stopped = False # A 404 means the session is already gone, so the local file is moot too. - removable = response.ok or status_code == 404 + removable = stopped or status_code == 404 file_deleted = False retained_reason = None @@ -413,9 +450,9 @@ def add_common(parser: argparse.ArgumentParser) -> None: ) parser.add_argument( "--token", - default=os.environ.get("BROWSERLESS_TOKEN"), + default=os.environ.get("BROWSERLESS_TOKEN") or os.environ.get("BROWSERLESS_API_KEY"), help=( - "Browserless API token (env BROWSERLESS_TOKEN). Required for create; " + "Browserless API token (env BROWSERLESS_TOKEN or BROWSERLESS_API_KEY). Required for create; " "query/stop read the token-bearing URL from the session file." ), ) diff --git a/skills/web-use/scripts/test.sh b/skills/web-use/scripts/test.sh new file mode 100755 index 0000000..a6bf7ec --- /dev/null +++ b/skills/web-use/scripts/test.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +pass=0 +fail=0 + +check() { + local name="$1" + shift + if "$@" >/tmp/web-use-test.out 2>/tmp/web-use-test.err; then + printf 'ok - %s\n' "$name" + pass=$((pass + 1)) + else + printf 'not ok - %s\n' "$name" + sed 's/^/ /' /tmp/web-use-test.err + fail=$((fail + 1)) + fi +} + +echo "=== web-use skill baseline tests ===" + +check "Frontmatter name matches directory name" grep -Eq '^name:[[:space:]]*"?web-use"?[[:space:]]*$' SKILL.md +check "browserless_extract helper parses" python3 -m py_compile scripts/browserless_extract.py +check "browserless_extract help works without credentials" python3 scripts/browserless_extract.py --help +check "browserless_session helper parses" python3 -m py_compile scripts/browserless_session.py +check "browserless_session help works without credentials" python3 scripts/browserless_session.py --help +check "browserless_media_requests helper parses" python3 -m py_compile scripts/browserless_media_requests.py +check "browserless_media_requests help works without credentials" python3 scripts/browserless_media_requests.py --help +check "tinyfish helper parses" python3 -m py_compile scripts/tinyfish_browser_extract.py +check "tinyfish help works without optional deps" python3 scripts/tinyfish_browser_extract.py --help + +rm -f /tmp/web-use-test.out /tmp/web-use-test.err + +echo "Passed: $pass" +echo "Failed: $fail" + +if [ "$fail" -ne 0 ]; then + exit 1 +fi diff --git a/skills/web-use/scripts/tinyfish_browser_extract.py b/skills/web-use/scripts/tinyfish_browser_extract.py index 3f327b0..684cd91 100755 --- a/skills/web-use/scripts/tinyfish_browser_extract.py +++ b/skills/web-use/scripts/tinyfish_browser_extract.py @@ -16,9 +16,8 @@ import re import sys from typing import Any - -import requests -import websockets +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen CREATE_SESSION_URL = "https://api.browser.tinyfish.ai/" DEFAULT_TIMEOUT_SECONDS = 300 @@ -26,6 +25,32 @@ DEFAULT_SNIPPET_CHARS = 3000 +class TinyFishRequestError(RuntimeError): + """Raised when TinyFish returns a failed HTTP response.""" + + +def post_json(url: str, payload: dict[str, Any], headers: dict[str, str], timeout: int) -> tuple[int, Any]: + body = json.dumps(payload).encode("utf-8") + request = Request( + url, + data=body, + headers={**headers, "Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=timeout) as response: + text = response.read().decode("utf-8", errors="replace") + try: + return response.status, json.loads(text) if text else {} + except json.JSONDecodeError: + return response.status, {"raw": text} + except HTTPError as exc: + error_body = exc.read().decode("utf-8", errors="replace") + raise TinyFishRequestError(f"HTTP {exc.code}: {error_body[:1000]}") from exc + except URLError as exc: + raise TinyFishRequestError(str(exc.reason)) from exc + + def sanitize_error(error: Exception, api_key: str | None, session: dict[str, Any] | None = None) -> str: message = str(error) secrets = [api_key] @@ -62,6 +87,13 @@ def signals(title: str, body: str) -> dict[str, bool]: async def evaluate_via_cdp(cdp_url: str, target_url: str, wait_seconds: int, snippet_chars: int) -> dict[str, Any]: + try: + import websockets + except ModuleNotFoundError as exc: + raise RuntimeError( + "TinyFish CDP extraction requires the optional Python package `websockets`." + ) from exc + async with websockets.connect(cdp_url, max_size=20_000_000) as ws: next_id = 0 @@ -157,14 +189,16 @@ def main() -> int: session: dict[str, Any] | None = None try: - response = requests.post( + status_code, session = post_json( CREATE_SESSION_URL, - headers={"X-API-Key": args.api_key, "Content-Type": "application/json"}, - json={"url": args.url, "timeout_seconds": args.session_timeout_seconds}, - timeout=60, + {"url": args.url, "timeout_seconds": args.session_timeout_seconds}, + {"X-API-Key": args.api_key}, + 60, ) - response.raise_for_status() - session = response.json() + if not (200 <= status_code < 300): + raise TinyFishRequestError(f"HTTP {status_code}: {session}") + if not isinstance(session, dict): + raise TinyFishRequestError("TinyFish response was not a JSON object") cdp_url = session.get("cdp_url") if not cdp_url: raise RuntimeError("TinyFish response missing cdp_url")