From 3457ae332280a5673276c526babdb557dbf8b6c0 Mon Sep 17 00:00:00 2001 From: wang2 <134143178+fatwang2@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:33:38 +0100 Subject: [PATCH] Add deepcrawl max_pages, path rules and JSON format, release 0.6.0 Sync the OpenAPI snapshot with the live contract (adds max_pages, include_paths, exclude_paths and format to POST /deepcrawl and the stats/failed/resultUrl/pages fields to the status response) and accept the new fields as keyword arguments on start_deepcrawl and deepcrawl in the sync and async clients. Unset arguments are not sent. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 + README.md | 26 +++ openapi/search1api.openapi.json | 332 +++++++++++++++++++++++++++++++- pyproject.toml | 2 +- src/search1api/__init__.py | 2 +- src/search1api/client.py | 96 ++++++++- src/search1api/types.py | 25 +++ tests/test_client.py | 124 ++++++++++++ 8 files changed, 598 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7b0174..2d7d5f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.6.0 - 2026-11-05 + +Adds keyword arguments to `start_deepcrawl` and `deepcrawl` on `Search1API` and `AsyncSearch1API`: `max_pages` (1-1000, default 1000, applied after the path rules in discovery order), `include_paths` and `exclude_paths` (up to 20 URL path rules each; a rule without `*` is a path prefix on segment boundaries, a rule with `*` matches the whole path; exclusion wins) and `format` (`"zip"`, the default, or `"json"`). Unset arguments are not sent. `DeepcrawlStatusResponse` adds `format`, `stats` (`discovered`, `matched`, `skipped`, `crawled`, `failed`), `failed` (`url`, `error`) and, for `format="json"`, `resultUrl` and `pages` (`url`, `title`, Markdown `content`; omitted above 5 MB, use `resultUrl`). Still 20 credits per task. The bundled OpenAPI snapshot is re-synced with the live contract. + ## 0.5.0 - 2026-11-03 Adds `maps(query, ...)` on `Search1API` and `AsyncSearch1API` for `POST /maps`: one Google Maps page of up to 20 places with name, place ID and Maps link, plus address, phone, website, rating, review count, coordinates and opening hours when available. Keyword arguments: `country`, `language`, `location` (`latitude`, `longitude`, optional `zoom`), `page` (1-6, pages after the first need `location`) and `max_results`. 3 credits per completed page; API key or OAuth only. The bundled OpenAPI snapshot also records the 50-item batch limit for search, news and crawl. diff --git a/README.md b/README.md index d9595ad..54d71be 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,32 @@ result = client.deepcrawl("https://example.com", type="all") print(result["zipUrl"]) ``` +`max_pages` (1-1000, default 1000) caps the number of pages. `include_paths` +and `exclude_paths` (up to 20 rules each) are matched against the URL path: a +rule without `*` is a path prefix on segment boundaries (`/docs` matches +`/docs/api` but not `/docsearch`); a rule with `*` must match the whole path, +and `*` also matches `/`. Exclusion wins, and `max_pages` applies after the path +rules, in discovery order. `format="json"` returns the pages in the result +instead of a ZIP: + +```python +result = client.deepcrawl( + "https://example.com", + max_pages=50, + include_paths=["/docs"], + format="json", +) + +for page in result.get("pages", []): + print(page["url"], page["title"], len(page["content"])) +``` + +Each page has `url`, `title` (may be `None`) and `content` as Markdown. +`pages` is omitted when the result is larger than 5 MB; download `resultUrl` +instead, a JSON file with the same `pages` and `failed` lists. Finished tasks +also report `stats` (`discovered`, `matched`, `skipped`, `crawled`, `failed`) +and `failed` (`url`, `error`). A task costs 20 credits regardless of page count. + Use `start_deepcrawl`, `get_deepcrawl_status`, and `wait_for_deepcrawl` when the application needs to control persistence or polling itself. diff --git a/openapi/search1api.openapi.json b/openapi/search1api.openapi.json index 0d81feb..c83eb36 100644 --- a/openapi/search1api.openapi.json +++ b/openapi/search1api.openapi.json @@ -68,6 +68,14 @@ }, "detail": { "type": "string" + }, + "failureCategory": { + "type": "string", + "description": "`/crawl` and `/extract`: why the page could not be read, e.g. `NOT_FOUND`, `GONE`, `FORBIDDEN`, `AUTH_REQUIRED`, `BLOCKED`, `RATE_LIMITED`, `DNS_NOT_FOUND`, `TARGET_UNREACHABLE`, `TIMEOUT`, `QUALITY_CHECK`." + }, + "upstreamStatus": { + "type": "integer", + "description": "`/crawl` and `/extract`: the HTTP status the target site itself returned, when it answered." } }, "additionalProperties": true @@ -521,6 +529,40 @@ "sitemap", "all" ] + }, + "max_pages": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "Maximum number of pages to crawl, 1–1000; default 1000. Applied after include_paths/exclude_paths, in discovery order." + }, + "include_paths": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "maxItems": 20, + "description": "Only crawl URLs whose path matches one of these rules (up to 20). A rule without * is a path prefix on segment boundaries: \"/docs\" matches /docs and /docs/api but not /docsearch. A rule with * must match the whole path, and * matches any characters including /, e.g. \"/blog/*\". Empty or omitted means every discovered URL." + }, + "exclude_paths": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "maxItems": 20, + "description": "Skip URLs whose path matches one of these rules (up to 20), using the same syntax as include_paths. Exclusion wins over inclusion." + }, + "format": { + "type": "string", + "enum": [ + "zip", + "json" + ], + "description": "Result format; default \"zip\". \"zip\" packages the pages as Markdown files behind zipUrl. \"json\" returns pages: [{url, title, content}] in the completed status response, plus resultUrl for the same JSON file." } }, "required": [ @@ -562,7 +604,7 @@ }, "SearchResultMetadata": { "type": "object", - "description": "Optional engine-specific fields. Omitted when none apply. GitHub and Reddit fields appear on POST /search; Hacker News fields appear on POST /news.", + "description": "Optional engine-specific fields. Omitted when none apply. GitHub, Reddit and X fields appear on POST /search; Hacker News fields appear on POST /news.", "properties": { "kind": { "type": "string", @@ -580,11 +622,11 @@ }, "language": { "type": "string", - "description": "Primary language of a `github` repository result (`metadata.kind: \"repo\"`). Omitted when GitHub reports none." + "description": "Primary language of a `github` repository result (`metadata.kind: \"repo\"`), or the language code X detected for an `x` post. Omitted when none is reported." }, "num_comments": { "type": "integer", - "description": "Comment count on a `github` issue, pull request or discussion, on a `reddit` post, or on a `hackernews` thread (POST /news)." + "description": "Comment count on a `github` issue, pull request or discussion, on a `reddit` post, on an `x` post (replies), or on a `hackernews` thread (POST /news)." }, "story_url": { "type": "string", @@ -594,6 +636,104 @@ "points": { "type": "integer", "description": "POST /news with `search_service: \"hackernews\"` only: upvotes on the thread." + }, + "post_id": { + "type": "string", + "description": "`search_service: \"x\"` only: the post id." + }, + "author_username": { + "type": "string", + "description": "`x` only: handle of the account that posted, without the @." + }, + "author_name": { + "type": "string", + "description": "`x` only: display name of the account that posted." + }, + "author_id": { + "type": "string", + "description": "`x` only: numeric user id of the account that posted." + }, + "author_image": { + "type": "string", + "format": "uri", + "description": "`x` only: avatar URL of the account that posted." + }, + "author_verified": { + "type": "boolean", + "description": "`x` only: whether the account shows a verified checkmark." + }, + "author_followers": { + "type": "integer", + "description": "`x` only: follower count of the account that posted." + }, + "like_count": { + "type": "integer", + "description": "`x` only: likes on the post." + }, + "repost_count": { + "type": "integer", + "description": "`x` only: reposts of the post." + }, + "quote_count": { + "type": "integer", + "description": "`x` only: quotes of the post." + }, + "bookmark_count": { + "type": "integer", + "description": "`x` only: bookmarks of the post." + }, + "view_count": { + "type": "integer", + "description": "`x` only: views of the post. Omitted when X reports none, as for older posts." + }, + "is_reply": { + "type": "boolean", + "description": "`x` only: whether the post replies to another post." + }, + "conversation_id": { + "type": "string", + "description": "`x` only: id of the first post in the thread." + }, + "in_reply_to_id": { + "type": "string", + "description": "`x` only: id of the post this one replies to. Omitted when it is not a reply." + }, + "media": { + "type": "array", + "description": "`x` only: photos, videos and GIFs attached to the post. Empty when there are none.", + "items": { + "type": "object", + "required": [ + "type", + "url" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "photo", + "video", + "gif" + ] + }, + "url": { + "type": "string", + "format": "uri", + "description": "Image URL; for a video or GIF, its poster frame." + }, + "width": { + "type": "integer" + }, + "height": { + "type": "integer" + }, + "video_url": { + "type": "string", + "format": "uri", + "description": "Playable MP4 (highest bitrate) for a video or GIF." + } + } + } } }, "additionalProperties": true @@ -1092,6 +1232,102 @@ "null" ], "format": "uri" + }, + "stats": { + "type": "object", + "description": "Page counts for a finished task: discovered links, how many matched the path rules, how many max_pages left out, and how many pages were crawled or failed.", + "required": [ + "discovered", + "matched", + "skipped", + "crawled", + "failed" + ], + "properties": { + "discovered": { + "type": "integer", + "minimum": 0 + }, + "matched": { + "type": "integer", + "minimum": 0 + }, + "skipped": { + "type": "integer", + "minimum": 0 + }, + "crawled": { + "type": "integer", + "minimum": 0 + }, + "failed": { + "type": "integer", + "minimum": 0 + } + } + }, + "format": { + "type": "string", + "enum": [ + "zip", + "json" + ] + }, + "failed": { + "type": "array", + "description": "Pages that could not be crawled, with the reason.", + "items": { + "type": "object", + "required": [ + "url", + "error" + ], + "properties": { + "url": { + "type": "string", + "format": "uri" + }, + "error": { + "type": "string" + } + } + } + }, + "resultUrl": { + "type": [ + "string", + "null" + ], + "format": "uri", + "description": "format=json only: the JSON result file, {\"pages\": [...], \"failed\": [...]}." + }, + "pages": { + "type": "array", + "description": "format=json only: the crawled pages in discovery order. Omitted when the result is larger than 5 MB; download resultUrl instead.", + "items": { + "type": "object", + "required": [ + "url", + "title", + "content" + ], + "properties": { + "url": { + "type": "string", + "format": "uri" + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "content": { + "type": "string", + "description": "Page body as Markdown." + } + } + } } }, "additionalProperties": true @@ -3093,6 +3329,26 @@ } } }, + "404": { + "description": "Not Found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, + "410": { + "description": "Gone", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ApiError" + } + } + } + }, "422": { "description": "Validation Error", "content": { @@ -3179,7 +3435,7 @@ "post": { "operationId": "deepcrawl", "summary": "Deep crawl a website across multiple pages", - "description": "Start an asynchronous crawl of an entire site and package the pages as documents. Use it for whole-site ingestion; a single page is POST /crawl. The call returns a task id immediately rather than the result — poll GET /deepcrawl/status/{taskId} until the task reports completion. Costs 20 credits per request.", + "description": "Start an asynchronous crawl of an entire site and package the pages as documents. Use it for whole-site ingestion; a single page is POST /crawl. Narrow the crawl with include_paths/exclude_paths and cap it with max_pages; a task costs the same regardless of page count. Set format to \"json\" to get the pages inline in the status response instead of a ZIP. The call returns a task id immediately rather than the result — poll GET /deepcrawl/status/{taskId} until the task reports completion. Costs 20 credits per request.", "tags": [ "Crawl" ], @@ -3308,12 +3564,78 @@ "sitemap", "all" ] + }, + "max_pages": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "Maximum number of pages to crawl, 1–1000; default 1000. Applied after include_paths/exclude_paths, in discovery order." + }, + "include_paths": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "maxItems": 20, + "description": "Only crawl URLs whose path matches one of these rules (up to 20). A rule without * is a path prefix on segment boundaries: \"/docs\" matches /docs and /docs/api but not /docsearch. A rule with * must match the whole path, and * matches any characters including /, e.g. \"/blog/*\". Empty or omitted means every discovered URL." + }, + "exclude_paths": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "maxItems": 20, + "description": "Skip URLs whose path matches one of these rules (up to 20), using the same syntax as include_paths. Exclusion wins over inclusion." + }, + "format": { + "type": "string", + "enum": [ + "zip", + "json" + ], + "description": "Result format; default \"zip\". \"zip\" packages the pages as Markdown files behind zipUrl. \"json\" returns pages: [{url, title, content}] in the completed status response, plus resultUrl for the same JSON file." } }, "required": [ "url" ], "additionalProperties": false + }, + "examples": { + "wholeSite": { + "summary": "Whole site", + "description": "Crawl every URL the site lists in its sitemap, up to 1000 pages.", + "value": { + "url": "https://example.com" + } + }, + "docsSection": { + "summary": "One section, capped", + "description": "Crawl at most 20 pages under /docs/, skipping the changelog.", + "value": { + "url": "https://hono.dev/docs/", + "max_pages": 20, + "include_paths": [ + "/docs/" + ], + "exclude_paths": [ + "/docs/changelog" + ] + } + }, + "jsonPages": { + "summary": "Pages as JSON", + "description": "Return url, title and Markdown content for each page in the status response instead of a ZIP.", + "value": { + "url": "https://hono.dev/docs/", + "max_pages": 20, + "format": "json" + } + } } } } @@ -3324,7 +3646,7 @@ "get": { "operationId": "deepcrawlStatus", "summary": "Check deepcrawl task status", - "description": "Poll a deepcrawl task started by POST /deepcrawl. Returns the task's current state and, once it finishes, where to retrieve the packaged result. Safe to call repeatedly. Free to call.", + "description": "Poll a deepcrawl task started by POST /deepcrawl. Returns the task's current state and, once it finishes, page counts (stats), failed URLs, and the result: zipUrl for format=zip, or the pages inline plus resultUrl for format=json. Safe to call repeatedly. Free to call.", "tags": [ "Crawl" ], diff --git a/pyproject.toml b/pyproject.toml index 1354065..f5cdd01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "search1api" -version = "0.5.0" +version = "0.6.0" description = "Official Python client for Search1API" readme = "README.md" license = { file = "LICENSE" } diff --git a/src/search1api/__init__.py b/src/search1api/__init__.py index 3498479..57cbc02 100644 --- a/src/search1api/__init__.py +++ b/src/search1api/__init__.py @@ -35,4 +35,4 @@ "UnprocessableEntityError", ] -__version__ = "0.5.0" +__version__ = "0.6.0" diff --git a/src/search1api/client.py b/src/search1api/client.py index ad1d822..1175f0d 100644 --- a/src/search1api/client.py +++ b/src/search1api/client.py @@ -27,6 +27,7 @@ CrawlResponse, CrawlType, DeepcrawlAcceptedResponse, + DeepcrawlFormat, DeepcrawlStatusResponse, ExtractResponse, FeedbackAgent, @@ -87,6 +88,27 @@ def _maps_payload( ) +def _deepcrawl_payload( + url: str, + *, + type: Optional[CrawlType], + max_pages: Optional[int], + include_paths: Optional[List[str]], + exclude_paths: Optional[List[str]], + format: Optional[DeepcrawlFormat], +) -> Dict[str, Any]: + return _compact( + { + "url": url, + "type": type, + "max_pages": max_pages, + "include_paths": include_paths, + "exclude_paths": exclude_paths, + "format": format, + } + ) + + def _search_payload( query: str, *, @@ -468,14 +490,34 @@ def extract( ) def start_deepcrawl( - self, url: str, *, type: Optional[CrawlType] = None + self, + url: str, + *, + type: Optional[CrawlType] = None, + max_pages: Optional[int] = None, + include_paths: Optional[List[str]] = None, + exclude_paths: Optional[List[str]] = None, + format: Optional[DeepcrawlFormat] = None, ) -> DeepcrawlAcceptedResponse: + """Start a deepcrawl task and return its task id. + + A task costs 20 credits regardless of how many pages it crawls. + ``max_pages`` (1-1000) applies after ``include_paths``/``exclude_paths``. + ``format="json"`` returns the pages in the completed status response. + """ return cast( DeepcrawlAcceptedResponse, self._request_json( "POST", "/deepcrawl", - json=_compact({"url": url, "type": type}), + json=_deepcrawl_payload( + url, + type=type, + max_pages=max_pages, + include_paths=include_paths, + exclude_paths=exclude_paths, + format=format, + ), retryable=False, ), ) @@ -512,10 +554,21 @@ def deepcrawl( url: str, *, type: Optional[CrawlType] = None, + max_pages: Optional[int] = None, + include_paths: Optional[List[str]] = None, + exclude_paths: Optional[List[str]] = None, + format: Optional[DeepcrawlFormat] = None, poll_interval: float = DEFAULT_DEEPCRAWL_POLL_INTERVAL, timeout: float = DEFAULT_DEEPCRAWL_TIMEOUT, ) -> DeepcrawlStatusResponse: - task = self.start_deepcrawl(url, type=type) + task = self.start_deepcrawl( + url, + type=type, + max_pages=max_pages, + include_paths=include_paths, + exclude_paths=exclude_paths, + format=format, + ) return self.wait_for_deepcrawl( task["taskId"], poll_interval=poll_interval, timeout=timeout ) @@ -869,14 +922,34 @@ async def extract( ) async def start_deepcrawl( - self, url: str, *, type: Optional[CrawlType] = None + self, + url: str, + *, + type: Optional[CrawlType] = None, + max_pages: Optional[int] = None, + include_paths: Optional[List[str]] = None, + exclude_paths: Optional[List[str]] = None, + format: Optional[DeepcrawlFormat] = None, ) -> DeepcrawlAcceptedResponse: + """Start a deepcrawl task and return its task id. + + A task costs 20 credits regardless of how many pages it crawls. + ``max_pages`` (1-1000) applies after ``include_paths``/``exclude_paths``. + ``format="json"`` returns the pages in the completed status response. + """ return cast( DeepcrawlAcceptedResponse, await self._request_json( "POST", "/deepcrawl", - json=_compact({"url": url, "type": type}), + json=_deepcrawl_payload( + url, + type=type, + max_pages=max_pages, + include_paths=include_paths, + exclude_paths=exclude_paths, + format=format, + ), retryable=False, ), ) @@ -916,10 +989,21 @@ async def deepcrawl( url: str, *, type: Optional[CrawlType] = None, + max_pages: Optional[int] = None, + include_paths: Optional[List[str]] = None, + exclude_paths: Optional[List[str]] = None, + format: Optional[DeepcrawlFormat] = None, poll_interval: float = DEFAULT_DEEPCRAWL_POLL_INTERVAL, timeout: float = DEFAULT_DEEPCRAWL_TIMEOUT, ) -> DeepcrawlStatusResponse: - task = await self.start_deepcrawl(url, type=type) + task = await self.start_deepcrawl( + url, + type=type, + max_pages=max_pages, + include_paths=include_paths, + exclude_paths=exclude_paths, + format=format, + ) return await self.wait_for_deepcrawl( task["taskId"], poll_interval=poll_interval, timeout=timeout ) diff --git a/src/search1api/types.py b/src/search1api/types.py index bd73777..ad97a5e 100644 --- a/src/search1api/types.py +++ b/src/search1api/types.py @@ -28,6 +28,7 @@ ] NewsEngine = Literal["google", "bing", "duckduckgo", "yahoo", "hackernews", "reuters"] CrawlType = Literal["sitemap", "all"] +DeepcrawlFormat = Literal["zip", "json"] ScreenshotFormat = Literal["png", "jpeg", "webp"] ScreenshotWaitUntil = Literal["domcontentloaded", "load", "networkidle"] @@ -293,6 +294,25 @@ class DeepcrawlStatusResponseRequired(TypedDict): taskId: str +class DeepcrawlStats(TypedDict): + discovered: int + matched: int + skipped: int + crawled: int + failed: int + + +class DeepcrawlFailedPage(TypedDict): + url: str + error: str + + +class DeepcrawlPage(TypedDict): + url: str + title: Optional[str] + content: str + + class DeepcrawlStatusResponse(DeepcrawlStatusResponseRequired, total=False): status: str success: bool @@ -300,6 +320,11 @@ class DeepcrawlStatusResponse(DeepcrawlStatusResponseRequired, total=False): error: Optional[str] r2Key: str zipUrl: Optional[str] + format: DeepcrawlFormat + stats: DeepcrawlStats + failed: List[DeepcrawlFailedPage] + resultUrl: Optional[str] + pages: List[DeepcrawlPage] class UsageResponse(TypedDict): diff --git a/tests/test_client.py b/tests/test_client.py index 0501288..5598d9c 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -248,6 +248,130 @@ def handler(request): http_client.close() +DEEPCRAWL_JSON_RESULT = { + "taskId": "task_2", + "status": "completed", + "success": True, + "format": "json", + "stats": {"discovered": 12, "matched": 4, "skipped": 1, "crawled": 2, "failed": 1}, + "failed": [{"url": "https://example.com/docs/broken", "error": "HTTP 404"}], + "resultUrl": "https://example.com/result.json", + "pages": [ + {"url": "https://example.com/docs", "title": "Docs", "content": "# Docs"}, + {"url": "https://example.com/docs/api", "title": None, "content": "API"}, + ], +} + + +def test_deepcrawl_sends_scope_and_json_format(): + seen = [] + responses = [ + (202, {"taskId": "task_2", "status": "queued"}), + (200, DEEPCRAWL_JSON_RESULT), + ] + + def handler(request): + seen.append(request) + status, body = responses.pop(0) + return json_response(status, body, request) + + http_client = httpx.Client(transport=httpx.MockTransport(handler)) + client = Search1API("test-key", client=http_client) + + result = client.deepcrawl( + "https://example.com", + max_pages=3, + include_paths=["/docs"], + exclude_paths=["/docs/internal/*"], + format="json", + poll_interval=0, + timeout=1, + ) + + assert result == DEEPCRAWL_JSON_RESULT + assert [page["url"] for page in result["pages"]] == [ + "https://example.com/docs", + "https://example.com/docs/api", + ] + assert result["stats"]["skipped"] == 1 + assert str(seen[0].url) == "https://api.search1api.com/deepcrawl" + assert json.loads(seen[0].content) == { + "url": "https://example.com", + "max_pages": 3, + "include_paths": ["/docs"], + "exclude_paths": ["/docs/internal/*"], + "format": "json", + } + assert str(seen[1].url) == "https://api.search1api.com/deepcrawl/status/task_2" + http_client.close() + + +def test_start_deepcrawl_sends_only_set_fields(): + seen = [] + + def handler(request): + seen.append(json.loads(request.content)) + return json_response(202, {"taskId": "task_3", "status": "queued"}, request) + + http_client = httpx.Client(transport=httpx.MockTransport(handler)) + client = Search1API("test-key", client=http_client) + + client.start_deepcrawl("https://example.com") + client.start_deepcrawl("https://example.com", type="sitemap", max_pages=1) + + assert seen == [ + {"url": "https://example.com"}, + {"url": "https://example.com", "type": "sitemap", "max_pages": 1}, + ] + http_client.close() + + +def test_async_deepcrawl_sends_scope_and_json_format(): + async def run(): + seen = [] + responses = [ + (202, {"taskId": "task_2", "status": "queued"}), + (200, {"taskId": "task_2", "status": "processing"}), + (200, DEEPCRAWL_JSON_RESULT), + ] + + async def handler(request): + seen.append(request) + status, body = responses.pop(0) + return json_response(status, body, request) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + client = AsyncSearch1API("test-key", client=http_client) + try: + result = await client.deepcrawl( + "https://example.com", + type="all", + max_pages=3, + include_paths=["/docs"], + exclude_paths=["/docs/internal/*"], + format="json", + poll_interval=0, + timeout=1, + ) + finally: + await http_client.aclose() + + assert result["pages"][1]["title"] is None + assert result["failed"] == DEEPCRAWL_JSON_RESULT["failed"] + assert result["resultUrl"] == "https://example.com/result.json" + assert json.loads(seen[0].content) == { + "url": "https://example.com", + "type": "all", + "max_pages": 3, + "include_paths": ["/docs"], + "exclude_paths": ["/docs/internal/*"], + "format": "json", + } + assert responses == [] + + asyncio.run(run()) + + def test_search_sends_page_and_new_engines(): bodies = []