diff --git a/README.md b/README.md index c74e57d..12c91e0 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,8 @@ The `fetch` function can be used to retrieve the content of a given web page in markdown format. You can use the `render_js` flag to execute the JavaScript code of the page before returning the -content, and ask to `include_raw_html` to the response if you feel like it. +content, and set `include_raw_content` to include the raw page content and its content type. +`include_raw_html` is deprecated in favor of `include_raw_content`. ```python import linkup @@ -146,7 +147,7 @@ client = linkup.Client() # API key can be read from the environment variable or fetch_response: linkup.FetchResponse = client.fetch( url="https://docs.linkup.so", render_js=False, - include_raw_html=True, + include_raw_content=True, ) print(fetch_response.model_dump()) ``` @@ -156,7 +157,8 @@ Which prints: ```bash { markdown="Get started for free, no credit card required...", - raw_html="......" + raw_content="......", + content_type="html" } ``` diff --git a/src/linkup/_client.py b/src/linkup/_client.py index d7cf223..fb299fe 100644 --- a/src/linkup/_client.py +++ b/src/linkup/_client.py @@ -238,8 +238,8 @@ def search( datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, the search results will not be filtered by date. to_date: The date until which the search results should be considered. Accepts a - datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, the search - results will not be filtered by date. + datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, defaults to the + current date. exclude_domains: If you want to exclude specific domains from your search. include_domains: If you want the search to only return results from certain domains. max_results: The maximum number of results to return. @@ -435,8 +435,8 @@ async def async_search( datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, the search results will not be filtered by date. to_date: The date until which the search results should be considered. Accepts a - datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, the search - results will not be filtered by date. + datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, defaults to the + current date. exclude_domains: If you want to exclude specific domains from your search. include_domains: If you want the search to only return results from certain domains. max_results: The maximum number of results to return. @@ -526,8 +526,8 @@ def research( datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, sources will not be filtered by a start date. to_date: The date until which the research sources should be considered. Accepts a - datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, sources will - not be filtered by an end date. + datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, defaults to the + current date. exclude_domains: Domains to exclude from the research sources. include_domains: Domains to restrict the research sources to. timeout: The timeout for the HTTP request, in seconds. If None, the request will have @@ -599,8 +599,8 @@ async def async_research( datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, sources will not be filtered by a start date. to_date: The date until which the research sources should be considered. Accepts a - datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, sources will - not be filtered by an end date. + datetime.date, YYYY-MM-DD, or full ISO datetime string. If None, defaults to the + current date. exclude_domains: Domains to exclude from the research sources. include_domains: Domains to restrict the research sources to. timeout: The timeout for the HTTP request, in seconds. If None, the request will have @@ -1001,6 +1001,7 @@ def fetch( render_js: bool | None = None, extract_images: bool | None = None, timeout: float | None = None, + include_raw_content: bool | None = None, ) -> LinkupFetchResponse: """Fetch the content of a web page using the Linkup API /fetch endpoint. @@ -1011,11 +1012,14 @@ def fetch( Args: url: The URL of the web page to fetch. include_raw_html: Whether to include the raw HTML of the webpage in the response. + Deprecated; use include_raw_content instead. render_js: Whether the API should render the JavaScript of the webpage. extract_images: Whether the API should extract images from the webpage and return them in the response. timeout: The timeout for the HTTP request, in seconds. If None, the request will have no timeout. + include_raw_content: Whether to include the raw page content and its content type in the + response. Returns: The response of the web page fetch, containing the web page content. @@ -1032,6 +1036,7 @@ def fetch( params: dict[str, str | bool] = self._get_fetch_params( url=url, include_raw_html=include_raw_html, + include_raw_content=include_raw_content, render_js=render_js, extract_images=extract_images, ) @@ -1052,6 +1057,7 @@ async def async_fetch( render_js: bool | None = None, extract_images: bool | None = None, timeout: float | None = None, + include_raw_content: bool | None = None, ) -> LinkupFetchResponse: """Asynchronously fetch the content of a web page using the Linkup API /fetch endpoint. @@ -1062,11 +1068,14 @@ async def async_fetch( Args: url: The URL of the web page to fetch. include_raw_html: Whether to include the raw HTML of the webpage in the response. + Deprecated; use include_raw_content instead. render_js: Whether the API should render the JavaScript of the webpage. extract_images: Whether the API should extract images from the webpage and return them in the response. timeout: The timeout for the HTTP request, in seconds. If None, the request will have no timeout. + include_raw_content: Whether to include the raw page content and its content type in the + response. Returns: The response of the web page fetch, containing the web page content. @@ -1083,6 +1092,7 @@ async def async_fetch( params: dict[str, str | bool] = self._get_fetch_params( url=url, include_raw_html=include_raw_html, + include_raw_content=include_raw_content, render_js=render_js, extract_images=extract_images, ) @@ -1592,6 +1602,7 @@ def _get_tasks_payload(self, tasks: list[LinkupTaskInput]) -> list[dict[str, Any "input": self._get_fetch_params( url=task.url, include_raw_html=task.include_raw_html, + include_raw_content=task.include_raw_content, render_js=task.render_js, extract_images=task.extract_images, ), @@ -1626,6 +1637,7 @@ def _get_fetch_params( self, url: str, include_raw_html: bool | None, + include_raw_content: bool | None, render_js: bool | None, extract_images: bool | None, ) -> dict[str, str | bool]: @@ -1634,6 +1646,8 @@ def _get_fetch_params( } if include_raw_html is not None: params["includeRawHtml"] = include_raw_html + if include_raw_content is not None: + params["includeRawContent"] = include_raw_content if render_js is not None: params["renderJs"] = render_js if extract_images is not None: diff --git a/src/linkup/_types.py b/src/linkup/_types.py index 1736d47..5c587e3 100644 --- a/src/linkup/_types.py +++ b/src/linkup/_types.py @@ -114,11 +114,15 @@ class LinkupFetchResponse(_LinkupBaseModel): Attributes: markdown: The cleaned up markdown content. - raw_html: The optional raw HTML content. - images: The optional list of image URLs. + raw_content: The optional raw page content. + content_type: The type of the raw page content, if returned. + raw_html: The optional raw HTML content. Deprecated; use raw_content instead. + images: The optional list of extracted images. """ markdown: str + raw_content: str | None = pydantic.Field(default=None, validation_alias="rawContent") + content_type: str | None = pydantic.Field(default=None, validation_alias="contentType") raw_html: str | None = pydantic.Field(default=None, validation_alias="rawHtml") images: list[LinkupFetchImageExtraction] | None = pydantic.Field(default=None) @@ -222,12 +226,17 @@ class LinkupFetchTaskInput(_LinkupBaseModel): Attributes: url: The URL requested for fetching. - include_raw_html: Whether raw HTML should be included in the fetch response. + include_raw_content: Whether raw page content should be included in the fetch response. + include_raw_html: Whether raw HTML should be included in the fetch response. Deprecated; + use include_raw_content instead. render_js: Whether JavaScript rendering should be enabled. extract_images: Whether image extraction should be enabled. """ url: str + include_raw_content: bool | None = pydantic.Field( + default=None, validation_alias="includeRawContent" + ) include_raw_html: bool | None = pydantic.Field(default=None, validation_alias="includeRawHtml") render_js: bool | None = pydantic.Field(default=None, validation_alias="renderJs") extract_images: bool | None = pydantic.Field(default=None, validation_alias="extractImages") diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index c65d12d..50c3d74 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -900,6 +900,28 @@ def test_research_with_iso_datetime_string_dates( b'{"markdown": "Some web page content"}', linkup.FetchResponse(markdown="Some web page content", raw_html=None), ), + ( + { + "url": "https://example.com", + "include_raw_content": True, + }, + { + "url": "https://example.com", + "includeRawContent": True, + }, + b""" + { + "markdown": "# Some web page content", + "rawContent": "...", + "contentType": "html" + } + """, + linkup.FetchResponse( + markdown="# Some web page content", + raw_content="...", + content_type="html", + ), + ), ( { "url": "https://example.com", @@ -1178,16 +1200,19 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None: "id": "42057d84-72ea-4029-9598-1bf7424a6113", "input": { "extractImages": true, + "includeRawContent": true, "url": "https://example.com" }, "output": { + "contentType": "html", "images": [ { "alt": "hero", "url": "https://example.com/image.png" } ], - "markdown": "Fetched content" + "markdown": "Fetched content", + "rawContent": "Fetched content" }, "status": "completed", "type": "fetch", @@ -1209,6 +1234,7 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None: linkup.FetchTaskInput( url="https://example.com", extract_images=True, + include_raw_content=True, ), ] ) @@ -1231,6 +1257,7 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None: "input": { "url": "https://example.com", "extractImages": True, + "includeRawContent": True, }, }, ], @@ -1243,6 +1270,8 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None: assert tasks_response[1].output is not None assert tasks_response[1].output.images is not None assert tasks_response[1].output.images[0].url == "https://example.com/image.png" + assert tasks_response[1].output.raw_content == "Fetched content" + assert tasks_response[1].output.content_type == "html" def test_create_tasks_research_model(mocker: MockerFixture, client: linkup.Client) -> None: