feat: support fetch mode selection

README.md

@@ -137,17 +137,18 @@ The `fetch` function can be used to retrieve the content of a given web page in
 markdown format, together with the website's favicon URL.
 
 You can use the `render_js` flag to execute the JavaScript code of the page before returning the
-content, and set `include_raw_content` to include the raw page content and its content type.
-`include_raw_html` is deprecated in favor of `include_raw_content`.
+content, set `include_raw_content` to include the raw page content and its content type in the
+output, or set `mode` to `"pro"` for significantly higher success rates on hard-to-retrieve pages.
 
 ```python
 import linkup
 
 client = linkup.Client()  # API key can be read from the environment variable or passed as an argument
 fetch_response: linkup.FetchResponse = client.fetch(
     url="https://docs.linkup.so",
-    render_js=False,
+    render_js=True,
     include_raw_content=True,
+    mode="pro",
 )
 print(fetch_response.model_dump())
 ```

src/linkup/_client.py

@@ -1003,6 +1003,7 @@ def fetch(
         extract_images: bool | None = None,
         timeout: float | None = None,
         include_raw_content: bool | None = None,
+        mode: Literal["standard", "pro"] | None = None,
     ) -> LinkupFetchResponse:
         """Fetch the content of a web page using the Linkup API /fetch endpoint.
 
@@ -1021,6 +1022,8 @@ def fetch(
                 no timeout.
             include_raw_content: Whether to include the raw page content and its content type in the
                 response.
+            mode: The fetch strategy to use. "pro" delivers significantly higher success rates on
+                hard-to-retrieve pages.
 
         Returns:
             The response of the web page fetch, containing the web page content.
@@ -1040,6 +1043,7 @@ def fetch(
             include_raw_content=include_raw_content,
             render_js=render_js,
             extract_images=extract_images,
+            mode=mode,
         )
 
         response: httpx.Response = self._request(
@@ -1059,6 +1063,7 @@ async def async_fetch(
         extract_images: bool | None = None,
         timeout: float | None = None,
         include_raw_content: bool | None = None,
+        mode: Literal["standard", "pro"] | None = None,
     ) -> LinkupFetchResponse:
         """Asynchronously fetch the content of a web page using the Linkup API /fetch endpoint.
 
@@ -1077,6 +1082,8 @@ async def async_fetch(
                 no timeout.
             include_raw_content: Whether to include the raw page content and its content type in the
                 response.
+            mode: The fetch strategy to use. "pro" delivers significantly higher success rates on
+                hard-to-retrieve pages.
 
         Returns:
             The response of the web page fetch, containing the web page content.
@@ -1096,6 +1103,7 @@ async def async_fetch(
             include_raw_content=include_raw_content,
             render_js=render_js,
             extract_images=extract_images,
+            mode=mode,
         )
 
         response: httpx.Response = await self._async_request(
@@ -1612,6 +1620,7 @@ def _get_tasks_payload(self, tasks: list[LinkupTaskInput]) -> list[dict[str, Any
                             include_raw_content=task.include_raw_content,
                             render_js=task.render_js,
                             extract_images=task.extract_images,
+                            mode=task.mode,
                         ),
                     }
                 )
@@ -1647,6 +1656,7 @@ def _get_fetch_params(
         include_raw_content: bool | None,
         render_js: bool | None,
         extract_images: bool | None,
+        mode: Literal["standard", "pro"] | None,
     ) -> dict[str, str | bool]:
         params: dict[str, str | bool] = {
             "url": url,
@@ -1659,6 +1669,8 @@ def _get_fetch_params(
             params["renderJs"] = render_js
         if extract_images is not None:
             params["extractImages"] = extract_images
+        if mode is not None:
+            params["mode"] = mode
         return params
 
     def _parse_search_response(

src/linkup/_types.py

@@ -233,6 +233,7 @@ class LinkupFetchTaskInput(_LinkupBaseModel):
             use include_raw_content instead.
         render_js: Whether JavaScript rendering should be enabled.
         extract_images: Whether image extraction should be enabled.
+        mode: The fetch strategy to use.
     """
 
     url: str
@@ -242,6 +243,7 @@ class LinkupFetchTaskInput(_LinkupBaseModel):
     include_raw_html: bool | None = pydantic.Field(default=None, validation_alias="includeRawHtml")
     render_js: bool | None = pydantic.Field(default=None, validation_alias="renderJs")
     extract_images: bool | None = pydantic.Field(default=None, validation_alias="extractImages")
+    mode: Literal["standard", "pro"] | None = None
 
 
 LinkupTaskInput = LinkupSearchTaskInput | LinkupFetchTaskInput | LinkupResearchTaskInput

tests/unit/client_test.py

@@ -987,6 +987,20 @@ def test_research_with_iso_datetime_string_dates(
             raw_html=None,
         ),
     ),
+    (
+        {"url": "https://example.com", "mode": "pro"},
+        {"url": "https://example.com", "mode": "pro"},
+        b"""
+        {
+            "favicon": "https://favicons.linkup.so?domain=example.com",
+            "markdown": "Some web page content"
+        }
+        """,
+        linkup.FetchResponse(
+            favicon="https://favicons.linkup.so?domain=example.com",
+            markdown="Some web page content",
+        ),
+    ),
 ]
 
 
@@ -1244,6 +1258,7 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None:
                     "input": {
                         "extractImages": true,
                         "includeRawContent": true,
+                        "mode": "pro",
                         "url": "https://example.com"
                     },
                     "output": {
@@ -1279,6 +1294,7 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None:
                 url="https://example.com",
                 extract_images=True,
                 include_raw_content=True,
+                mode="pro",
             ),
         ]
     )
@@ -1302,6 +1318,7 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None:
                     "url": "https://example.com",
                     "extractImages": True,
                     "includeRawContent": True,
+                    "mode": "pro",
                 },
             },
         ],
@@ -1311,6 +1328,7 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None:
     assert tasks_response[0].input.query == "query"
     assert tasks_response[0].input.structured_output_schema == {"type": "object"}
     assert isinstance(tasks_response[1], linkup.FetchTask)
+    assert tasks_response[1].input.mode == "pro"
     assert tasks_response[1].output is not None
     assert tasks_response[1].output.favicon == "https://favicons.linkup.so?domain=example.com"
     assert tasks_response[1].output.images is not None