From 4d97742b698fddb09aaff9972a3adeb6feb97ccc Mon Sep 17 00:00:00 2001 From: cjumel Date: Thu, 27 Aug 2026 12:22:20 +0200 Subject: [PATCH 1/2] chore: remove cross field valition from SDK We want to leave this responsability to the API instead of duplicating this logic accross the SDKs. --- AGENTS.md | 8 +++++-- src/linkup/_client.py | 36 ++++++++++------------------ src/linkup/_types.py | 16 ------------- tests/unit/client_test.py | 50 --------------------------------------- tests/unit/types_test.py | 34 -------------------------- 5 files changed, 18 insertions(+), 126 deletions(-) delete mode 100644 tests/unit/types_test.py diff --git a/AGENTS.md b/AGENTS.md index 5eb221c..617c9cf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,8 +8,12 @@ Keep the SDK aligned with the current public, stable Linkup API while: - implementing a Pythonic public interface, - documenting the features through docstrings, -- and adding type safety through type hints or input data validation with `pydantic` models when - relevant. +- and adding type safety through type hints and `pydantic` models when relevant. + +However, **do not**: + +- duplicate API request-semantic validation in the SDK: SDK models may validate local data shape, + but the API owns cross-field, conditional, and product constraints. ## Working Rules diff --git a/src/linkup/_client.py b/src/linkup/_client.py index 4071663..bbb3f4b 100644 --- a/src/linkup/_client.py +++ b/src/linkup/_client.py @@ -261,10 +261,10 @@ def search( True Raises: - TypeError: If structured_output_schema is not provided or is not a string, dictionary, - or pydantic.BaseModel when output_type is "structured". - LinkupInvalidRequestError: If structured_output_schema doesn't represent a valid object - JSON schema when output_type is "structured". + TypeError: If structured_output_schema is not a string, dictionary, or + pydantic.BaseModel when provided. + LinkupInvalidRequestError: If the request parameters are invalid, including an invalid + or missing structured_output_schema when output_type is "structured". LinkupAuthenticationError: If the Linkup API key is invalid. LinkupInsufficientCreditError: If you have run out of credit. LinkupBudgetLimitExceededError: If the API key has reached its configured budget limit. @@ -458,10 +458,10 @@ async def async_search( True Raises: - TypeError: If structured_output_schema is not provided or is not a string, dictionary, - or pydantic.BaseModel when output_type is "structured". - LinkupInvalidRequestError: If structured_output_schema doesn't represent a valid object - JSON schema when output_type is "structured". + TypeError: If structured_output_schema is not a string, dictionary, or + pydantic.BaseModel when provided. + LinkupInvalidRequestError: If the request parameters are invalid, including an invalid + or missing structured_output_schema when output_type is "structured". LinkupAuthenticationError: If the Linkup API key is invalid. LinkupInsufficientCreditError: If you have run out of credit. LinkupBudgetLimitExceededError: If the API key has reached its configured budget limit. @@ -539,9 +539,8 @@ def research( The newly created research task, with "pending" status and no output. Raises: - TypeError: If structured_output_schema is not provided when output_type is - "structured", or if it is not a string, dictionary, or pydantic.BaseModel when - provided. + TypeError: If structured_output_schema is not a string, dictionary, or + pydantic.BaseModel when provided. LinkupInvalidRequestError: If the request parameters are invalid. LinkupAuthenticationError: If the Linkup API key is invalid. LinkupInsufficientCreditError: If you have run out of credit. @@ -612,9 +611,8 @@ async def async_research( The newly created research task, with "pending" status and no output. Raises: - TypeError: If structured_output_schema is not provided when output_type is - "structured", or if it is not a string, dictionary, or pydantic.BaseModel when - provided. + TypeError: If structured_output_schema is not a string, dictionary, or + pydantic.BaseModel when provided. LinkupInvalidRequestError: If the request parameters are invalid. LinkupAuthenticationError: If the Linkup API key is invalid. LinkupInsufficientCreditError: If you have run out of credit. @@ -1458,11 +1456,6 @@ def _get_search_params( include_inline_citations: bool | None, include_sources: bool | None, ) -> dict[str, str | bool | int | list[str]]: - if output_type == "structured" and structured_output_schema is None: - raise TypeError( - "structured_output_schema must be provided when output_type is 'structured'" - ) - params: dict[str, str | bool | int | list[str]] = { "q": query, "depth": depth, @@ -1512,11 +1505,6 @@ def _get_research_params( exclude_domains: list[str] | None, include_domains: list[str] | None, ) -> dict[str, str | bool | list[str]]: - if output_type == "structured" and structured_output_schema is None: - raise TypeError( - "structured_output_schema must be provided when output_type is 'structured'" - ) - params: dict[str, str | bool | list[str]] = { "q": query, "outputType": output_type, diff --git a/src/linkup/_types.py b/src/linkup/_types.py index 6197a07..3827132 100644 --- a/src/linkup/_types.py +++ b/src/linkup/_types.py @@ -170,14 +170,6 @@ class LinkupSearchTaskInput(_LinkupBaseModel): pydantic.Field(default=None, validation_alias="structuredOutputSchema") ) - @pydantic.model_validator(mode="after") - def _validate_structured_output_schema(self) -> LinkupSearchTaskInput: - if self.output_type == "structured" and self.structured_output_schema is None: - raise ValueError( - "structured_output_schema must be provided when output_type is 'structured'" - ) - return self - class LinkupResearchTaskInput(_LinkupBaseModel): """Input for creating or retrieving a research task. @@ -214,14 +206,6 @@ class LinkupResearchTaskInput(_LinkupBaseModel): pydantic.Field(default=None, validation_alias="structuredOutputSchema") ) - @pydantic.model_validator(mode="after") - def _validate_structured_output_schema(self) -> LinkupResearchTaskInput: - if self.output_type == "structured" and self.structured_output_schema is None: - raise ValueError( - "structured_output_schema must be provided when output_type is 'structured'" - ) - return self - class LinkupFetchTaskInput(_LinkupBaseModel): """Input for creating or retrieving a fetch task. diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index 1276f24..988d31d 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -385,18 +385,6 @@ def test_search_structured_output_model_dump_preserves_data( } -def test_search_structured_output_requires_schema(client: linkup.Client) -> None: - with pytest.raises( - TypeError, - match="structured_output_schema must be provided", - ): - client.search( - query="query", - depth="standard", - output_type="structured", - ) - - @pytest.mark.asyncio @pytest.mark.parametrize( ( @@ -434,19 +422,6 @@ async def test_async_search( assert search_response == expected_search_response -@pytest.mark.asyncio -async def test_async_search_structured_output_requires_schema(client: linkup.Client) -> None: - with pytest.raises( - TypeError, - match="structured_output_schema must be provided", - ): - await client.async_search( - query="query", - depth="standard", - output_type="structured", - ) - - test_search_error_parameters = [ ( 402, @@ -742,17 +717,6 @@ def test_research(mocker: MockerFixture, client: linkup.Client) -> None: ) -def test_research_structured_output_requires_schema(client: linkup.Client) -> None: - with pytest.raises( - TypeError, - match="structured_output_schema must be provided", - ): - client.research( - query="query", - output_type="structured", - ) - - def test_get_research_structured_output_keeps_sourced_answer_shape_raw( mocker: MockerFixture, client: linkup.Client ) -> None: @@ -843,20 +807,6 @@ async def test_async_research(mocker: MockerFixture, client: linkup.Client) -> N assert research_response.input.structured_output_schema == {"type": "object"} -@pytest.mark.asyncio -async def test_async_research_structured_output_requires_schema( - client: linkup.Client, -) -> None: - with pytest.raises( - TypeError, - match="structured_output_schema must be provided", - ): - await client.async_research( - query="query", - output_type="structured", - ) - - def test_research_with_iso_datetime_string_dates( mocker: MockerFixture, client: linkup.Client ) -> None: diff --git a/tests/unit/types_test.py b/tests/unit/types_test.py deleted file mode 100644 index 7369bd6..0000000 --- a/tests/unit/types_test.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import Any - -import pydantic -import pytest - -import linkup - - -@pytest.mark.parametrize( - "task_input", - [ - pytest.param( - lambda: linkup.SearchTaskInput( - query="query", - depth="standard", - output_type="structured", - ), - id="search", - ), - pytest.param( - lambda: linkup.ResearchTaskInput( - query="query", - output_type="structured", - ), - id="research", - ), - ], -) -def test_structured_task_input_requires_schema(task_input: Any) -> None: # noqa: ANN401 - with pytest.raises( - pydantic.ValidationError, - match="structured_output_schema must be provided", - ): - task_input() From 7170ea219f4720bd2cd7a13b7c110a7742006ac8 Mon Sep 17 00:00:00 2001 From: cjumel Date: Thu, 27 Aug 2026 12:35:44 +0200 Subject: [PATCH 2/2] feat: support structured fetch extraction --- README.md | 17 ++++++++++++----- src/linkup/_client.py | 39 +++++++++++++++++++++++++++++++++++---- src/linkup/_types.py | 8 ++++++++ tests/unit/client_test.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 32ce772..def4361 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ markdown format, together with the website's favicon URL. You can use the `render_js` flag to execute the JavaScript code of the page before returning the content, set `include_raw_content` to include the raw page content and its content type in the output, or set `mode` to `"pro"` for significantly higher success rates on hard-to-retrieve pages. +You can also pass an object JSON `schema`, with optional `instructions`, to extract structured data. ```python import linkup @@ -149,18 +150,24 @@ fetch_response: linkup.FetchResponse = client.fetch( render_js=True, include_raw_content=True, mode="pro", + schema={ + "type": "object", + "properties": {"title": {"type": "string"}}, + }, + instructions="Extract the page title.", ) print(fetch_response.model_dump()) ``` Which prints: -```bash +```python { - markdown="Get started for free, no credit card required...", - favicon="https://favicons.linkup.so?domain=docs.linkup.so", - raw_content="......", - content_type="html" + "markdown": "The production-grade web search API for AI.", + "favicon": "https://favicons.linkup.so?domain=docs.linkup.so", + "raw_content": "......", + "content_type": "html", + "data": {"title": "The production-grade web search API for AI."}, } ``` diff --git a/src/linkup/_client.py b/src/linkup/_client.py index bbb3f4b..ef1797b 100644 --- a/src/linkup/_client.py +++ b/src/linkup/_client.py @@ -1003,6 +1003,8 @@ def fetch( timeout: float | None = None, include_raw_content: bool | None = None, mode: Literal["standard", "pro"] | None = None, + schema: type[pydantic.BaseModel] | dict[str, Any] | str | None = None, + instructions: str | None = None, ) -> LinkupFetchResponse: """Fetch the content of a web page using the Linkup API /fetch endpoint. @@ -1023,6 +1025,10 @@ def fetch( response. mode: The fetch strategy to use. "pro" delivers significantly higher success rates on hard-to-retrieve pages. + schema: An object JSON schema describing an optional structured data to extract. + Supported formats are a pydantic.BaseModel, a Python dictionary, or a JSON string. + instructions: Optional instructions guiding structured data extraction if schema is + passed. Returns: The response of the web page fetch, containing the web page content. @@ -1037,13 +1043,15 @@ def fetch( type. LinkupTimeoutError: If the request times out. """ - params: dict[str, str | bool] = self._get_fetch_params( + params: dict[str, Any] = self._get_fetch_params( url=url, include_raw_html=include_raw_html, include_raw_content=include_raw_content, render_js=render_js, extract_images=extract_images, mode=mode, + schema=schema, + instructions=instructions, ) response: httpx.Response = self._request( @@ -1064,6 +1072,8 @@ async def async_fetch( timeout: float | None = None, include_raw_content: bool | None = None, mode: Literal["standard", "pro"] | None = None, + schema: type[pydantic.BaseModel] | dict[str, Any] | str | None = None, + instructions: str | None = None, ) -> LinkupFetchResponse: """Asynchronously fetch the content of a web page using the Linkup API /fetch endpoint. @@ -1084,6 +1094,10 @@ async def async_fetch( response. mode: The fetch strategy to use. "pro" delivers significantly higher success rates on hard-to-retrieve pages. + schema: An object JSON schema describing an optional structured data to extract. + Supported formats are a pydantic.BaseModel, a Python dictionary, or a JSON string. + instructions: Optional instructions guiding structured data extraction if schema is + passed. Returns: The response of the web page fetch, containing the web page content. @@ -1098,13 +1112,15 @@ async def async_fetch( type. LinkupTimeoutError: If the request times out. """ - params: dict[str, str | bool] = self._get_fetch_params( + params: dict[str, Any] = self._get_fetch_params( url=url, include_raw_html=include_raw_html, include_raw_content=include_raw_content, render_js=render_js, extract_images=extract_images, mode=mode, + schema=schema, + instructions=instructions, ) response: httpx.Response = await self._async_request( @@ -1618,6 +1634,8 @@ def _get_tasks_payload(self, tasks: list[LinkupTaskInput]) -> list[dict[str, Any render_js=task.render_js, extract_images=task.extract_images, mode=task.mode, + schema=task.schema_, + instructions=task.instructions, ), } ) @@ -1654,8 +1672,10 @@ def _get_fetch_params( render_js: bool | None, extract_images: bool | None, mode: Literal["standard", "pro"] | None, - ) -> dict[str, str | bool]: - params: dict[str, str | bool] = { + schema: type[pydantic.BaseModel] | str | dict[str, Any] | None, + instructions: str | None, + ) -> dict[str, Any]: + params: dict[str, Any] = { "url": url, } if include_raw_html is not None: @@ -1668,6 +1688,17 @@ def _get_fetch_params( params["extractImages"] = extract_images if mode is not None: params["mode"] = mode + if schema is not None: + if isinstance(schema, str): + params["schema"] = schema + elif isinstance(schema, dict): + params["schema"] = json.dumps(schema) + elif issubclass(schema, pydantic.BaseModel): + params["schema"] = json.dumps(schema.model_json_schema()) + else: + raise TypeError(f"Unexpected schema type: '{type(schema)}'") + if instructions is not None: + params["instructions"] = instructions return params def _parse_search_response( diff --git a/src/linkup/_types.py b/src/linkup/_types.py index 3827132..bc88bee 100644 --- a/src/linkup/_types.py +++ b/src/linkup/_types.py @@ -119,6 +119,7 @@ class LinkupFetchResponse(_LinkupBaseModel): content_type: The type of the raw page content, if returned. raw_html: The optional raw HTML content. Deprecated; use raw_content instead. images: The optional list of extracted images. + data: Structured data extracted from the webpage, if a schema was provided. """ markdown: str @@ -127,6 +128,7 @@ class LinkupFetchResponse(_LinkupBaseModel): content_type: str | None = pydantic.Field(default=None, validation_alias="contentType") raw_html: str | None = pydantic.Field(default=None, validation_alias="rawHtml") images: list[LinkupFetchImageExtraction] | None = pydantic.Field(default=None) + data: JSONObject | None = None class LinkupSearchTaskInput(_LinkupBaseModel): @@ -218,6 +220,8 @@ class LinkupFetchTaskInput(_LinkupBaseModel): render_js: Whether JavaScript rendering should be enabled. extract_images: Whether image extraction should be enabled. mode: The fetch strategy to use. + schema_: The object JSON schema describing the data to extract, if any. + instructions: Instructions guiding structured data extraction, if any. """ url: str @@ -228,6 +232,10 @@ class LinkupFetchTaskInput(_LinkupBaseModel): render_js: bool | None = pydantic.Field(default=None, validation_alias="renderJs") extract_images: bool | None = pydantic.Field(default=None, validation_alias="extractImages") mode: Literal["standard", "pro"] | None = None + schema_: type[pydantic.BaseModel] | str | dict[str, Any] | None = pydantic.Field( + default=None, validation_alias="schema" + ) + instructions: str | None = None LinkupTaskInput = LinkupSearchTaskInput | LinkupFetchTaskInput | LinkupResearchTaskInput diff --git a/tests/unit/client_test.py b/tests/unit/client_test.py index 988d31d..e46d253 100644 --- a/tests/unit/client_test.py +++ b/tests/unit/client_test.py @@ -951,6 +951,30 @@ def test_research_with_iso_datetime_string_dates( markdown="Some web page content", ), ), + ( + { + "url": "https://example.com", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, + "instructions": "Extract the product name.", + }, + { + "url": "https://example.com", + "schema": json.dumps({"type": "object", "properties": {"name": {"type": "string"}}}), + "instructions": "Extract the product name.", + }, + b""" + { + "data": {"name": "Example product"}, + "favicon": "https://favicons.linkup.so?domain=example.com", + "markdown": "Example product" + } + """, + linkup.FetchResponse( + data={"name": "Example product"}, + favicon="https://favicons.linkup.so?domain=example.com", + markdown="Example product", + ), + ), ] @@ -1226,6 +1250,9 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None: }, "output": { "contentType": "html", + "data": { + "name": "Example product" + }, "favicon": "https://favicons.linkup.so?domain=example.com", "images": [ { @@ -1257,6 +1284,8 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None: url="https://example.com", extract_images=True, include_raw_content=True, + schema_={"type": "object", "properties": {"name": {"type": "string"}}}, + instructions="Extract the product name.", mode="pro", ), ] @@ -1281,6 +1310,10 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None: "url": "https://example.com", "extractImages": True, "includeRawContent": True, + "schema": json.dumps( + {"type": "object", "properties": {"name": {"type": "string"}}} + ), + "instructions": "Extract the product name.", "mode": "pro", }, }, @@ -1298,6 +1331,7 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None: assert tasks_response[1].output.images[0].url == "https://example.com/image.png" assert tasks_response[1].output.raw_content == "Fetched content" assert tasks_response[1].output.content_type == "html" + assert tasks_response[1].output.data == {"name": "Example product"} def test_create_tasks_research_model(mocker: MockerFixture, client: linkup.Client) -> None: