Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ for the detailed list of available parameters.
#### 🪝 Fetch

The `fetch` function can be used to retrieve the content of a given web page in a cleaned up
markdown format.
markdown format, together with the website's favicon URL.

You can use the `render_js` flag to execute the JavaScript code of the page before returning the
content, and set `include_raw_content` to include the raw page content and its content type.
Expand All @@ -157,6 +157,7 @@ Which prints:
```bash
{
markdown="Get started for free, no credit card required...",
favicon="https://favicons.linkup.so?domain=docs.linkup.so",
raw_content="<!DOCTYPE html><html lang=\"en\"><head>...</head><body>...</body></html>",
content_type="html"
}
Expand Down
2 changes: 2 additions & 0 deletions src/linkup/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,13 +114,15 @@ class LinkupFetchResponse(_LinkupBaseModel):

Attributes:
markdown: The cleaned up markdown content.
favicon: The URL of the website favicon.
raw_content: The optional raw page content.
content_type: The type of the raw page content, if returned.
raw_html: The optional raw HTML content. Deprecated; use raw_content instead.
images: The optional list of extracted images.
"""

markdown: str
favicon: str
raw_content: str | None = pydantic.Field(default=None, validation_alias="rawContent")
content_type: str | None = pydantic.Field(default=None, validation_alias="contentType")
raw_html: str | None = pydantic.Field(default=None, validation_alias="rawHtml")
Expand Down
44 changes: 38 additions & 6 deletions tests/unit/client_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -910,8 +910,17 @@ def test_research_with_iso_datetime_string_dates(
(
{"url": "https://example.com"},
{"url": "https://example.com"},
b'{"markdown": "Some web page content"}',
linkup.FetchResponse(markdown="Some web page content", raw_html=None),
b"""
{
"favicon": "https://favicons.linkup.so?domain=example.com",
"markdown": "Some web page content"
}
""",
linkup.FetchResponse(
favicon="https://favicons.linkup.so?domain=example.com",
markdown="Some web page content",
raw_html=None,
),
),
(
{
Expand All @@ -924,12 +933,14 @@ def test_research_with_iso_datetime_string_dates(
},
b"""
{
"favicon": "https://favicons.linkup.so?domain=example.com",
"markdown": "# Some web page content",
"rawContent": "<html>...</html>",
"contentType": "html"
}
""",
linkup.FetchResponse(
favicon="https://favicons.linkup.so?domain=example.com",
markdown="# Some web page content",
raw_content="<html>...</html>",
content_type="html",
Expand All @@ -948,14 +959,33 @@ def test_research_with_iso_datetime_string_dates(
"renderJs": True,
"extractImages": True,
},
b'{"markdown": "#Some web page content", "rawHtml": "<html>...</html>"}',
linkup.FetchResponse(markdown="#Some web page content", raw_html="<html>...</html>"),
b"""
{
"favicon": "https://favicons.linkup.so?domain=example.com",
"markdown": "#Some web page content",
"rawHtml": "<html>...</html>"
}
""",
linkup.FetchResponse(
favicon="https://favicons.linkup.so?domain=example.com",
markdown="#Some web page content",
raw_html="<html>...</html>",
),
),
(
{"url": "https://example.com", "timeout": 15.0},
{"url": "https://example.com"},
b'{"markdown": "Some web page content"}',
linkup.FetchResponse(markdown="Some web page content", raw_html=None),
b"""
{
"favicon": "https://favicons.linkup.so?domain=example.com",
"markdown": "Some web page content"
}
""",
linkup.FetchResponse(
favicon="https://favicons.linkup.so?domain=example.com",
markdown="Some web page content",
raw_html=None,
),
),
]

Expand Down Expand Up @@ -1218,6 +1248,7 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None:
},
"output": {
"contentType": "html",
"favicon": "https://favicons.linkup.so?domain=example.com",
"images": [
{
"alt": "hero",
Expand Down Expand Up @@ -1281,6 +1312,7 @@ def test_create_tasks(mocker: MockerFixture, client: linkup.Client) -> None:
assert tasks_response[0].input.structured_output_schema == {"type": "object"}
assert isinstance(tasks_response[1], linkup.FetchTask)
assert tasks_response[1].output is not None
assert tasks_response[1].output.favicon == "https://favicons.linkup.so?domain=example.com"
assert tasks_response[1].output.images is not None
assert tasks_response[1].output.images[0].url == "https://example.com/image.png"
assert tasks_response[1].output.raw_content == "<html>Fetched content</html>"
Expand Down
Loading