Skip to content

fix(cli): honor --render-js for .html extract output #189

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vedaant00 merged 2 commits into from
Sep 21, 2026
+47 −5
Merged

fix(cli): honor --render-js for .html extract output #189

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cfd2a9b
fix(cli): honor --render-js for .html extract output
dyk1454683243-sudo Sep 19, 2026
bd75e02
style(cli): nest extract mocks for 3.9-safe ruff format
dyk1454683243-sudo Sep 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions src/pyscrappy/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from __future__ import annotations

import argparse
import warnings


def run_extract(
Expand All @@ -22,10 +23,11 @@ def run_extract(
- ``.md`` -> Markdown (``ScrapeResult.to_markdown``)
- ``.json`` -> JSON (``ScrapeResult.to_json``)
- ``.txt`` -> extracted page text
- ``.html`` -> raw page HTML
- ``.html`` -> page HTML (honors ``--render-js``)

With ``--css-selector`` the matched elements' text is extracted instead of the
whole page. Returns a short status line for the CLI to print.
whole page (not applied to ``.html``; a warning is emitted). Returns a short
status line for the CLI to print.
"""
from pyscrappy import GenericScraper, scrape

Expand All @@ -35,9 +37,19 @@ def run_extract(

if ext == "html":
# Raw HTML: the structured scrape result doesn't retain the source markup,
# so fetch the page's HTML directly.
# so fetch the page's HTML directly. Honor --render-js via the scraper's
# browser-aware fetch; keep the plain HTTP client when the flag is off.
if css_selector:
warnings.warn(
"--css-selector does not apply to .html output; writing the full page markup.",
UserWarning,
stacklevel=2,
)
with GenericScraper() as gs:
content = gs.http.get_html(url)
if render_js:
content = gs.fetch_html(url, render_js=True)
else:
content = gs.http.get_html(url)
else:
selectors = {"match": css_selector} if css_selector else None
result = scrape(url, selectors=selectors, render_js=render_js)
Expand Down
32 changes: 31 additions & 1 deletion tests/test_cli/test_extract.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,37 @@ def test_extract_html_fetches_raw_markup(tmp_path):
fake_gs.__exit__.return_value = False
fake_gs.http.get_html.return_value = "<html><body>raw</body></html>"
with patch("pyscrappy.GenericScraper", return_value=fake_gs):
run_extract("http://x", str(out))
run_extract("http://x", str(out), render_js=False)
assert out.read_text() == "<html><body>raw</body></html>"
fake_gs.http.get_html.assert_called_once_with("http://x")
fake_gs.fetch_html.assert_not_called()


def test_extract_html_render_js_uses_browser_path(tmp_path):
"""``.html`` + ``render_js=True`` must use fetch_html, not plain HTTP."""
out = tmp_path / "out.html"
fake_gs = MagicMock()
fake_gs.__enter__.return_value = fake_gs
fake_gs.__exit__.return_value = False
fake_gs.fetch_html.return_value = "<html><body>rendered</body></html>"
with patch("pyscrappy.GenericScraper", return_value=fake_gs):
with patch("pyscrappy.scrape") as scrape:
run_extract("http://x", str(out), render_js=True)
assert out.read_text() == "<html><body>rendered</body></html>"
fake_gs.fetch_html.assert_called_once_with("http://x", render_js=True)
fake_gs.http.get_html.assert_not_called()
scrape.assert_not_called()


def test_extract_html_css_selector_warns(tmp_path):
out = tmp_path / "out.html"
fake_gs = MagicMock()
fake_gs.__enter__.return_value = fake_gs
fake_gs.__exit__.return_value = False
fake_gs.http.get_html.return_value = "<html><body>raw</body></html>"
with patch("pyscrappy.GenericScraper", return_value=fake_gs):
with pytest.warns(UserWarning, match="css-selector"):
run_extract("http://x", str(out), css_selector=".product")
assert out.read_text() == "<html><body>raw</body></html>"
fake_gs.http.get_html.assert_called_once_with("http://x")

Expand Down
Loading