From cfd2a9b6d8608ff756d5985f0cc2e4319684755a Mon Sep 17 00:00:00 2001 From: David <244863912+dyk1454683243-sudo@users.noreply.github.com> Date: Sat, 19 Sep 2026 15:41:58 +0000 Subject: [PATCH 1/2] fix(cli): honor --render-js for .html extract output The html branch of run_extract always used HttpClient.get_html, so pyscrappy extract URL out.html --render-js wrote the un-rendered page. Use GenericScraper.fetch_html when render_js is true, keep http.get_html when false, and warn that --css-selector does not apply to .html. Co-authored-by: David --- src/pyscrappy/cli.py | 20 ++++++++++++++++---- tests/test_cli/test_extract.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/pyscrappy/cli.py b/src/pyscrappy/cli.py index c18cd01..17b96f1 100644 --- a/src/pyscrappy/cli.py +++ b/src/pyscrappy/cli.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +import warnings def run_extract( @@ -22,10 +23,11 @@ def run_extract( - ``.md`` -> Markdown (``ScrapeResult.to_markdown``) - ``.json`` -> JSON (``ScrapeResult.to_json``) - ``.txt`` -> extracted page text - - ``.html`` -> raw page HTML + - ``.html`` -> page HTML (honors ``--render-js``) With ``--css-selector`` the matched elements' text is extracted instead of the - whole page. Returns a short status line for the CLI to print. + whole page (not applied to ``.html``; a warning is emitted). Returns a short + status line for the CLI to print. """ from pyscrappy import GenericScraper, scrape @@ -35,9 +37,19 @@ def run_extract( if ext == "html": # Raw HTML: the structured scrape result doesn't retain the source markup, - # so fetch the page's HTML directly. + # so fetch the page's HTML directly. Honor --render-js via the scraper's + # browser-aware fetch; keep the plain HTTP client when the flag is off. + if css_selector: + warnings.warn( + "--css-selector does not apply to .html output; writing the full page markup.", + UserWarning, + stacklevel=2, + ) with GenericScraper() as gs: - content = gs.http.get_html(url) + if render_js: + content = gs.fetch_html(url, render_js=True) + else: + content = gs.http.get_html(url) else: selectors = {"match": css_selector} if css_selector else None result = scrape(url, selectors=selectors, render_js=render_js) diff --git a/tests/test_cli/test_extract.py b/tests/test_cli/test_extract.py index 414cb66..53a98b2 100644 --- a/tests/test_cli/test_extract.py +++ b/tests/test_cli/test_extract.py @@ -56,7 +56,38 @@ def test_extract_html_fetches_raw_markup(tmp_path): fake_gs.__exit__.return_value = False fake_gs.http.get_html.return_value = "raw" with patch("pyscrappy.GenericScraper", return_value=fake_gs): - run_extract("http://x", str(out)) + run_extract("http://x", str(out), render_js=False) + assert out.read_text() == "raw" + fake_gs.http.get_html.assert_called_once_with("http://x") + fake_gs.fetch_html.assert_not_called() + + +def test_extract_html_render_js_uses_browser_path(tmp_path): + """``.html`` + ``render_js=True`` must use fetch_html, not plain HTTP.""" + out = tmp_path / "out.html" + fake_gs = MagicMock() + fake_gs.__enter__.return_value = fake_gs + fake_gs.__exit__.return_value = False + fake_gs.fetch_html.return_value = "rendered" + with patch("pyscrappy.GenericScraper", return_value=fake_gs), patch( + "pyscrappy.scrape" + ) as scrape: + run_extract("http://x", str(out), render_js=True) + assert out.read_text() == "rendered" + fake_gs.fetch_html.assert_called_once_with("http://x", render_js=True) + fake_gs.http.get_html.assert_not_called() + scrape.assert_not_called() + + +def test_extract_html_css_selector_warns(tmp_path): + out = tmp_path / "out.html" + fake_gs = MagicMock() + fake_gs.__enter__.return_value = fake_gs + fake_gs.__exit__.return_value = False + fake_gs.http.get_html.return_value = "raw" + with patch("pyscrappy.GenericScraper", return_value=fake_gs): + with pytest.warns(UserWarning, match="css-selector"): + run_extract("http://x", str(out), css_selector=".product") assert out.read_text() == "raw" fake_gs.http.get_html.assert_called_once_with("http://x") From bd75e0238d53159a7c81919d235e897ae50cbdaf Mon Sep 17 00:00:00 2001 From: David <244863912+dyk1454683243-sudo@users.noreply.github.com> Date: Sat, 19 Sep 2026 15:43:09 +0000 Subject: [PATCH 2/2] style(cli): nest extract mocks for 3.9-safe ruff format Co-authored-by: David --- tests/test_cli/test_extract.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_cli/test_extract.py b/tests/test_cli/test_extract.py index 53a98b2..6171501 100644 --- a/tests/test_cli/test_extract.py +++ b/tests/test_cli/test_extract.py @@ -69,10 +69,9 @@ def test_extract_html_render_js_uses_browser_path(tmp_path): fake_gs.__enter__.return_value = fake_gs fake_gs.__exit__.return_value = False fake_gs.fetch_html.return_value = "rendered" - with patch("pyscrappy.GenericScraper", return_value=fake_gs), patch( - "pyscrappy.scrape" - ) as scrape: - run_extract("http://x", str(out), render_js=True) + with patch("pyscrappy.GenericScraper", return_value=fake_gs): + with patch("pyscrappy.scrape") as scrape: + run_extract("http://x", str(out), render_js=True) assert out.read_text() == "rendered" fake_gs.fetch_html.assert_called_once_with("http://x", render_js=True) fake_gs.http.get_html.assert_not_called()