diff --git a/src/pyscrappy/cli.py b/src/pyscrappy/cli.py index c18cd01..17b96f1 100644 --- a/src/pyscrappy/cli.py +++ b/src/pyscrappy/cli.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +import warnings def run_extract( @@ -22,10 +23,11 @@ def run_extract( - ``.md`` -> Markdown (``ScrapeResult.to_markdown``) - ``.json`` -> JSON (``ScrapeResult.to_json``) - ``.txt`` -> extracted page text - - ``.html`` -> raw page HTML + - ``.html`` -> page HTML (honors ``--render-js``) With ``--css-selector`` the matched elements' text is extracted instead of the - whole page. Returns a short status line for the CLI to print. + whole page (not applied to ``.html``; a warning is emitted). Returns a short + status line for the CLI to print. """ from pyscrappy import GenericScraper, scrape @@ -35,9 +37,19 @@ def run_extract( if ext == "html": # Raw HTML: the structured scrape result doesn't retain the source markup, - # so fetch the page's HTML directly. + # so fetch the page's HTML directly. Honor --render-js via the scraper's + # browser-aware fetch; keep the plain HTTP client when the flag is off. + if css_selector: + warnings.warn( + "--css-selector does not apply to .html output; writing the full page markup.", + UserWarning, + stacklevel=2, + ) with GenericScraper() as gs: - content = gs.http.get_html(url) + if render_js: + content = gs.fetch_html(url, render_js=True) + else: + content = gs.http.get_html(url) else: selectors = {"match": css_selector} if css_selector else None result = scrape(url, selectors=selectors, render_js=render_js) diff --git a/tests/test_cli/test_extract.py b/tests/test_cli/test_extract.py index 414cb66..6171501 100644 --- a/tests/test_cli/test_extract.py +++ b/tests/test_cli/test_extract.py @@ -56,7 +56,37 @@ def test_extract_html_fetches_raw_markup(tmp_path): fake_gs.__exit__.return_value = False fake_gs.http.get_html.return_value = "raw" with patch("pyscrappy.GenericScraper", return_value=fake_gs): - run_extract("http://x", str(out)) + run_extract("http://x", str(out), render_js=False) + assert out.read_text() == "raw" + fake_gs.http.get_html.assert_called_once_with("http://x") + fake_gs.fetch_html.assert_not_called() + + +def test_extract_html_render_js_uses_browser_path(tmp_path): + """``.html`` + ``render_js=True`` must use fetch_html, not plain HTTP.""" + out = tmp_path / "out.html" + fake_gs = MagicMock() + fake_gs.__enter__.return_value = fake_gs + fake_gs.__exit__.return_value = False + fake_gs.fetch_html.return_value = "rendered" + with patch("pyscrappy.GenericScraper", return_value=fake_gs): + with patch("pyscrappy.scrape") as scrape: + run_extract("http://x", str(out), render_js=True) + assert out.read_text() == "rendered" + fake_gs.fetch_html.assert_called_once_with("http://x", render_js=True) + fake_gs.http.get_html.assert_not_called() + scrape.assert_not_called() + + +def test_extract_html_css_selector_warns(tmp_path): + out = tmp_path / "out.html" + fake_gs = MagicMock() + fake_gs.__enter__.return_value = fake_gs + fake_gs.__exit__.return_value = False + fake_gs.http.get_html.return_value = "raw" + with patch("pyscrappy.GenericScraper", return_value=fake_gs): + with pytest.warns(UserWarning, match="css-selector"): + run_extract("http://x", str(out), css_selector=".product") assert out.read_text() == "raw" fake_gs.http.get_html.assert_called_once_with("http://x")