diff --git a/src/pyscrappy/core/scraper_api.py b/src/pyscrappy/core/scraper_api.py index 0bd8890..a083b3a 100644 --- a/src/pyscrappy/core/scraper_api.py +++ b/src/pyscrappy/core/scraper_api.py @@ -5,11 +5,13 @@ handles proxies and anti-bot challenges. This module maps a target URL into a request to such a service, given a ``scraper_api`` config. -Supported providers (all have free tiers): +Supported providers (the hosted services have free tiers): * ``scraperapi`` - https://www.scraperapi.com * ``scrapeops`` - https://scrapeops.io * ``scrapingbee`` - https://www.scrapingbee.com +* ``proxlane`` - https://github.com/proxlane/proxlane (self-hosted; + set ``endpoint`` in the config to your instance URL) Config shape:: @@ -41,6 +43,14 @@ "key_param": "api_key", "render_param": "render_js", }, + "proxlane": { + # Proxlane speaks ScraperAPI's parameter names, but it is self-hosted, + # so ``build_request`` honours an ``endpoint`` override in the config. + "endpoint": "http://localhost:8787/v1", + "url_param": "url", + "key_param": "api_key", + "render_param": "render", + }, } @@ -80,4 +90,5 @@ def build_request(target_url: str, scraper_api: dict[str, Any]) -> tuple[str, di if scraper_api.get("render_js"): params[spec["render_param"]] = "true" - return spec["endpoint"], params + endpoint = scraper_api.get("endpoint") or spec["endpoint"] + return endpoint, params diff --git a/tests/test_scraper_api_proxlane.py b/tests/test_scraper_api_proxlane.py new file mode 100644 index 0000000..7b81789 --- /dev/null +++ b/tests/test_scraper_api_proxlane.py @@ -0,0 +1,39 @@ +"""Tests for the Proxlane provider entry in the scraper-API mapping.""" + +from __future__ import annotations + +from pyscrappy.core.scraper_api import build_request, is_configured + + +def test_proxlane_uses_scraperapi_param_names() -> None: + _endpoint, params = build_request( + "https://example.com", + {"provider": "proxlane", "api_key": "KEY", "render_js": True}, + ) + assert params["api_key"] == "KEY" + assert params["url"] == "https://example.com" + assert params["render"] == "true" + + +def test_proxlane_endpoint_can_be_overridden() -> None: + endpoint, _ = build_request( + "https://example.com", + { + "provider": "proxlane", + "api_key": "KEY", + "endpoint": "http://proxlane.local:9000/", + }, + ) + assert endpoint == "http://proxlane.local:9000/" + + +def test_proxlane_missing_endpoint_falls_back_to_default() -> None: + endpoint, _ = build_request( + "https://example.com", + {"provider": "proxlane", "api_key": "KEY"}, + ) + assert endpoint == "http://localhost:8787/v1" + + +def test_proxlane_is_configured_with_key() -> None: + assert is_configured({"provider": "proxlane", "api_key": "KEY"})