From 82924ee7643fc280c01d9250a7d577b56649c021 Mon Sep 17 00:00:00 2001 From: mercael Date: Fri, 18 Sep 2026 15:16:51 +0300 Subject: [PATCH 1/6] Add Proxlane provider with configurable endpoint --- src/pyscrappy/core/scraper_api.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/pyscrappy/core/scraper_api.py b/src/pyscrappy/core/scraper_api.py index 0bd8890..d19fe94 100644 --- a/src/pyscrappy/core/scraper_api.py +++ b/src/pyscrappy/core/scraper_api.py @@ -10,6 +10,8 @@ * ``scraperapi`` - https://www.scraperapi.com * ``scrapeops`` - https://scrapeops.io * ``scrapingbee`` - https://www.scrapingbee.com +* ``proxlane`` - https://github.com/proxlane/proxlane (self-hosted; + set ``endpoint`` in the config to your instance URL) Config shape:: @@ -41,6 +43,14 @@ "key_param": "api_key", "render_param": "render_js", }, + "proxlane": { + # Proxlane speaks ScraperAPI's parameter names, but it is self-hosted, + # so ``build_request`` honours an ``endpoint`` override in the config. + "endpoint": "http://localhost:8000/", + "url_param": "url", + "key_param": "api_key", + "render_param": "render", + }, } @@ -80,4 +90,5 @@ def build_request(target_url: str, scraper_api: dict[str, Any]) -> tuple[str, di if scraper_api.get("render_js"): params[spec["render_param"]] = "true" - return spec["endpoint"], params + endpoint = scraper_api.get("endpoint") or spec["endpoint"] + return endpoint, params From bc21d099db49f47bb614662a07a556d69b847dd8 Mon Sep 17 00:00:00 2001 From: mercael Date: Fri, 18 Sep 2026 15:16:52 +0300 Subject: [PATCH 2/6] Add Proxlane provider with configurable endpoint --- tests/test_scraper_api_proxlane.py | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/test_scraper_api_proxlane.py diff --git a/tests/test_scraper_api_proxlane.py b/tests/test_scraper_api_proxlane.py new file mode 100644 index 0000000..5cf9093 --- /dev/null +++ b/tests/test_scraper_api_proxlane.py @@ -0,0 +1,39 @@ +"""Tests for the Proxlane provider entry in the scraper-API mapping.""" + +from __future__ import annotations + +from pyscrappy.core.scraper_api import build_request, is_configured + + +def test_proxlane_uses_scraperapi_param_names() -> None: + endpoint, params = build_request( + "https://example.com", + {"provider": "proxlane", "api_key": "KEY", "render_js": True}, + ) + assert params["api_key"] == "KEY" + assert params["url"] == "https://example.com" + assert params["render"] == "true" + + +def test_proxlane_endpoint_can_be_overridden() -> None: + endpoint, _ = build_request( + "https://example.com", + { + "provider": "proxlane", + "api_key": "KEY", + "endpoint": "http://proxlane.local:9000/", + }, + ) + assert endpoint == "http://proxlane.local:9000/" + + +def test_proxlane_missing_endpoint_falls_back_to_default() -> None: + endpoint, _ = build_request( + "https://example.com", + {"provider": "proxlane", "api_key": "KEY"}, + ) + assert endpoint == "http://localhost:8000/" + + +def test_proxlane_is_configured_with_key() -> None: + assert is_configured({"provider": "proxlane", "api_key": "KEY"}) From e06ecc057ded543098957f10e30bc9da8c9788cb Mon Sep 17 00:00:00 2001 From: mercael Date: Fri, 18 Sep 2026 15:34:13 +0300 Subject: [PATCH 3/6] fix: use the proxlane default endpoint from its docs (localhost:8787/v1) --- src/pyscrappy/core/scraper_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyscrappy/core/scraper_api.py b/src/pyscrappy/core/scraper_api.py index d19fe94..89fe0ce 100644 --- a/src/pyscrappy/core/scraper_api.py +++ b/src/pyscrappy/core/scraper_api.py @@ -46,7 +46,7 @@ "proxlane": { # Proxlane speaks ScraperAPI's parameter names, but it is self-hosted, # so ``build_request`` honours an ``endpoint`` override in the config. - "endpoint": "http://localhost:8000/", + "endpoint": "http://localhost:8787/v1", "url_param": "url", "key_param": "api_key", "render_param": "render", From f3486a3e1cfa3532dd113608e4c00048d142b109 Mon Sep 17 00:00:00 2001 From: mercael Date: Fri, 18 Sep 2026 15:34:14 +0300 Subject: [PATCH 4/6] test: assert the documented proxlane default endpoint --- tests/test_scraper_api_proxlane.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_scraper_api_proxlane.py b/tests/test_scraper_api_proxlane.py index 5cf9093..576ec3c 100644 --- a/tests/test_scraper_api_proxlane.py +++ b/tests/test_scraper_api_proxlane.py @@ -32,7 +32,7 @@ def test_proxlane_missing_endpoint_falls_back_to_default() -> None: "https://example.com", {"provider": "proxlane", "api_key": "KEY"}, ) - assert endpoint == "http://localhost:8000/" + assert endpoint == "http://localhost:8787/v1" def test_proxlane_is_configured_with_key() -> None: From 595d259ed9f3121da2d03d8393e82f63b684fc8b Mon Sep 17 00:00:00 2001 From: mercael Date: Fri, 18 Sep 2026 16:22:40 +0300 Subject: [PATCH 5/6] test: ignore the unused endpoint var (ruff F841) --- tests/test_scraper_api_proxlane.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_scraper_api_proxlane.py b/tests/test_scraper_api_proxlane.py index 576ec3c..7b81789 100644 --- a/tests/test_scraper_api_proxlane.py +++ b/tests/test_scraper_api_proxlane.py @@ -6,7 +6,7 @@ def test_proxlane_uses_scraperapi_param_names() -> None: - endpoint, params = build_request( + _endpoint, params = build_request( "https://example.com", {"provider": "proxlane", "api_key": "KEY", "render_js": True}, ) From 58e2f3fc4f6511a630000b58b05be7f0c31d01f9 Mon Sep 17 00:00:00 2001 From: mercael Date: Fri, 18 Sep 2026 16:22:41 +0300 Subject: [PATCH 6/6] docs: proxlane is self-hosted, not a free-tier service --- src/pyscrappy/core/scraper_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyscrappy/core/scraper_api.py b/src/pyscrappy/core/scraper_api.py index 89fe0ce..a083b3a 100644 --- a/src/pyscrappy/core/scraper_api.py +++ b/src/pyscrappy/core/scraper_api.py @@ -5,7 +5,7 @@ handles proxies and anti-bot challenges. This module maps a target URL into a request to such a service, given a ``scraper_api`` config. -Supported providers (all have free tiers): +Supported providers (the hosted services have free tiers): * ``scraperapi`` - https://www.scraperapi.com * ``scrapeops`` - https://scrapeops.io