diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d8e2e2..3f84189 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,10 @@ nobody discovers it from a bill or a broken cron job. - CI only: the canary bills the `SLEEPNUMBER_PROXY` exit, so it now runs weekly (Mondays 11:23 UTC) instead of daily; a manual dispatch still runs it. +### Fixed + +- The pyppeteer and Selenium proxy meter counts only `http://` / `https://` requests and responses that came over the network. Chrome's own loads (the `chrome://` new-tab page, `data:`, `blob:`), cache hits (`requestServedFromCache`, `fromDiskCache`, `fromPrefetchCache`, `fromServiceWorker`) and Chrome's error page count nothing. A sibling repo's Selenium run had counted about 2.8 MB of new-tab scripts while its proxy carried about 6 KB. The pyppeteer block sent once the budget is spent now reaches the page's own event loop even when the budget runs out on another thread. + ## [0.3.0] — 2026-10-16 > **Behaviour changes:** diff --git a/README.md b/README.md index 5897389..dd60069 100644 --- a/README.md +++ b/README.md @@ -509,7 +509,7 @@ every finished or failed request; pyppeteer reads CDP `Network.loadingFinished.encodedDataLength` (bytes received, headers included) plus each request's line, headers and body from `Network.requestWillBeSent`; Selenium reads the same CDP events from -chromedriver's performance log around every page load; `http_scraper.py` +chromedriver's performance log around every page load; `http_scraper. Only `http(s)` requests and responses that came over the network are counted: Chrome's own `chrome://`, `data:` and `blob:` loads, cache hits and Chrome's error page count nothing.py` counts each response's status line, headers and body (its `Content-Length`, i.e. the compressed size, when the server sends one) plus the request it sent, for every redirect hop. The figure is an **estimate of the payload diff --git a/proxy_meter.py b/proxy_meter.py index b4eb4a6..2b47427 100644 --- a/proxy_meter.py +++ b/proxy_meter.py @@ -298,46 +298,88 @@ def _request_estimate(request: dict) -> int: class CdpCounter: """Turns CDP Network events into bytes on a meter. - A request is counted when it ends: `loadingFinished` adds the request - estimate plus `encodedDataLength` (every byte received for it, headers - included); `loadingFailed` adds the request estimate plus whatever - `dataReceived` had already delivered — unless the browser itself - blocked it (media blocking, the budget's own block), in which case - nothing left the machine. A redirect hop adds its own response's - `encodedDataLength` and its own request. + Only what crossed the network is counted: + + * a request is counted only for an `http://` / `https://` URL. The + browser's own loads (`chrome://` new-tab page scripts, `data:`, + `blob:`, extensions) never reach the proxy; measured on a sibling + repo's Selenium run, the new-tab page alone was ~2.8 MB of + `loadingFinished` events while the proxy carried ~6 KB; + * response bytes are counted only after a `responseReceived` that came + over the network: not a cache hit (`fromDiskCache`, + `fromPrefetchCache`, `fromServiceWorker`, or a + `requestServedFromCache` event, which also cancels the request's + own estimate), and not Chrome's own error page, which is reported as + a finished load of the failed URL with no `responseReceived`. + + `loadingFinished` then adds the request estimate plus + `encodedDataLength` (every byte received, headers included); + `loadingFailed` adds the request estimate plus what `dataReceived` had + delivered, unless the browser blocked the request itself (media + blocking, the budget's own block). A redirect hop adds its own + request and its response's `encodedDataLength`. """ def __init__(self, meter: ProxyMeter): self.meter = meter self._req: Dict[str, int] = {} self._recv: Dict[str, int] = {} + self._responded: Dict[str, bool] = {} + + @staticmethod + def _network(url) -> bool: + return str(url or "").startswith(("http://", "https://")) def feed(self, method: str, params: dict) -> None: + if not isinstance(params, dict): + return rid = params.get("requestId") if method == "Network.requestWillBeSent": redirect = params.get("redirectResponse") if redirect and rid in self._req: self.meter.add(self._req.pop(rid, 0) + int(redirect.get("encodedDataLength") or 0)) - self._recv.pop(rid, None) - self._req[rid] = _request_estimate(params.get("request") or {}) + self._recv.pop(rid, None) + self._responded.pop(rid, None) + request = params.get("request") or {} + if self._network(request.get("url")): + self._req[rid] = _request_estimate(request) + else: + self._req.pop(rid, None) + elif method == "Network.requestServedFromCache": + self._req.pop(rid, None) + self._responded[rid] = False + elif method == "Network.responseReceived": + response = params.get("response") or {} + cached = (response.get("fromDiskCache") + or response.get("fromPrefetchCache") + or response.get("fromServiceWorker")) + if cached: + self._req.pop(rid, None) + if self._responded.get(rid) is not False: + self._responded[rid] = bool( + self._network(response.get("url")) and not cached) elif method == "Network.dataReceived": - self._recv[rid] = (self._recv.get(rid, 0) - + int(params.get("encodedDataLength") or 0)) + if self._responded.get(rid): + self._recv[rid] = (self._recv.get(rid, 0) + + int(params.get("encodedDataLength") or 0)) elif method == "Network.loadingFinished": - got = int(params.get("encodedDataLength") or 0) - self.meter.add(self._req.pop(rid, 0) - + max(got, self._recv.pop(rid, 0))) + got = (int(params.get("encodedDataLength") or 0) + if self._responded.pop(rid, False) else 0) + sent = self._req.pop(rid, 0) + self.meter.add(sent + max(got, self._recv.pop(rid, 0))) elif method == "Network.loadingFailed": sent = self._req.pop(rid, 0) got = self._recv.pop(rid, 0) + self._responded.pop(rid, None) if params.get("blockedReason") or "BLOCKED_BY_CLIENT" in str( params.get("errorText", "")): return self.meter.add(sent + got) -EVENTS = ("Network.requestWillBeSent", "Network.dataReceived", +EVENTS = ("Network.requestWillBeSent", "Network.requestServedFromCache", + "Network.responseReceived", "Network.dataReceived", "Network.loadingFinished", "Network.loadingFailed") @@ -353,9 +395,19 @@ async def install_pyppeteer(page, meter: Optional[ProxyMeter]) -> None: session.on(name, (lambda n: lambda params: counter.feed(n, params))(name)) await session.send("Network.enable", {}) + # The budget can trip on another thread (a concurrent worker, or a + # thread that drains another engine's events) that has no event loop of + # its own: hand the block to the loop this page runs on. + loop = asyncio.get_event_loop() + + async def _send_block(): + try: + await session.send("Network.setBlockedURLs", {"urls": ["*"]}) + except Exception: # noqa: BLE001 — the next navigation's check still stops the run + pass + def _block(): - asyncio.ensure_future(session.send("Network.setBlockedURLs", - {"urls": ["*"]})) + loop.call_soon_threadsafe(lambda: asyncio.ensure_future(_send_block())) meter.on_exceed(_block) diff --git a/smoke_test.py b/smoke_test.py index e58108b..3f0b3d0 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2698,7 +2698,13 @@ def test_proxy_meter_units(skips): c = proxy_meter.CdpCounter(m) req = {"method": "GET", "url": "http://x.test/a", "headers": {"A": "b"}} est = len("GET http://x.test/a HTTP/1.1\r\n") + 2 + len("A: b\r\n") + + def _resp(rid, **extra): + return {"requestId": rid, + "response": dict({"url": "http://x.test/a", "status": 200}, + **extra)} c.feed("Network.requestWillBeSent", {"requestId": "1", "request": req}) + c.feed("Network.responseReceived", _resp("1")) c.feed("Network.dataReceived", {"requestId": "1", "encodedDataLength": 99}) c.feed("Network.loadingFinished", {"requestId": "1", "encodedDataLength": 1234}) @@ -2713,6 +2719,7 @@ def test_proxy_meter_units(skips): ok &= _savers_check("CDP: a request the browser blocked costs nothing", m.bytes == est + 1234) c.feed("Network.requestWillBeSent", {"requestId": "3", "request": req}) + c.feed("Network.responseReceived", _resp("3")) c.feed("Network.dataReceived", {"requestId": "3", "encodedDataLength": 50}) c.feed("Network.loadingFailed", {"requestId": "3", "errorText": "net::ERR_CONNECTION_RESET"}) @@ -2722,11 +2729,52 @@ def test_proxy_meter_units(skips): c.feed("Network.requestWillBeSent", { "requestId": "4", "request": req, "redirectResponse": {"encodedDataLength": 300}}) + c.feed("Network.responseReceived", _resp("4")) c.feed("Network.loadingFinished", {"requestId": "4", "encodedDataLength": 500}) ok &= _savers_check("CDP: a redirect hop is counted as its own exchange", m.bytes == 4 * est + 1234 + 50 + 300 + 500) + # Only what crossed the network. A sibling repo's Selenium run counted + # Chrome's own new-tab page (chrome:// scripts, ~2.8 MB of + # loadingFinished) while its proxy carried ~6 KB. + before = m.bytes + tab = {"method": "GET", "url": "chrome://new-tab-page/x.js", + "headers": {"A": "b"}} + c.feed("Network.requestWillBeSent", {"requestId": "t", "request": tab}) + c.feed("Network.responseReceived", {"requestId": "t", "response": { + "url": "chrome://new-tab-page/x.js", "status": 200}}) + c.feed("Network.dataReceived", {"requestId": "t", + "encodedDataLength": 2_779_683}) + c.feed("Network.loadingFinished", {"requestId": "t", + "encodedDataLength": 2_779_683}) + for scheme_url in ("data:text/plain,x", "blob:http://x.test/1"): + c.feed("Network.requestWillBeSent", {"requestId": scheme_url, + "request": dict(req, url=scheme_url)}) + c.feed("Network.loadingFinished", {"requestId": scheme_url, + "encodedDataLength": 900}) + ok &= _savers_check(f"CDP: a chrome:// new-tab load, data: and blob: cost nothing " + f"({m.bytes - before} B)", m.bytes == before) + c.feed("Network.requestWillBeSent", {"requestId": "k1", "request": req}) + c.feed("Network.requestServedFromCache", {"requestId": "k1"}) + c.feed("Network.responseReceived", _resp("k1")) + c.feed("Network.loadingFinished", {"requestId": "k1", + "encodedDataLength": 5000}) + for flag in ("fromDiskCache", "fromPrefetchCache", "fromServiceWorker"): + c.feed("Network.requestWillBeSent", {"requestId": flag, "request": req}) + c.feed("Network.responseReceived", _resp(flag, **{flag: True})) + c.feed("Network.loadingFinished", {"requestId": flag, + "encodedDataLength": 5000}) + ok &= _savers_check(f"CDP: a cache hit (requestServedFromCache, fromDiskCache, " + f"fromPrefetchCache, fromServiceWorker) costs nothing " + f"({m.bytes - before} B)", m.bytes == before) + c.feed("Network.requestWillBeSent", {"requestId": "e", "request": req}) + c.feed("Network.loadingFinished", {"requestId": "e", + "encodedDataLength": 250_000}) + ok &= _savers_check(f"CDP: Chrome's own error page (no responseReceived) costs only " + f"the request that was attempted ({m.bytes - before} B)", + m.bytes - before == est) + # -- Playwright: sizes() of every finished/failed request --------------- m = proxy_meter.ProxyMeter(0.001) ctx = mock.MagicMock() @@ -2765,6 +2813,8 @@ def test_proxy_meter_units(skips): log = [{"message": json.dumps({"message": { "method": "Network.requestWillBeSent", "params": {"requestId": "9", "request": req}}})}, + {"message": json.dumps({"message": { + "method": "Network.responseReceived", "params": _resp("9")}})}, {"message": json.dumps({"message": { "method": "Network.loadingFinished", "params": {"requestId": "9", "encodedDataLength": 2000}}})}, @@ -2796,6 +2846,7 @@ async def _drive(): await proxy_meter.install_pyppeteer(page, m) on = {call[0][0]: call[0][1] for call in cdp.on.call_args_list} on["Network.requestWillBeSent"]({"requestId": "p", "request": req}) + on["Network.responseReceived"](_resp("p")) on["Network.loadingFinished"]({"requestId": "p", "encodedDataLength": 1500}) await asyncio.sleep(0) @@ -2808,6 +2859,32 @@ async def _drive(): ok &= _savers_check("pyppeteer: over budget, every URL is blocked over CDP", ("Network.setBlockedURLs", {"urls": ["*"]}) in sent) + # The cap can be crossed on another thread (a concurrent worker) that has + # no event loop: the block must still reach this page's own loop. + import threading + m = proxy_meter.ProxyMeter(0.001) + cdp = mock.MagicMock() + cdp.send = mock.AsyncMock() + page = mock.MagicMock() + page.target.createCDPSession = mock.AsyncMock(return_value=cdp) + crossed = [] + + async def _drive_threaded(): + await proxy_meter.install_pyppeteer(page, m) + worker = threading.Thread(target=lambda: crossed.append( + _base_exception(lambda: m.add(5_000)))) + worker.start() + worker.join() + for _ in range(5): + await asyncio.sleep(0) + + asyncio.run(_drive_threaded()) + sent = [call[0] for call in cdp.send.await_args_list] + ok &= _savers_check(f"pyppeteer: a budget crossed on a worker thread still " + f"blocks every URL on the page's loop (thread raised " + f"{crossed})", crossed == [None] + and ("Network.setBlockedURLs", {"urls": ["*"]}) in sent) + # -- an HTTP client, through a real stub proxy with known sizes --------- import requests stub = _ByteCountingProxy({"/": 20_000})