From 1ad5de00c2ee9081b4eae25b1af46e68882f2cc9 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Wed, 21 Oct 2026 12:07:31 +0000 Subject: [PATCH] Count only network traffic in the pyppeteer and Selenium proxy meter Chrome's own chrome:// new-tab loads, data:/blob: URLs, cache hits and its error page were counted as proxy traffic. Only http(s) requests and responses that came over the network are counted now, and the pyppeteer budget block reaches the page's loop when the budget runs out on another thread. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CKsd7q25N19WzUZjrAdzx8 --- CHANGELOG.md | 4 +++ README.md | 2 +- proxy_meter.py | 86 ++++++++++++++++++++++++++++++++++++++++---------- smoke_test.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d8e2e2..3f84189 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,10 @@ nobody discovers it from a bill or a broken cron job. - CI only: the canary bills the `SLEEPNUMBER_PROXY` exit, so it now runs weekly (Mondays 11:23 UTC) instead of daily; a manual dispatch still runs it. +### Fixed + +- The pyppeteer and Selenium proxy meter counts only `http://` / `https://` requests and responses that came over the network. Chrome's own loads (the `chrome://` new-tab page, `data:`, `blob:`), cache hits (`requestServedFromCache`, `fromDiskCache`, `fromPrefetchCache`, `fromServiceWorker`) and Chrome's error page count nothing. A sibling repo's Selenium run had counted about 2.8 MB of new-tab scripts while its proxy carried about 6 KB. The pyppeteer block sent once the budget is spent now reaches the page's own event loop even when the budget runs out on another thread. + ## [0.3.0] — 2026-10-16 > **Behaviour changes:** diff --git a/README.md b/README.md index 5897389..dd60069 100644 --- a/README.md +++ b/README.md @@ -509,7 +509,7 @@ every finished or failed request; pyppeteer reads CDP `Network.loadingFinished.encodedDataLength` (bytes received, headers included) plus each request's line, headers and body from `Network.requestWillBeSent`; Selenium reads the same CDP events from -chromedriver's performance log around every page load; `http_scraper.py` +chromedriver's performance log around every page load; `http_scraper. Only `http(s)` requests and responses that came over the network are counted: Chrome's own `chrome://`, `data:` and `blob:` loads, cache hits and Chrome's error page count nothing.py` counts each response's status line, headers and body (its `Content-Length`, i.e. the compressed size, when the server sends one) plus the request it sent, for every redirect hop. The figure is an **estimate of the payload diff --git a/proxy_meter.py b/proxy_meter.py index b4eb4a6..2b47427 100644 --- a/proxy_meter.py +++ b/proxy_meter.py @@ -298,46 +298,88 @@ def _request_estimate(request: dict) -> int: class CdpCounter: """Turns CDP Network events into bytes on a meter. - A request is counted when it ends: `loadingFinished` adds the request - estimate plus `encodedDataLength` (every byte received for it, headers - included); `loadingFailed` adds the request estimate plus whatever - `dataReceived` had already delivered — unless the browser itself - blocked it (media blocking, the budget's own block), in which case - nothing left the machine. A redirect hop adds its own response's - `encodedDataLength` and its own request. + Only what crossed the network is counted: + + * a request is counted only for an `http://` / `https://` URL. The + browser's own loads (`chrome://` new-tab page scripts, `data:`, + `blob:`, extensions) never reach the proxy; measured on a sibling + repo's Selenium run, the new-tab page alone was ~2.8 MB of + `loadingFinished` events while the proxy carried ~6 KB; + * response bytes are counted only after a `responseReceived` that came + over the network: not a cache hit (`fromDiskCache`, + `fromPrefetchCache`, `fromServiceWorker`, or a + `requestServedFromCache` event, which also cancels the request's + own estimate), and not Chrome's own error page, which is reported as + a finished load of the failed URL with no `responseReceived`. + + `loadingFinished` then adds the request estimate plus + `encodedDataLength` (every byte received, headers included); + `loadingFailed` adds the request estimate plus what `dataReceived` had + delivered, unless the browser blocked the request itself (media + blocking, the budget's own block). A redirect hop adds its own + request and its response's `encodedDataLength`. """ def __init__(self, meter: ProxyMeter): self.meter = meter self._req: Dict[str, int] = {} self._recv: Dict[str, int] = {} + self._responded: Dict[str, bool] = {} + + @staticmethod + def _network(url) -> bool: + return str(url or "").startswith(("http://", "https://")) def feed(self, method: str, params: dict) -> None: + if not isinstance(params, dict): + return rid = params.get("requestId") if method == "Network.requestWillBeSent": redirect = params.get("redirectResponse") if redirect and rid in self._req: self.meter.add(self._req.pop(rid, 0) + int(redirect.get("encodedDataLength") or 0)) - self._recv.pop(rid, None) - self._req[rid] = _request_estimate(params.get("request") or {}) + self._recv.pop(rid, None) + self._responded.pop(rid, None) + request = params.get("request") or {} + if self._network(request.get("url")): + self._req[rid] = _request_estimate(request) + else: + self._req.pop(rid, None) + elif method == "Network.requestServedFromCache": + self._req.pop(rid, None) + self._responded[rid] = False + elif method == "Network.responseReceived": + response = params.get("response") or {} + cached = (response.get("fromDiskCache") + or response.get("fromPrefetchCache") + or response.get("fromServiceWorker")) + if cached: + self._req.pop(rid, None) + if self._responded.get(rid) is not False: + self._responded[rid] = bool( + self._network(response.get("url")) and not cached) elif method == "Network.dataReceived": - self._recv[rid] = (self._recv.get(rid, 0) - + int(params.get("encodedDataLength") or 0)) + if self._responded.get(rid): + self._recv[rid] = (self._recv.get(rid, 0) + + int(params.get("encodedDataLength") or 0)) elif method == "Network.loadingFinished": - got = int(params.get("encodedDataLength") or 0) - self.meter.add(self._req.pop(rid, 0) - + max(got, self._recv.pop(rid, 0))) + got = (int(params.get("encodedDataLength") or 0) + if self._responded.pop(rid, False) else 0) + sent = self._req.pop(rid, 0) + self.meter.add(sent + max(got, self._recv.pop(rid, 0))) elif method == "Network.loadingFailed": sent = self._req.pop(rid, 0) got = self._recv.pop(rid, 0) + self._responded.pop(rid, None) if params.get("blockedReason") or "BLOCKED_BY_CLIENT" in str( params.get("errorText", "")): return self.meter.add(sent + got) -EVENTS = ("Network.requestWillBeSent", "Network.dataReceived", +EVENTS = ("Network.requestWillBeSent", "Network.requestServedFromCache", + "Network.responseReceived", "Network.dataReceived", "Network.loadingFinished", "Network.loadingFailed") @@ -353,9 +395,19 @@ async def install_pyppeteer(page, meter: Optional[ProxyMeter]) -> None: session.on(name, (lambda n: lambda params: counter.feed(n, params))(name)) await session.send("Network.enable", {}) + # The budget can trip on another thread (a concurrent worker, or a + # thread that drains another engine's events) that has no event loop of + # its own: hand the block to the loop this page runs on. + loop = asyncio.get_event_loop() + + async def _send_block(): + try: + await session.send("Network.setBlockedURLs", {"urls": ["*"]}) + except Exception: # noqa: BLE001 — the next navigation's check still stops the run + pass + def _block(): - asyncio.ensure_future(session.send("Network.setBlockedURLs", - {"urls": ["*"]})) + loop.call_soon_threadsafe(lambda: asyncio.ensure_future(_send_block())) meter.on_exceed(_block) diff --git a/smoke_test.py b/smoke_test.py index e58108b..3f0b3d0 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2698,7 +2698,13 @@ def test_proxy_meter_units(skips): c = proxy_meter.CdpCounter(m) req = {"method": "GET", "url": "http://x.test/a", "headers": {"A": "b"}} est = len("GET http://x.test/a HTTP/1.1\r\n") + 2 + len("A: b\r\n") + + def _resp(rid, **extra): + return {"requestId": rid, + "response": dict({"url": "http://x.test/a", "status": 200}, + **extra)} c.feed("Network.requestWillBeSent", {"requestId": "1", "request": req}) + c.feed("Network.responseReceived", _resp("1")) c.feed("Network.dataReceived", {"requestId": "1", "encodedDataLength": 99}) c.feed("Network.loadingFinished", {"requestId": "1", "encodedDataLength": 1234}) @@ -2713,6 +2719,7 @@ def test_proxy_meter_units(skips): ok &= _savers_check("CDP: a request the browser blocked costs nothing", m.bytes == est + 1234) c.feed("Network.requestWillBeSent", {"requestId": "3", "request": req}) + c.feed("Network.responseReceived", _resp("3")) c.feed("Network.dataReceived", {"requestId": "3", "encodedDataLength": 50}) c.feed("Network.loadingFailed", {"requestId": "3", "errorText": "net::ERR_CONNECTION_RESET"}) @@ -2722,11 +2729,52 @@ def test_proxy_meter_units(skips): c.feed("Network.requestWillBeSent", { "requestId": "4", "request": req, "redirectResponse": {"encodedDataLength": 300}}) + c.feed("Network.responseReceived", _resp("4")) c.feed("Network.loadingFinished", {"requestId": "4", "encodedDataLength": 500}) ok &= _savers_check("CDP: a redirect hop is counted as its own exchange", m.bytes == 4 * est + 1234 + 50 + 300 + 500) + # Only what crossed the network. A sibling repo's Selenium run counted + # Chrome's own new-tab page (chrome:// scripts, ~2.8 MB of + # loadingFinished) while its proxy carried ~6 KB. + before = m.bytes + tab = {"method": "GET", "url": "chrome://new-tab-page/x.js", + "headers": {"A": "b"}} + c.feed("Network.requestWillBeSent", {"requestId": "t", "request": tab}) + c.feed("Network.responseReceived", {"requestId": "t", "response": { + "url": "chrome://new-tab-page/x.js", "status": 200}}) + c.feed("Network.dataReceived", {"requestId": "t", + "encodedDataLength": 2_779_683}) + c.feed("Network.loadingFinished", {"requestId": "t", + "encodedDataLength": 2_779_683}) + for scheme_url in ("data:text/plain,x", "blob:http://x.test/1"): + c.feed("Network.requestWillBeSent", {"requestId": scheme_url, + "request": dict(req, url=scheme_url)}) + c.feed("Network.loadingFinished", {"requestId": scheme_url, + "encodedDataLength": 900}) + ok &= _savers_check(f"CDP: a chrome:// new-tab load, data: and blob: cost nothing " + f"({m.bytes - before} B)", m.bytes == before) + c.feed("Network.requestWillBeSent", {"requestId": "k1", "request": req}) + c.feed("Network.requestServedFromCache", {"requestId": "k1"}) + c.feed("Network.responseReceived", _resp("k1")) + c.feed("Network.loadingFinished", {"requestId": "k1", + "encodedDataLength": 5000}) + for flag in ("fromDiskCache", "fromPrefetchCache", "fromServiceWorker"): + c.feed("Network.requestWillBeSent", {"requestId": flag, "request": req}) + c.feed("Network.responseReceived", _resp(flag, **{flag: True})) + c.feed("Network.loadingFinished", {"requestId": flag, + "encodedDataLength": 5000}) + ok &= _savers_check(f"CDP: a cache hit (requestServedFromCache, fromDiskCache, " + f"fromPrefetchCache, fromServiceWorker) costs nothing " + f"({m.bytes - before} B)", m.bytes == before) + c.feed("Network.requestWillBeSent", {"requestId": "e", "request": req}) + c.feed("Network.loadingFinished", {"requestId": "e", + "encodedDataLength": 250_000}) + ok &= _savers_check(f"CDP: Chrome's own error page (no responseReceived) costs only " + f"the request that was attempted ({m.bytes - before} B)", + m.bytes - before == est) + # -- Playwright: sizes() of every finished/failed request --------------- m = proxy_meter.ProxyMeter(0.001) ctx = mock.MagicMock() @@ -2765,6 +2813,8 @@ def test_proxy_meter_units(skips): log = [{"message": json.dumps({"message": { "method": "Network.requestWillBeSent", "params": {"requestId": "9", "request": req}}})}, + {"message": json.dumps({"message": { + "method": "Network.responseReceived", "params": _resp("9")}})}, {"message": json.dumps({"message": { "method": "Network.loadingFinished", "params": {"requestId": "9", "encodedDataLength": 2000}}})}, @@ -2796,6 +2846,7 @@ async def _drive(): await proxy_meter.install_pyppeteer(page, m) on = {call[0][0]: call[0][1] for call in cdp.on.call_args_list} on["Network.requestWillBeSent"]({"requestId": "p", "request": req}) + on["Network.responseReceived"](_resp("p")) on["Network.loadingFinished"]({"requestId": "p", "encodedDataLength": 1500}) await asyncio.sleep(0) @@ -2808,6 +2859,32 @@ async def _drive(): ok &= _savers_check("pyppeteer: over budget, every URL is blocked over CDP", ("Network.setBlockedURLs", {"urls": ["*"]}) in sent) + # The cap can be crossed on another thread (a concurrent worker) that has + # no event loop: the block must still reach this page's own loop. + import threading + m = proxy_meter.ProxyMeter(0.001) + cdp = mock.MagicMock() + cdp.send = mock.AsyncMock() + page = mock.MagicMock() + page.target.createCDPSession = mock.AsyncMock(return_value=cdp) + crossed = [] + + async def _drive_threaded(): + await proxy_meter.install_pyppeteer(page, m) + worker = threading.Thread(target=lambda: crossed.append( + _base_exception(lambda: m.add(5_000)))) + worker.start() + worker.join() + for _ in range(5): + await asyncio.sleep(0) + + asyncio.run(_drive_threaded()) + sent = [call[0] for call in cdp.send.await_args_list] + ok &= _savers_check(f"pyppeteer: a budget crossed on a worker thread still " + f"blocks every URL on the page's loop (thread raised " + f"{crossed})", crossed == [None] + and ("Network.setBlockedURLs", {"urls": ["*"]}) in sent) + # -- an HTTP client, through a real stub proxy with known sizes --------- import requests stub = _ByteCountingProxy({"/": 20_000})