Count only network traffic in the pyppeteer and Selenium proxy meter

CHANGELOG.md

@@ -74,6 +74,10 @@ nobody discovers it from a bill or a broken cron job.
 - CI only: the canary bills the `SLEEPNUMBER_PROXY` exit, so it now runs weekly
   (Mondays 11:23 UTC) instead of daily; a manual dispatch still runs it.
 
+### Fixed
+
+- The pyppeteer and Selenium proxy meter counts only `http://` / `https://` requests and responses that came over the network. Chrome's own loads (the `chrome://` new-tab page, `data:`, `blob:`), cache hits (`requestServedFromCache`, `fromDiskCache`, `fromPrefetchCache`, `fromServiceWorker`) and Chrome's error page count nothing. A sibling repo's Selenium run had counted about 2.8 MB of new-tab scripts while its proxy carried about 6 KB. The pyppeteer block sent once the budget is spent now reaches the page's own event loop even when the budget runs out on another thread.
+
 ## [0.3.0] — 2026-10-16
 
 > **Behaviour changes:**

README.md

@@ -509,7 +509,7 @@ every finished or failed request; pyppeteer reads CDP
 `Network.loadingFinished.encodedDataLength` (bytes received, headers
 included) plus each request's line, headers and body from
 `Network.requestWillBeSent`; Selenium reads the same CDP events from
-chromedriver's performance log around every page load; `http_scraper.py`
+chromedriver's performance log around every page load; `http_scraper. Only `http(s)` requests and responses that came over the network are counted: Chrome's own `chrome://`, `data:` and `blob:` loads, cache hits and Chrome's error page count nothing.py`
 counts each response's status line, headers and body (its `Content-Length`,
 i.e. the compressed size, when the server sends one) plus the request it
 sent, for every redirect hop. The figure is an **estimate of the payload

proxy_meter.py

@@ -298,46 +298,88 @@ def _request_estimate(request: dict) -> int:
 class CdpCounter:
     """Turns CDP Network events into bytes on a meter.
 
-    A request is counted when it ends: `loadingFinished` adds the request
-    estimate plus `encodedDataLength` (every byte received for it, headers
-    included); `loadingFailed` adds the request estimate plus whatever
-    `dataReceived` had already delivered — unless the browser itself
-    blocked it (media blocking, the budget's own block), in which case
-    nothing left the machine. A redirect hop adds its own response's
-    `encodedDataLength` and its own request.
+    Only what crossed the network is counted:
+
+    * a request is counted only for an `http://` / `https://` URL. The
+      browser's own loads (`chrome://` new-tab page scripts, `data:`,
+      `blob:`, extensions) never reach the proxy; measured on a sibling
+      repo's Selenium run, the new-tab page alone was ~2.8 MB of
+      `loadingFinished` events while the proxy carried ~6 KB;
+    * response bytes are counted only after a `responseReceived` that came
+      over the network: not a cache hit (`fromDiskCache`,
+      `fromPrefetchCache`, `fromServiceWorker`, or a
+      `requestServedFromCache` event, which also cancels the request's
+      own estimate), and not Chrome's own error page, which is reported as
+      a finished load of the failed URL with no `responseReceived`.
+
+    `loadingFinished` then adds the request estimate plus
+    `encodedDataLength` (every byte received, headers included);
+    `loadingFailed` adds the request estimate plus what `dataReceived` had
+    delivered, unless the browser blocked the request itself (media
+    blocking, the budget's own block). A redirect hop adds its own
+    request and its response's `encodedDataLength`.
     """
 
     def __init__(self, meter: ProxyMeter):
         self.meter = meter
         self._req: Dict[str, int] = {}
         self._recv: Dict[str, int] = {}
+        self._responded: Dict[str, bool] = {}
+
+    @staticmethod
+    def _network(url) -> bool:
+        return str(url or "").startswith(("http://", "https://"))
 
     def feed(self, method: str, params: dict) -> None:
+        if not isinstance(params, dict):
+            return
         rid = params.get("requestId")
         if method == "Network.requestWillBeSent":
             redirect = params.get("redirectResponse")
             if redirect and rid in self._req:
                 self.meter.add(self._req.pop(rid, 0)
                                + int(redirect.get("encodedDataLength") or 0))
-                self._recv.pop(rid, None)
-            self._req[rid] = _request_estimate(params.get("request") or {})
+            self._recv.pop(rid, None)
+            self._responded.pop(rid, None)
+            request = params.get("request") or {}
+            if self._network(request.get("url")):
+                self._req[rid] = _request_estimate(request)
+            else:
+                self._req.pop(rid, None)
+        elif method == "Network.requestServedFromCache":
+            self._req.pop(rid, None)
+            self._responded[rid] = False
+        elif method == "Network.responseReceived":
+            response = params.get("response") or {}
+            cached = (response.get("fromDiskCache")
+                      or response.get("fromPrefetchCache")
+                      or response.get("fromServiceWorker"))
+            if cached:
+                self._req.pop(rid, None)
+            if self._responded.get(rid) is not False:
+                self._responded[rid] = bool(
+                    self._network(response.get("url")) and not cached)
         elif method == "Network.dataReceived":
-            self._recv[rid] = (self._recv.get(rid, 0)
-                               + int(params.get("encodedDataLength") or 0))
+            if self._responded.get(rid):
+                self._recv[rid] = (self._recv.get(rid, 0)
+                                   + int(params.get("encodedDataLength") or 0))
         elif method == "Network.loadingFinished":
-            got = int(params.get("encodedDataLength") or 0)
-            self.meter.add(self._req.pop(rid, 0)
-                           + max(got, self._recv.pop(rid, 0)))
+            got = (int(params.get("encodedDataLength") or 0)
+                   if self._responded.pop(rid, False) else 0)
+            sent = self._req.pop(rid, 0)
+            self.meter.add(sent + max(got, self._recv.pop(rid, 0)))
         elif method == "Network.loadingFailed":
             sent = self._req.pop(rid, 0)
             got = self._recv.pop(rid, 0)
+            self._responded.pop(rid, None)
             if params.get("blockedReason") or "BLOCKED_BY_CLIENT" in str(
                     params.get("errorText", "")):
                 return
             self.meter.add(sent + got)
 
 
-EVENTS = ("Network.requestWillBeSent", "Network.dataReceived",
+EVENTS = ("Network.requestWillBeSent", "Network.requestServedFromCache",
+          "Network.responseReceived", "Network.dataReceived",
           "Network.loadingFinished", "Network.loadingFailed")
 
 
@@ -353,9 +395,19 @@ async def install_pyppeteer(page, meter: Optional[ProxyMeter]) -> None:
         session.on(name, (lambda n: lambda params: counter.feed(n, params))(name))
     await session.send("Network.enable", {})
 
+    # The budget can trip on another thread (a concurrent worker, or a
+    # thread that drains another engine's events) that has no event loop of
+    # its own: hand the block to the loop this page runs on.
+    loop = asyncio.get_event_loop()
+
+    async def _send_block():
+        try:
+            await session.send("Network.setBlockedURLs", {"urls": ["*"]})
+        except Exception:  # noqa: BLE001 — the next navigation's check still stops the run
+            pass
+
     def _block():
-        asyncio.ensure_future(session.send("Network.setBlockedURLs",
-                                           {"urls": ["*"]}))
+        loop.call_soon_threadsafe(lambda: asyncio.ensure_future(_send_block()))
     meter.on_exceed(_block)
 
 

smoke_test.py

@@ -2698,7 +2698,13 @@ def test_proxy_meter_units(skips):
     c = proxy_meter.CdpCounter(m)
     req = {"method": "GET", "url": "http://x.test/a", "headers": {"A": "b"}}
     est = len("GET http://x.test/a HTTP/1.1\r\n") + 2 + len("A: b\r\n")
+
+    def _resp(rid, **extra):
+        return {"requestId": rid,
+                "response": dict({"url": "http://x.test/a", "status": 200},
+                                 **extra)}
     c.feed("Network.requestWillBeSent", {"requestId": "1", "request": req})
+    c.feed("Network.responseReceived", _resp("1"))
     c.feed("Network.dataReceived", {"requestId": "1", "encodedDataLength": 99})
     c.feed("Network.loadingFinished", {"requestId": "1",
                                        "encodedDataLength": 1234})
@@ -2713,6 +2719,7 @@ def test_proxy_meter_units(skips):
     ok &= _savers_check("CDP: a request the browser blocked costs nothing",
                 m.bytes == est + 1234)
     c.feed("Network.requestWillBeSent", {"requestId": "3", "request": req})
+    c.feed("Network.responseReceived", _resp("3"))
     c.feed("Network.dataReceived", {"requestId": "3", "encodedDataLength": 50})
     c.feed("Network.loadingFailed", {"requestId": "3",
                                      "errorText": "net::ERR_CONNECTION_RESET"})
@@ -2722,11 +2729,52 @@ def test_proxy_meter_units(skips):
     c.feed("Network.requestWillBeSent", {
         "requestId": "4", "request": req,
         "redirectResponse": {"encodedDataLength": 300}})
+    c.feed("Network.responseReceived", _resp("4"))
     c.feed("Network.loadingFinished", {"requestId": "4",
                                        "encodedDataLength": 500})
     ok &= _savers_check("CDP: a redirect hop is counted as its own exchange",
                 m.bytes == 4 * est + 1234 + 50 + 300 + 500)
 
+    # Only what crossed the network. A sibling repo's Selenium run counted
+    # Chrome's own new-tab page (chrome:// scripts, ~2.8 MB of
+    # loadingFinished) while its proxy carried ~6 KB.
+    before = m.bytes
+    tab = {"method": "GET", "url": "chrome://new-tab-page/x.js",
+           "headers": {"A": "b"}}
+    c.feed("Network.requestWillBeSent", {"requestId": "t", "request": tab})
+    c.feed("Network.responseReceived", {"requestId": "t", "response": {
+        "url": "chrome://new-tab-page/x.js", "status": 200}})
+    c.feed("Network.dataReceived", {"requestId": "t",
+                                    "encodedDataLength": 2_779_683})
+    c.feed("Network.loadingFinished", {"requestId": "t",
+                                       "encodedDataLength": 2_779_683})
+    for scheme_url in ("data:text/plain,x", "blob:http://x.test/1"):
+        c.feed("Network.requestWillBeSent", {"requestId": scheme_url,
+               "request": dict(req, url=scheme_url)})
+        c.feed("Network.loadingFinished", {"requestId": scheme_url,
+                                           "encodedDataLength": 900})
+    ok &= _savers_check(f"CDP: a chrome:// new-tab load, data: and blob: cost nothing "
+                f"({m.bytes - before} B)", m.bytes == before)
+    c.feed("Network.requestWillBeSent", {"requestId": "k1", "request": req})
+    c.feed("Network.requestServedFromCache", {"requestId": "k1"})
+    c.feed("Network.responseReceived", _resp("k1"))
+    c.feed("Network.loadingFinished", {"requestId": "k1",
+                                       "encodedDataLength": 5000})
+    for flag in ("fromDiskCache", "fromPrefetchCache", "fromServiceWorker"):
+        c.feed("Network.requestWillBeSent", {"requestId": flag, "request": req})
+        c.feed("Network.responseReceived", _resp(flag, **{flag: True}))
+        c.feed("Network.loadingFinished", {"requestId": flag,
+                                           "encodedDataLength": 5000})
+    ok &= _savers_check(f"CDP: a cache hit (requestServedFromCache, fromDiskCache, "
+                f"fromPrefetchCache, fromServiceWorker) costs nothing "
+                f"({m.bytes - before} B)", m.bytes == before)
+    c.feed("Network.requestWillBeSent", {"requestId": "e", "request": req})
+    c.feed("Network.loadingFinished", {"requestId": "e",
+                                       "encodedDataLength": 250_000})
+    ok &= _savers_check(f"CDP: Chrome's own error page (no responseReceived) costs only "
+                f"the request that was attempted ({m.bytes - before} B)",
+                m.bytes - before == est)
+
     # -- Playwright: sizes() of every finished/failed request ---------------
     m = proxy_meter.ProxyMeter(0.001)
     ctx = mock.MagicMock()
@@ -2765,6 +2813,8 @@ def test_proxy_meter_units(skips):
     log = [{"message": json.dumps({"message": {
         "method": "Network.requestWillBeSent",
         "params": {"requestId": "9", "request": req}}})},
+        {"message": json.dumps({"message": {
+            "method": "Network.responseReceived", "params": _resp("9")}})},
         {"message": json.dumps({"message": {
             "method": "Network.loadingFinished",
             "params": {"requestId": "9", "encodedDataLength": 2000}}})},
@@ -2796,6 +2846,7 @@ async def _drive():
         await proxy_meter.install_pyppeteer(page, m)
         on = {call[0][0]: call[0][1] for call in cdp.on.call_args_list}
         on["Network.requestWillBeSent"]({"requestId": "p", "request": req})
+        on["Network.responseReceived"](_resp("p"))
         on["Network.loadingFinished"]({"requestId": "p",
                                        "encodedDataLength": 1500})
         await asyncio.sleep(0)
@@ -2808,6 +2859,32 @@ async def _drive():
     ok &= _savers_check("pyppeteer: over budget, every URL is blocked over CDP",
                 ("Network.setBlockedURLs", {"urls": ["*"]}) in sent)
 
+    # The cap can be crossed on another thread (a concurrent worker) that has
+    # no event loop: the block must still reach this page's own loop.
+    import threading
+    m = proxy_meter.ProxyMeter(0.001)
+    cdp = mock.MagicMock()
+    cdp.send = mock.AsyncMock()
+    page = mock.MagicMock()
+    page.target.createCDPSession = mock.AsyncMock(return_value=cdp)
+    crossed = []
+
+    async def _drive_threaded():
+        await proxy_meter.install_pyppeteer(page, m)
+        worker = threading.Thread(target=lambda: crossed.append(
+            _base_exception(lambda: m.add(5_000))))
+        worker.start()
+        worker.join()
+        for _ in range(5):
+            await asyncio.sleep(0)
+
+    asyncio.run(_drive_threaded())
+    sent = [call[0] for call in cdp.send.await_args_list]
+    ok &= _savers_check(f"pyppeteer: a budget crossed on a worker thread still "
+                f"blocks every URL on the page's loop (thread raised "
+                f"{crossed})", crossed == [None]
+                and ("Network.setBlockedURLs", {"urls": ["*"]}) in sent)
+
     # -- an HTTP client, through a real stub proxy with known sizes ---------
     import requests
     stub = _ByteCountingProxy({"/": 20_000})