diff --git a/CHANGELOG.md b/CHANGELOG.md index c0e6487..a1d5eb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Solverr follows its own [Semantic Versioning](https://semver.org/), starting at ## [Unreleased] +### Changes + +- **The passthrough cache now holds at most 256 MB, set with `PASSTHROUGH_CACHE_MAX_BYTES`.** It expired bodies on age but never limited how many it held at once, so a client working through many pages inside one cache window could pin all of them in memory, and large non-HTML documents counted for far more than pages do. Entries closest to expiry are evicted first, and a body over a quarter of the ceiling is served without being cached. + ## [1.4.0] ### Fixes diff --git a/README.md b/README.md index 72cbf0e..b109209 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,7 @@ Notes and limits: - **`GET`/`HEAD` only**; request bodies aren't forwarded. Most indexer definitions are `GET`. - Encode the mirror as a **bare host** (`example-site.tld`), not `https://…` — clients that normalise `//` in a path would otherwise corrupt an embedded scheme. - Successful bodies are cached for `PASSTHROUGH_CACHE_TTL`; challenge pages and non-2xx responses are not, so a transient block retries rather than sticking. +- The cache holds at most `PASSTHROUGH_CACHE_MAX_BYTES` in total. The TTL alone bounded how long a body was kept but not how much was kept, so a client walking many pages inside one TTL window could hold all of them at once. - It's still bound by IP reputation like any solve (see [Proxy & reliability](#proxy--reliability)). If a site blocks your IP, a residential `PROXY_URL` applies to passthrough solves too. ## Configuration @@ -337,6 +338,7 @@ A second HTTP port that returns solved page bodies directly, for clients that wo | `PASSTHROUGH_ALLOWED_HOSTS`| none | Comma-separated hosts it may fetch (the upstream is the first path segment). Empty = refuse every request, so it's never a blind open proxy. | | `PASSTHROUGH_PORT` | `8888` | Listening port. | | `PASSTHROUGH_CACHE_TTL` | `3600` | Seconds to cache a solved 2xx body (`0` disables). Challenge pages are never cached. | +| `PASSTHROUGH_CACHE_MAX_BYTES` | `268435456` | Ceiling on the total bytes the cache holds (`0` lifts it). Past the ceiling the soonest-to-expire entries are evicted first. A single body over a quarter of the ceiling is served but not cached. | | `PASSTHROUGH_TIMEOUT_MS` | `90000` | `maxTimeout` handed to the solver per request. Kept under the ~100s an indexer app waits before recording a failure and backing the indexer off. | ### Browser, logging & server diff --git a/src/config.py b/src/config.py index 8e38e22..0ff55c1 100644 --- a/src/config.py +++ b/src/config.py @@ -211,6 +211,18 @@ def passthrough_cache_ttl() -> int: return _int_env('PASSTHROUGH_CACHE_TTL', 3600) +def passthrough_cache_max_bytes() -> int: + """Ceiling on the total bytes the response cache may hold (0 or less lifts it). + + The TTL bounds how long a body is kept, not how much: every distinct path a + client asks for inside one TTL window accumulated with no ceiling, and a + non-HTML document is held as decoded bytes, so a handful of large ones cost + proportionally more than pages do. 256 MB is generous next to the image and + the two browsers it runs. + """ + return _int_env('PASSTHROUGH_CACHE_MAX_BYTES', 268435456) + + def passthrough_timeout_ms() -> int: """maxTimeout handed to the solver for each passthrough request. diff --git a/src/passthrough.py b/src/passthrough.py index 7eb0874..6c08244 100644 --- a/src/passthrough.py +++ b/src/passthrough.py @@ -41,11 +41,18 @@ _ALLOWED_HOSTS = set() _DEFAULT_HOST = None _CACHE_TTL = 0 +_CACHE_MAX_BYTES = 0 _TIMEOUT_MS = 120000 +# One body may occupy at most this share of the cap. Without it a single large +# document evicts most of the cache to make room for itself, which is worse than +# not caching it at all. +_MAX_BODY_SHARE = 0.25 + _HTML_CONTENT_TYPE = "text/html; charset=utf-8" _cache = {} # request path -> (expires_monotonic, status, body_bytes, content_type) +_cache_bytes = 0 # running total of the body bytes in _cache, guarded by _lock _inflight = {} # request path -> _Pending _lock = threading.Lock() @@ -113,6 +120,60 @@ def _solve(target: str): return status, raw.encode("utf-8", errors="replace"), _HTML_CONTENT_TYPE, res.solution +def reset_cache() -> None: + """Drop every cached body. Exists for tests; the server never needs it.""" + global _cache_bytes + with _lock: + _cache.clear() + _cache_bytes = 0 + + +def _cache_store(raw: str, status: int, body: bytes, content_type: str) -> bool: + """Cache `body` under `raw`, evicting as needed to stay under the byte cap. + + Returns whether it was stored, which is not the same as whether it was + eligible: a body over the per-body ceiling is refused outright. + + Age alone used to bound this. Expired entries were pruned so nothing + outlived its TTL, but nothing capped how much could accumulate inside one + TTL window, so a client walking pagination for an hour pinned every distinct + body for that hour. Non-HTML documents are held as decoded bytes, which is + what makes the total worth measuring in bytes rather than entries. + """ + global _cache_bytes + size = len(body) + if 0 < _CACHE_MAX_BYTES < size / _MAX_BODY_SHARE: + logging.debug("[pt] %s not cached: %d bytes is over the per-body ceiling", raw, size) + return False + + with _lock: + stored_at = time.monotonic() + # Reading an entry only skips it once it expires, so drop the dead ones + # here: every distinct path a client crawls would otherwise pin its body + # for the process lifetime. + for stale in [k for k, v in _cache.items() if v[0] <= stored_at]: + _drop(stale) + # Re-storing a path replaces it, so its old bytes leave the total first. + _drop(raw) + if _CACHE_MAX_BYTES > 0: + # Soonest-expiring first, so eviction takes what was going to go anyway. + for key in sorted(_cache, key=lambda k: _cache[k][0]): + if _cache_bytes + size <= _CACHE_MAX_BYTES: + break + _drop(key) + _cache[raw] = (stored_at + _CACHE_TTL, status, body, content_type) + _cache_bytes += size + return True + + +def _drop(key: str) -> None: + """Remove one entry and its bytes from the total. Caller holds ``_lock``.""" + global _cache_bytes + entry = _cache.pop(key, None) + if entry is not None: + _cache_bytes -= len(entry[2]) + + class _Handler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -210,21 +271,15 @@ def _handle(self): _CACHE_TTL > 0 and 200 <= status < 300 and not detection.looks_like_challenge_html(solution.response) ) - with _lock: - if cacheable: - stored_at = time.monotonic() - # Reading an entry only skips it once it expires, so drop the dead - # ones here: every distinct path a client crawls would otherwise - # pin its body (now possibly a whole file) for the process lifetime. - for stale in [k for k, v in _cache.items() if v[0] <= stored_at]: - del _cache[stale] - _cache[raw] = (stored_at + _CACHE_TTL, status, body, content_type) + # Eligible is not the same as stored: the byte cap can still refuse it, + # so the log below reports what actually happened. + cached = cacheable and _cache_store(raw, status, body, content_type) pending.status = status pending.body = body pending.content_type = content_type logging.info("[pt %s] %s %s <- %d in %.1fs (%d bytes%s)", rid, self.command, raw, status, time.monotonic() - started, - len(body), ", cached" if cacheable else "") + len(body), ", cached" if cached else "") self._send(status, body, content_type) finally: with _lock: @@ -247,12 +302,13 @@ def start(): if not config.passthrough_enabled(): return - global _ALLOWED_HOSTS, _DEFAULT_HOST, _CACHE_TTL, _TIMEOUT_MS + global _ALLOWED_HOSTS, _DEFAULT_HOST, _CACHE_TTL, _CACHE_MAX_BYTES, _TIMEOUT_MS hosts = config.passthrough_allowed_hosts() _ALLOWED_HOSTS = set(hosts) # First allow-listed host is the mirror used for site-internal absolute links. _DEFAULT_HOST = hosts[0] if hosts else None _CACHE_TTL = config.passthrough_cache_ttl() + _CACHE_MAX_BYTES = config.passthrough_cache_max_bytes() _TIMEOUT_MS = config.passthrough_timeout_ms() port = config.passthrough_port() @@ -261,7 +317,15 @@ def start(): logging.info(" allowed hosts: %s (default: %s)", ", ".join(hosts), _DEFAULT_HOST) else: logging.warning(" PASSTHROUGH_ALLOWED_HOSTS is empty; every request is refused (403)") - logging.info(" cache ttl: %ds, request timeout: %dms", _CACHE_TTL, _TIMEOUT_MS) + if _CACHE_MAX_BYTES <= 0: + cap = "unbounded" + elif _CACHE_MAX_BYTES >= 1024 * 1024: + cap = f"{_CACHE_MAX_BYTES // (1024 * 1024)} MB" + else: + # Reporting a sub-megabyte cap in whole MB rounds it to "0 MB", which + # reads as caching being off rather than tight. + cap = f"{_CACHE_MAX_BYTES} bytes" + logging.info(" cache ttl: %ds (max %s), request timeout: %dms", _CACHE_TTL, cap, _TIMEOUT_MS) server = ThreadingHTTPServer(("0.0.0.0", port), _Handler) threading.Thread(target=server.serve_forever, daemon=True, name="passthrough").start() diff --git a/src/test_passthrough_cache.py b/src/test_passthrough_cache.py new file mode 100644 index 0000000..d294633 --- /dev/null +++ b/src/test_passthrough_cache.py @@ -0,0 +1,109 @@ +"""The passthrough response cache stays under its byte ceiling. + +The TTL bounds how long a body is kept, not how much is kept, so these cover the +size half: eviction order, the per-body ceiling, and the running total staying +honest across replaces and expiries. +""" +import unittest + +import passthrough + + +def store(path: str, size: int) -> bool: + return passthrough._cache_store(path, 200, b"x" * size, "text/html; charset=utf-8") + + +def cached_paths() -> list: + return list(passthrough._cache.keys()) + + +class PassthroughCacheBytesTest(unittest.TestCase): + + def setUp(self): + passthrough.reset_cache() + passthrough._CACHE_TTL = 3600 + passthrough._CACHE_MAX_BYTES = 1000 + + def tearDown(self): + passthrough.reset_cache() + passthrough._CACHE_TTL = 0 + passthrough._CACHE_MAX_BYTES = 0 + + def test_a_stored_body_adds_its_size_to_the_total(self): + store("/a", 100) + self.assertEqual(passthrough._cache_bytes, 100) + + def test_a_body_within_the_ceiling_is_stored(self): + self.assertTrue(store("/a", 250)) + + def test_a_body_over_the_per_body_ceiling_is_refused(self): + self.assertFalse(store("/big", 251)) + + def test_a_refused_body_leaves_the_cache_empty(self): + store("/big", 251) + self.assertEqual(cached_paths(), []) + + def test_the_total_stays_under_the_cap_when_stores_exceed_it(self): + store("/a", 200) + store("/b", 200) + store("/c", 200) + store("/d", 200) + store("/e", 200) + store("/f", 200) + self.assertLessEqual(passthrough._cache_bytes, 1000) + + def test_the_oldest_entry_is_evicted_first(self): + store("/a", 200) + store("/b", 200) + store("/c", 200) + store("/d", 200) + store("/e", 200) + store("/f", 200) + self.assertNotIn("/a", cached_paths()) + + def test_the_newest_entry_survives_eviction(self): + store("/a", 200) + store("/b", 200) + store("/c", 200) + store("/d", 200) + store("/e", 200) + store("/f", 200) + self.assertIn("/f", cached_paths()) + + def test_re_storing_a_path_does_not_double_count_its_bytes(self): + store("/a", 100) + store("/a", 100) + self.assertEqual(passthrough._cache_bytes, 100) + + def test_re_storing_a_path_keeps_one_entry(self): + store("/a", 100) + store("/a", 300) + self.assertEqual(cached_paths(), ["/a"]) + + def test_an_expired_entry_is_dropped_on_the_next_store(self): + store("/old", 100) + expires, status, body, ctype = passthrough._cache["/old"] + passthrough._cache["/old"] = (0, status, body, ctype) + store("/new", 100) + self.assertNotIn("/old", cached_paths()) + + def test_an_expired_entry_releases_its_bytes(self): + store("/old", 200) + expires, status, body, ctype = passthrough._cache["/old"] + passthrough._cache["/old"] = (0, status, body, ctype) + store("/new", 100) + self.assertEqual(passthrough._cache_bytes, 100) + + def test_a_zero_cap_lifts_the_ceiling(self): + passthrough._CACHE_MAX_BYTES = 0 + self.assertTrue(store("/huge", 10_000)) + + def test_a_zero_cap_evicts_nothing(self): + passthrough._CACHE_MAX_BYTES = 0 + store("/a", 5_000) + store("/b", 5_000) + self.assertEqual(len(cached_paths()), 2) + + +if __name__ == '__main__': + unittest.main()