From cbf85c67335c19ef81f4ecb40d11526bac5c5f97 Mon Sep 17 00:00:00 2001 From: allen0099 Date: Sat, 26 Sep 2026 12:26:05 +0000 Subject: [PATCH] fix(redis): deduplicate keys returned by SCAN SCAN may return a key more than once when the keyspace shrinks during the iteration, and get_all_keys() passed the duplicates through to CacheManager and the monitoring routes. Closes #173 --- CHANGELOG.md | 6 ++++++ fastapi_cachex/backends/redis.py | 10 ++++++---- tests/backends/test_redis.py | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 475a41f..03ee00c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,12 @@ Note that 0.3.3 was never released; 0.3.4 follows 0.3.2. now raises `ValueError`. ([#180](https://github.com/allen0099/FastAPI-CacheX/issues/180)) +- **Redis `get_all_keys()` no longer lists a key twice.** SCAN may return a + key more than once when the keyspace shrinks during the iteration, and the + duplicates were passed through to `get_all_keys()`, `CacheManager` and the + monitoring routes. Scanned keys are now deduplicated. + ([#173](https://github.com/allen0099/FastAPI-CacheX/issues/173)) + ## [0.3.7] - 2026-09-25 ### Added diff --git a/fastapi_cachex/backends/redis.py b/fastapi_cachex/backends/redis.py index 4f2edba..02f17e7 100644 --- a/fastapi_cachex/backends/redis.py +++ b/fastapi_cachex/backends/redis.py @@ -213,17 +213,19 @@ def _prefix_pattern(self) -> str: async def _scan_keys(self, pattern: str) -> list[str]: """Collect every key matching ``pattern`` (a full, prefixed glob). - Uses SCAN instead of KEYS so the server is never blocked. + Uses SCAN instead of KEYS so the server is never blocked. SCAN may + return a key more than once (when the keyspace shrinks mid-iteration), + so the result is deduplicated, keeping the order keys were first seen. """ cursor = 0 - keys: list[str] = [] + keys: dict[str, None] = {} while True: cursor, page = await self.client.scan( cursor, match=pattern, count=_BATCH_SIZE ) - keys.extend(page) + keys.update(dict.fromkeys(page)) if cursor == 0: - return keys + return list(keys) async def _delete_keys(self, keys: list[str]) -> int: """Delete prefixed keys in batches; returns how many existed.""" diff --git a/tests/backends/test_redis.py b/tests/backends/test_redis.py index b8e58d0..993e3e9 100644 --- a/tests/backends/test_redis.py +++ b/tests/backends/test_redis.py @@ -820,6 +820,40 @@ async def test_redis_scan_walks_every_page( assert await async_redis_backend.get_all_keys() == [] +@requires_redis +@pytest.mark.asyncio +async def test_redis_scan_results_are_deduplicated( + async_redis_backend: AsyncRedisCacheBackend, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """SCAN may return a key twice when the keyspace shrinks mid-scan (#173). + + That only happens under a concurrent resize, so the client here repeats + the first key it returned on every later page. + """ + total = _BATCH_SIZE * 3 + for index in range(total): + await async_redis_backend.set( + f"dup|||localhost|||/item/{index}|||", + CacheEntry(fingerprint="e", content=b"v"), + ) + + real_scan = async_redis_backend.client.scan + returned: list[str] = [] + + async def scan_with_repeats(*args: Any, **kwargs: Any) -> tuple[int, list[str]]: + cursor, page = await real_scan(*args, **kwargs) + page = [*page, *returned[:1]] + returned.extend(page) + return cursor, page + + monkeypatch.setattr(async_redis_backend.client, "scan", scan_with_repeats) + + keys = await async_redis_backend.get_all_keys() + assert len(returned) > total # the repeat was actually injected + assert len(keys) == len(set(keys)) == total + + @requires_redis @pytest.mark.asyncio async def test_redis_get_cache_data_reports_absolute_expiry(