From 0133733d0c37e86a74262d82c3f03fd887533d27 Mon Sep 17 00:00:00 2001 From: allen0099 Date: Sat, 26 Sep 2026 11:02:53 +0000 Subject: [PATCH] fix(redis): warn when the backend is configured with a non-UTF-8 encoding Entries are always written as UTF-8 JSON, but the client decodes replies with the configured encoding, so under encoding="latin-1" a stored b"\xe9" read back as b"\xc3\xa9" without any error. AsyncRedisCacheBackend, and so load_from_config(), now emit a RuntimeWarning for any encoding other than UTF-8 (aliases such as "UTF8" are accepted). An unknown codec is still rejected by redis-py itself. #126 removes the parameter in 0.4.0. Closes #122 --- CHANGELOG.md | 20 +++++++---- docs/BACKENDS.md | 8 ++++- fastapi_cachex/backends/redis.py | 31 ++++++++++++++++- tests/backends/test_redis.py | 57 ++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcb9bde..d085818 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,13 @@ Note that 0.3.3 was never released; 0.3.4 follows 0.3.2. the full entries on the documentation site. The changelog itself keeps the details. +### Deprecated + +- **Passing the Redis key prefix in a `clear_pattern()` pattern.** When such a + pattern clears nothing, the prefix-stripped form is still tried and emits a + `DeprecationWarning` if it clears anything. The retry will be removed in + 0.4.0. ([#125](https://github.com/allen0099/FastAPI-CacheX/issues/125)) + ### Fixed - **Redis `clear_pattern()` no longer strips a pattern that starts with the key @@ -38,12 +45,13 @@ Note that 0.3.3 was never released; 0.3.4 follows 0.3.2. keys also start with `cache:`, `clear_pattern("user:*")` used to clear nothing. ([#109](https://github.com/allen0099/FastAPI-CacheX/issues/109)) -### Deprecated - -- **Passing the Redis key prefix in a `clear_pattern()` pattern.** When such a - pattern clears nothing, the prefix-stripped form is still tried and emits a - `DeprecationWarning` if it clears anything. The retry will be removed in - 0.4.0. ([#125](https://github.com/allen0099/FastAPI-CacheX/issues/125)) +- **The Redis backend warns when `encoding` is not UTF-8.** Entries are always + written as UTF-8 JSON, but the client decoded replies with the configured + encoding, so under `encoding="latin-1"` non-ASCII content came back + corrupted without any error. `AsyncRedisCacheBackend` and + `load_from_config()` now emit a `RuntimeWarning` for any encoding other than + UTF-8; the parameter is removed in 0.4.0 (#126). + ([#122](https://github.com/allen0099/FastAPI-CacheX/issues/122)) ## [0.3.7] - 2026-09-25 diff --git a/docs/BACKENDS.md b/docs/BACKENDS.md index 028c4fe..1fde999 100644 --- a/docs/BACKENDS.md +++ b/docs/BACKENDS.md @@ -71,7 +71,7 @@ config = RedisConfig( port=6379, password=None, # SecretStr | None db=0, - encoding="utf-8", # how the client decodes server responses + encoding="utf-8", # keep UTF-8; see below socket_timeout=1.0, # seconds; applies to reads/writes socket_connect_timeout=1.0, key_prefix="fastapi_cachex:", @@ -81,6 +81,12 @@ backend = AsyncRedisCacheBackend.load_from_config(config) BackendProxy.set(backend) ``` +Keep `encoding="utf-8"`. Entries are always written as UTF-8 JSON, and the client +decodes replies with `encoding`, so any other value corrupts non-ASCII content on the way +back (with `"latin-1"`, a stored `b"\xe9"` reads back as `b"\xc3\xa9"`). The backend +emits a `RuntimeWarning` for a non-UTF-8 encoding, and the parameter will be removed in +0.4.0. + Keep `protocol=2` unless you need RESP3 features *and* your `hiredis` build supports it (RESP3 needs hiredis >= 3.0). Redis 8.0 speaks RESP3, but an older hiredis will fail to negotiate it. diff --git a/fastapi_cachex/backends/redis.py b/fastapi_cachex/backends/redis.py index 23ead90..4f2edba 100644 --- a/fastapi_cachex/backends/redis.py +++ b/fastapi_cachex/backends/redis.py @@ -1,5 +1,6 @@ """Redis cache backend implementation.""" +import codecs import logging import time import warnings @@ -71,6 +72,28 @@ def _escape_glob(text: str) -> str: """ +def _warn_if_not_utf8(encoding: str) -> None: + r"""Warn when replies would be decoded with anything but UTF-8. + + The shared codec writes UTF-8 JSON, and the client decodes each reply with + ``encoding``, so under e.g. latin-1 a stored ``b"\xe9"`` reads back as + ``b"\xc3\xa9"``. Aliases such as ``"UTF8"`` and ``"utf_8"`` are accepted. + """ + try: + name = codecs.lookup(encoding).name + except LookupError: + return # the client rejects an unknown encoding itself + if name != "utf-8": + warnings.warn( + f"AsyncRedisCacheBackend(encoding={encoding!r}) will corrupt non-ASCII " + "cached content: entries are always written as UTF-8, and replies " + "are decoded with this encoding. Use encoding='utf-8'. The encoding " + "parameter will be removed in version 0.4.0.", + RuntimeWarning, + stacklevel=3, + ) + + class AsyncRedisCacheBackend(BaseCacheBackend): """Async Redis cache backend implementation. @@ -102,7 +125,11 @@ def __init__( port: Redis port password: Redis password db: Redis database number - encoding: Character encoding to use + encoding: Character encoding the client decodes replies with. + Leave it as UTF-8: entries are always written as UTF-8 JSON, so + any other encoding corrupts non-ASCII content on the way back, + and a ``RuntimeWarning`` says so. The parameter will be removed + in 0.4.0. decode_responses: Whether to decode response automatically socket_timeout: Timeout for socket operations (in seconds) socket_connect_timeout: Timeout for socket connection (in seconds) @@ -124,6 +151,8 @@ def __init__( ) raise CacheXError(msg) + _warn_if_not_utf8(encoding) + # `protocol` is not in the types-redis stubs (added in redis-py 5.x). # Pass it via **kwargs so mypy doesn't complain about an unknown keyword. kwargs.setdefault("protocol", protocol) diff --git a/tests/backends/test_redis.py b/tests/backends/test_redis.py index 8a3b703..b8e58d0 100644 --- a/tests/backends/test_redis.py +++ b/tests/backends/test_redis.py @@ -52,6 +52,63 @@ def test_redis_load_from_config_initializes_client_and_prefix() -> None: assert hasattr(backend.client, "connection_pool") +@requires_redis_package +@pytest.mark.parametrize("encoding", ["utf-8", "UTF8", "utf_8"]) +def test_redis_utf8_encoding_does_not_warn(encoding: str) -> None: + """UTF-8 under any of its aliases is accepted silently (#122).""" + with warnings.catch_warnings(): + warnings.simplefilter("error") + AsyncRedisCacheBackend(port=UNCONNECTED_PORT, encoding=encoding) + + +@requires_redis_package +@pytest.mark.parametrize("encoding", ["latin-1", "utf-16", "ascii"]) +def test_redis_non_utf8_encoding_warns(encoding: str) -> None: + """Replies decoded as anything but UTF-8 corrupt cached content (#122).""" + with pytest.warns(RuntimeWarning, match="corrupt non-ASCII") as record: + AsyncRedisCacheBackend(port=UNCONNECTED_PORT, encoding=encoding) + ours = [w for w in record if "corrupt non-ASCII" in str(w.message)] + assert [w.filename for w in ours] == [__file__] + + +@requires_redis_package +def test_redis_unknown_encoding_is_left_to_the_client() -> None: + """An unknown codec is rejected by redis-py, not warned about (#122).""" + with warnings.catch_warnings(): + warnings.simplefilter("error") + with pytest.raises(LookupError): + AsyncRedisCacheBackend(port=UNCONNECTED_PORT, encoding="no-such-codec") + + +@requires_redis_package +def test_redis_load_from_config_warns_on_non_utf8_encoding() -> None: + """RedisConfig goes through the same check (#122).""" + from fastapi_cachex.backends.config import RedisConfig + + with pytest.warns(RuntimeWarning, match="encoding='latin-1'"): + AsyncRedisCacheBackend.load_from_config(RedisConfig(encoding="latin-1")) + + +@pytest.mark.asyncio +async def test_redis_latin1_encoding_corrupts_non_ascii_content() -> None: + """What the warning is about, on a live server (#122).""" + reason = redis_skip_reason() + if reason is not None: + pytest.skip(reason) + + with pytest.warns(RuntimeWarning): + backend = AsyncRedisCacheBackend( + host=REDIS_HOST, port=REDIS_PORT, encoding="latin-1" + ) + try: + await backend.set("k", CacheEntry(fingerprint="f", content=b"\xe9")) + entry = await backend.get("k") + assert entry is not None + assert entry.content == b"\xc3\xa9" + finally: + await backend.clear() + + @requires_redis_package def test_redis_load_from_config_passes_all_fields() -> None: """load_from_config must forward all RedisConfig fields to the backend."""