Skip to content

Test with chardet 6.0.0.post1 #1017

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Kludex merged 2 commits into from
Jun 2, 2026
+29 −26
Merged

Test with chardet 6.0.0.post1 #1017

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7babf38
Test with chardet 6.0.0.post1
musicinmybrain Jun 2, 2026
2591068
Use unambiguous WINDOWS-1252 sample text in autodetect tests
Kludex Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ dev = [
"httpx2[brotli,cli,http2,socks,zstd]",
"httpcore2[asyncio,trio,http2,socks]",
# Tests
"chardet==5.2.0",
"chardet==6.0.0.post1",
"coverage[toml]==7.10.6",
"cryptography==46.0.7",
"pytest>=9.0.3",
Expand Down
28 changes: 16 additions & 12 deletions tests/httpx2/client/test_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,17 +410,19 @@ def test_client_decode_text_using_autodetect() -> None:
# encoding autodetection to be used when no "Content-Type: text/plain; charset=..."
# info is present.
#
# Here we have some french text encoded with ISO-8859-1, rather than UTF-8.
# Here we have some french text encoded with WINDOWS-1252, rather than UTF-8.
# The curly quotes and em dash occupy bytes 0x80-0x9F, which are control
# characters in ISO-8859-1, so the encoding is unambiguously WINDOWS-1252.
text = (
"Non-seulement Despréaux ne se trompait pas, mais de tous les écrivains "
"Non-seulement Despréaux ne se trompait pas — mais de tous les écrivains "
"que la France a produits, sans excepter Voltaire lui-même, imprégné de "
"l'esprit anglais par son séjour à Londres, c'est incontestablement "
"Molière ou Poquelin qui reproduit avec l'exactitude la plus vive et la "
"l’esprit anglais par son séjour à Londres, c’est incontestablement "
"“Molière” ou Poquelin qui reproduit avec l’exactitude la plus vive et la "
"plus complète le fond du génie français."
)

def cp1252_but_no_content_type(request: httpx2.Request) -> httpx2.Response:
content = text.encode("ISO-8859-1")
content = text.encode("WINDOWS-1252")
return httpx2.Response(200, content=content)

transport = httpx2.MockTransport(cp1252_but_no_content_type)
Expand All @@ -429,25 +431,27 @@ def cp1252_but_no_content_type(request: httpx2.Request) -> httpx2.Response:

assert response.status_code == 200
assert response.reason_phrase == "OK"
assert response.encoding == "ISO-8859-1"
assert response.encoding == "WINDOWS-1252"
assert response.text == text


def test_client_decode_text_using_explicit_encoding() -> None:
# Ensure that a 'default_encoding="..."' on the response is used for text decoding
# when no "Content-Type: text/plain; charset=..."" info is present.
#
# Here we have some french text encoded with ISO-8859-1, rather than UTF-8.
# Here we have some french text encoded with WINDOWS-1252, rather than UTF-8.
# The curly quotes and em dash occupy bytes 0x80-0x9F, which are control
# characters in ISO-8859-1, so the encoding is unambiguously WINDOWS-1252.
text = (
"Non-seulement Despréaux ne se trompait pas, mais de tous les écrivains "
"Non-seulement Despréaux ne se trompait pas — mais de tous les écrivains "
"que la France a produits, sans excepter Voltaire lui-même, imprégné de "
"l'esprit anglais par son séjour à Londres, c'est incontestablement "
"Molière ou Poquelin qui reproduit avec l'exactitude la plus vive et la "
"l’esprit anglais par son séjour à Londres, c’est incontestablement "
"“Molière” ou Poquelin qui reproduit avec l’exactitude la plus vive et la "
"plus complète le fond du génie français."
)

def cp1252_but_no_content_type(request: httpx2.Request) -> httpx2.Response:
content = text.encode("ISO-8859-1")
content = text.encode("WINDOWS-1252")
return httpx2.Response(200, content=content)

transport = httpx2.MockTransport(cp1252_but_no_content_type)
Expand All @@ -456,5 +460,5 @@ def cp1252_but_no_content_type(request: httpx2.Request) -> httpx2.Response:

assert response.status_code == 200
assert response.reason_phrase == "OK"
assert response.encoding == "ISO-8859-1"
assert response.encoding == "WINDOWS-1252"
assert response.text == text
17 changes: 8 additions & 9 deletions tests/httpx2/models/test_responses.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -999,23 +999,22 @@ def test_response_decode_text_using_autodetect() -> None:
# encoding autodetection to be used when no "Content-Type: text/plain; charset=..."
# info is present.
#
# Here we have some french text encoded with ISO-8859-1, rather than UTF-8.
# Here we have some french text encoded with WINDOWS-1252, rather than UTF-8.
# The curly quotes and em dash occupy bytes 0x80-0x9F, which are control
# characters in ISO-8859-1, so the encoding is unambiguously WINDOWS-1252.
text = (
"Non-seulement Despréaux ne se trompait pas, mais de tous les écrivains "
"Non-seulement Despréaux ne se trompait pas — mais de tous les écrivains "
"que la France a produits, sans excepter Voltaire lui-même, imprégné de "
"l'esprit anglais par son séjour à Londres, c'est incontestablement "
"Molière ou Poquelin qui reproduit avec l'exactitude la plus vive et la "
"l’esprit anglais par son séjour à Londres, c’est incontestablement "
"“Molière” ou Poquelin qui reproduit avec l’exactitude la plus vive et la "
"plus complète le fond du génie français."
)
content = text.encode("ISO-8859-1")
content = text.encode("WINDOWS-1252")
response = httpx2.Response(200, content=content, default_encoding=autodetect)

assert response.status_code == 200
assert response.reason_phrase == "OK"
# The encoded byte string is consistent with either ISO-8859-1 or
# WINDOWS-1252. Versions <6.0 of chardet claim the former, while chardet
# 6.0 detects the latter.
assert response.encoding in ("ISO-8859-1", "WINDOWS-1252")
assert response.encoding == "WINDOWS-1252"
assert response.text == text


Expand Down
8 changes: 4 additions & 4 deletions uv.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading