diff --git a/CHANGELOG.md b/CHANGELOG.md index 693b7b06..ea8414ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,39 @@ compatibility (see [RELEASING.md](RELEASING.md)). ### Added +- **Ten CVEs on encoding rather than code points, and the survey method behind them + (36 → 46 rows).** Found by sweeping NVD across the operations disarm performs, then + verified one ID at a time. + + | Class | Added | + |---|---| + | Overlong / invalid byte sequences | CVE-2024-46954, CVE-2026-44288, CVE-2009-4142 | + | Lone surrogates | CVE-2022-31116, CVE-2025-64439, CVE-2008-4066 | + | Full-width evasion of a detector | CVE-2007-2688, CVE-2001-0669 | + | Encoding layers disarm does not own | CVE-2022-3782, CVE-2006-2753 | + + **The byte-level rows are the first on the page whose input is not a `str`.** + `decode_to_utf8` replaces overlong sequences rather than decoding them, so the + Ghostscript traversal never materializes and `strict=True` refuses outright. + CVE-2026-44288 names the correct behaviour exactly — protobufjs decoded overlong + sequences "to canonical characters instead of replacing them" — which is what makes + `not-affected` measurable rather than asserted. + + **CVE-2025-64439 is a Unicode edge case reaching RCE through an error path**: illegal + surrogates made msgpack serialization fail in LangGraph, and the fallback was JSON + deserialization of untrusted data. disarm substitutes rather than drops, which is what + keeps `keyvalue` from colliding with `keyvalue` — the CVE-2022-31116 shape. + + **CVE-2007-2688 is the Threat Model's ordering rule eighteen years early.** Cisco IPS, + Check Point and IBM ISS Proventia all shipped the same missing normalization step in the + same month. It is also the same fold `TestFullwidthUnmaskingHazard` pins as a hazard — + both readings are correct, and pipeline position decides which applies. + + `docs/security/cve-validation.md` now records **how rows are found**: the NVD sweeps by + mechanism rather than by product, the per-ID verification that caught CVE-2017-20190 + having no CVSS at all, and the non-CVE research that informed rows — Paul Butler on + variation-selector smuggling, and the CoreText Telugu crash. + - **The comparator table no longer contradicts the matrix.** CVE-2026-23950 rendered as `Neutralized` in the matrix and as `no` under both disarm columns in the comparison a hundred lines below, because the comparison scores every row against two *fixed* disarm diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index cba60429..ebae49d8 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -251,8 +251,8 @@ expanding the bundled mapping data is exactly how this layer improves. The scope above is grounded in the literature, not asserted: - **The scope above is also tested against named CVEs.** [`tests/test_cve_vectors.py`](https://github.com/raeq/disarm/blob/main/tests/test_cve_vectors.py) - reconstructs the vector described in each of 36 published CVEs and asserts what - disarm does with it — including the 11 it does **not** handle, whose negatives are + reconstructs the vector described in each of 46 published CVEs and asserts what + disarm does with it — including the 15 it does **not** handle, whose negatives are pinned so the limits stay tested rather than merely stated. Rendered as a matrix in [`docs/security/cve-validation.md`](https://github.com/raeq/disarm/blob/main/docs/security/cve-validation.md). diff --git a/benchmarks/cve_comparators.py b/benchmarks/cve_comparators.py index d3da6840..b8a516fe 100644 --- a/benchmarks/cve_comparators.py +++ b/benchmarks/cve_comparators.py @@ -55,6 +55,10 @@ def _has_tag_chars(text: str) -> bool: return any(0xE0000 <= ord(ch) <= 0xE007F for ch in text) +def _no_surrogates(text: str) -> bool: + return not any(0xD800 <= ord(ch) <= 0xDFFF for ch in text) + + def _no_bidi(text: str) -> bool: return not any(ch in text for ch in BIDI) @@ -83,6 +87,7 @@ def _no_bidi(text: str) -> bool: # clear this one — its key builders do — so it is comparable without being # part of the canonicalizer-clearable set. ("CVE-2026-23950", "groß.txt", "gross.txt"), + ("CVE-2007-2688", "＜script＞alert(1)＜/script＞", ""), ] #: (cve, attack, predicate) — handled when the primitive is gone from the output. @@ -136,6 +141,11 @@ def _no_bidi(text: str) -> bool: # A pile of marks, bounded rather than removed: the defense is that the run # cannot reach a downstream stage, not that the base character goes away. ("CVE-2017-20190", "a" + ("\u0301" * 2_000), lambda out: len(out) <= 4), + # decancer raises on these: it is Rust-backed and takes &str, so a lone + # surrogate cannot cross the boundary at all. _apply turns that into a + # non-result, which is the right score — it did not handle the input. + ("CVE-2022-31116", "key\udc00value", _no_surrogates), + ("CVE-2025-64439", "a\ud800b", _no_surrogates), ] #: Rows whose neutralizer in the matrix is neither of the two fixed disarm diff --git a/docs/security/cve-validation.md b/docs/security/cve-validation.md index bfe6556e..63a131b8 100644 --- a/docs/security/cve-validation.md +++ b/docs/security/cve-validation.md @@ -83,6 +83,16 @@ ranking. | [CVE-2026-3276](https://nvd.nist.gov/vuln/detail/CVE-2026-3276) | CPython — unicodedata.normalize() CPU blowup on alternating-CCC runs | 6.3 (v4.0) | Not affected + detected | `normalize` | `is_zalgo`, `has_anomalies` | | [CVE-2023-46695](https://nvd.nist.gov/vuln/detail/CVE-2023-46695) | Django — NFKC normalization slow on Windows, DoS via UsernameField | 7.5 (v3.1) | Not affected | `normalize` | — | | [CVE-2017-20190](https://nvd.nist.gov/vuln/detail/CVE-2017-20190) | Windows — performance degradation from piled combining marks (Zalgo) | none (SSVC only) | Neutralized + detected | `strip_zalgo`, `canonicalize`, `strip_obfuscation` | `is_zalgo`, `has_anomalies` | +| [CVE-2024-46954](https://nvd.nist.gov/vuln/detail/CVE-2024-46954) | Ghostscript — overlong UTF-8 decoded to a real ../ traversal | 7.8 (v3.1) | Not affected | `decode_to_utf8` | — | +| [CVE-2026-44288](https://nvd.nist.gov/vuln/detail/CVE-2026-44288) | protobufjs — overlong UTF-8 decoded to canonical characters | 5.3 (v3.1) | Not affected | `decode_to_utf8` | — | +| [CVE-2009-4142](https://nvd.nist.gov/vuln/detail/CVE-2009-4142) | PHP htmlspecialchars — overlong UTF-8 and invalid Shift_JIS/EUC-JP | 4.3 (v2.0) | Not affected | `decode_to_utf8` | — | +| [CVE-2022-31116](https://nvd.nist.gov/vuln/detail/CVE-2022-31116) | UltraJSON — lone surrogates causing dictionary key confusion | 7.5 (v3.1) | Neutralized | `canonicalize`, `canonicalize_strict`, `strip_obfuscation` | — | +| [CVE-2025-64439](https://nvd.nist.gov/vuln/detail/CVE-2025-64439) | LangGraph — illegal surrogates falling back to insecure deserialization | 7.4 (v4.0) | Neutralized | `canonicalize`, `canonicalize_strict`, `strip_obfuscation` | — | +| [CVE-2008-4066](https://nvd.nist.gov/vuln/detail/CVE-2008-4066) | Firefox — HTML-escaped low surrogate ignored by the parser | 4.3 (v2.0) | Out of scope | — | — | +| [CVE-2007-2688](https://nvd.nist.gov/vuln/detail/CVE-2007-2688) | Cisco IPS — HTTP detection evaded by full-width Unicode | 7.8 (v2.0) | Neutralized | `canonicalize`, `canonicalize_strict`, `strip_obfuscation` | — | +| [CVE-2001-0669](https://nvd.nist.gov/vuln/detail/CVE-2001-0669) | Snort, Cisco IDS, Dragon, RealSecure — evaded by %u encoding | 7.5 (v2.0) | Out of scope | — | — | +| [CVE-2022-3782](https://nvd.nist.gov/vuln/detail/CVE-2022-3782) | Keycloak — path traversal via double URL encoding | 9.1 (v3.1) | Out of scope | — | — | +| [CVE-2006-2753](https://nvd.nist.gov/vuln/detail/CVE-2006-2753) | MySQL — mysql_real_escape_string bypassed by multibyte charsets | 7.5 (v2.0) | Out of scope | — | — | ## If you only make one call @@ -144,7 +154,7 @@ page, and `TestOneCall` gates both halves of the claim — including a test that fails if any single entry point ever *does* become sufficient, so the guidance gets revisited rather than silently left stale. -These 22 vectors are a spot check, not a measurement of the confusable space. +These 25 vectors are a spot check, not a measurement of the confusable space. Treat this as "no vector here needs a third call", not as coverage. ### Detection has no equivalent @@ -172,8 +182,8 @@ flagged forwards those five untouched. Three tools, three jobs, one predicate applied identically to each. Regenerate with `python benchmarks/cve_comparators.py --markdown`. -**22 of the 36 rows are compared.** The other 14 cannot be, and it is worth being -explicit about which: 11 are out of scope, so no tool neutralizes them and there +**25 of the 46 rows are compared.** The other 14 cannot be, and it is worth being +explicit about which: 15 are out of scope, so no tool neutralizes them and there is nothing to compare; 2 are *not affected*, which is a statement about normalization cost rather than a transformation; and 1 is detected without being neutralized. `test_every_registry_row_is_compared_or_has_a_reason_not_to` holds @@ -199,6 +209,7 @@ except ImportError: | CVE | `disarm.canonicalize` | `disarm.strip_obfuscation` | `decancer.parse` | `unidecode` | |---|---|---|---|---| +| CVE-2007-2688 | yes | yes | yes | yes | | CVE-2008-2383 | yes | yes | **no** | **no** | | CVE-2009-3376 | yes | yes | **no** | yes | | CVE-2013-7236 | yes | yes | yes | yes | @@ -213,6 +224,7 @@ except ImportError: | CVE-2020-12063 † | yes | yes | yes | yes | | CVE-2021-42574 | yes | yes | **no** | yes | | CVE-2021-42694 | yes | yes | yes | yes | +| CVE-2022-31116 | yes | yes | **no** | yes | | CVE-2023-24329 | yes | yes | yes | **no** | | CVE-2023-33955 | yes | yes | **no** | yes | | CVE-2023-37275 | yes | yes | yes | **no** | @@ -220,8 +232,9 @@ except ImportError: | CVE-2024-52005 | yes | yes | yes | **no** | | CVE-2025-32711 | yes | yes | **no** | yes | | CVE-2025-55754 | yes | yes | yes | **no** | +| CVE-2025-64439 | yes | yes | **no** | yes | | CVE-2026-23950 † | **no** | **no** | **no** | yes | -| **Handled** | **20/22** | **20/22** | **16/22** | **14/22** | +| **Handled** | **23/25** | **23/25** | **17/25** | **17/25** | † The matrix neutralizes these rows with an entry point that is not one of the two disarm columns above, so a `no` here means *not this function* rather than *not disarm*: CVE-2019-19844 → `canonicalize_strict`, CVE-2020-12063 → `normalize_confusables`, CVE-2026-23950 → `fold_case`. @@ -523,6 +536,143 @@ check reads the unmasked string; validate first and canonicalize after, and the check approved a host the canonical form no longer names. See the Threat Model's *Pipeline placement* section. +## Bytes, not code points + +Every other section on this page starts with text that already decoded. These +rows are attacks on the decoder itself, and they are the only ones where the +input cannot be written as a Python `str` at all. + +### Overlong UTF-8 + +An overlong sequence encodes an ASCII character in more bytes than the shortest +form. A decoder that accepts one yields the real character; the standard says it +must not. CVE-2024-46954 turned that into `../` inside Ghostscript, and +CVE-2025-46646 is the *incomplete fix* for it — a fair measure of how easy this +is to get wrong twice. CVE-2026-44288 names the correct behaviour precisely: +protobufjs decoded overlong sequences "to canonical characters instead of +replacing them", letting an attacker bypass byte-level checks. + +disarm is marked **not affected**, and here is what that rests on: + +```python +from disarm import decode_to_utf8 + +# `../` written as three overlong two-byte sequences +text, had_errors = decode_to_utf8(b"\xc0\xae\xc0\xae\xc0\xaf", encoding="utf-8") + +assert had_errors is True +assert set(text) == {"\ufffd"} # replaced, never decoded +assert "../" not in text +``` + +`strict=True` refuses outright rather than returning a lossy string: + +```python +from disarm import DisarmError + +try: + decode_to_utf8(b"\xc0\xaf", encoding="utf-8", strict=True) +except DisarmError: + pass # the caller who cannot tolerate substitution gets an error +``` + +CVE-2009-4142 is the same class through a different door — an invalid Shift_JIS +or EUC-JP lead byte placed *before* a special character, so that the escaping +routine downstream never sees the character. The payload survives intact here +rather than being swallowed: + +```python +text, had_errors = decode_to_utf8(b"\x81\x00" # the fix +``` + +This is the same fold that `TestFullwidthUnmaskingHazard` pins as a *hazard*, +and both readings are correct. Which one applies is decided entirely by pipeline +position: folding before a **detector** is the fix, folding before an **output +sink** is the hazard. + +CVE-2001-0669 — Snort, Cisco Secure IDS, Dragon and ISS RealSecure evaded by +`%u` encoding — is out of scope, and for an instructive reason. `%u003c` is a +Microsoft URL-encoding extension; the bytes on the wire are ASCII `%`, `u`, `0`. +There is no Unicode there yet, so URL decoding has to happen before disarm sees +the text. + +```python +assert canonicalize("%u003cscript%u003e") == "%u003cscript%u003e" +``` + +### Two encoding layers disarm does not own + +`CVE-2022-3782` (Keycloak, 9.1) is path traversal via **double URL encoding**. +disarm exposes `percent_encode` and no decoder at all, deliberately — how many +times to decode is a property of the protocol stack, and a library that guessed +would manufacture the ambiguity the CVE is about. + +`CVE-2006-2753` (MySQL, 7.5) is the classic **multibyte escape bypass**: in +SJIS, BIG5 and GBK a character can end in `0x5C`, so appending a backslash +produces a valid character instead of an escape. The fix is a charset-aware +escaper or parameterized queries. disarm performs no SQL quoting and never +replaces one. + +```python +assert canonicalize("%252e%252e%252fadmin") == "%252e%252e%252fadmin" +assert canonicalize("\u00bf' OR 1=1") == "\u00bf' OR 1=1" +``` + ## Ordering: normalize, then validate The Threat Model states the rule — *canonicalize first, then validate, @@ -836,6 +986,62 @@ Canonicalize on the way *in*, before a filter or a comparison. Never on the way *out*, into an execution or markup sink. The Threat Model lists metacharacter unmasking under *Out of scope*; the assertions above are what it looks like. +## Where these rows come from + +The matrix is not a reading list someone remembered. Rows are found by sweeping +NVD's keyword search across the operations disarm actually performs, then +verified one CVE at a time against the REST API before anything is written down. + +The sweeps that produced the current set: + +| Query | Class it surfaced | +|---|---| +| `homoglyph`, `right-to-left override` | Identity spoofing, RLO filenames | +| `unicode normalization` | The ordering class, and the cost class | +| `zero-width` | Invisible-character prefixes | +| `punycode` | IDN and address-bar spoofing | +| `ANSI escape sequence` | Terminal control, the largest single class | +| `combining characters` | Zalgo, and the eclipsing marks of CVE-2017-7833 | +| `case-insensitive bypass` | Case-folding collisions | +| `overlong UTF-8`, `UTF-7`, `character encoding` | Byte-level decoding | +| `surrogate` | Lone surrogates | +| `double encoding` | The URL layer, out of scope | +| `prompt injection` | The ML/LLM rows | + +Two habits are worth copying if you extend this. + +**Search by mechanism, not by product.** `ANSI escape sequence` returned twenty +CVEs across terminals, loggers, version-control clients and an LLM agent. No +list of products would have found that set, and the class turned out to be the +one place where disarm neutralizes everything and detects nothing. + +**Verify each ID individually.** Keyword results are summaries. Every row here +was fetched by ID before it was written, which is how CVE-2017-20190 turned out +to have no CVSS score at all and CVE-2026-3276 turned out to be scored under +CVSS v4.0 — both of which changed the registry schema rather than being rounded +off to fit it. + +### Reading beyond the CVE record + +Some of the sharpest work on this material is not in any database. Two that +directly informed rows above: + +- **[Smuggling arbitrary data through an emoji](https://paulbutler.org/2025/smuggling-arbitrary-data-through-an-emoji/)** + (Paul Butler, 2025) — there are exactly 256 variation selectors, which is + exactly one byte, and they are *preserved through copy-paste by design*. The + Tags-block channel in CVE-2025-32711 has a sibling that no CVE covers. +- **[Picking apart the crashing iOS string](https://manishearth.github.io/blog/2018/02/15/picking-apart-the-crashing-ios-string/)** + (Manish Goregaokar, 2018) — the Telugu "text bomb" that crashed CoreText and + took SpringBoard with it. The trigger was a zero-width non-joiner inside + multi-code-point glyph composition, in the ordinary word for "knowledge". + A reminder that these sequences are not all attacker inventions. + +The WAF-bypass literature is the same ordering rule in another register: a +filter matching `" + assert canonicalize("ＳＥＬＥＣＴ＊ＦＲＯＭ") == "SELECT*FROM" + assert canonicalize("／ｅｔｃ／ｐａｓｓｗｄ") == "/etc/passwd" + + def test_percent_u_encoding_is_out_of_scope(self) -> None: + """OUT-OF-SCOPE NEGATIVE for CVE-2001-0669. + + ``%u003c`` is a Microsoft URL-encoding extension, not a Unicode + representation of anything — the bytes on the wire are ASCII ``%``, + ``u``, ``0``… disarm has no URL decoder and correctly leaves it alone. + Decoding has to happen in the URL layer before disarm sees the text, + which is the same ordering rule the row above is about. + """ + assert canonicalize(PERCENT_U_SCRIPT) == PERCENT_U_SCRIPT + assert strip_obfuscation(PERCENT_U_SCRIPT) == PERCENT_U_SCRIPT + + +class TestEncodingLayersDisarmDoesNotOwn: + """CVE-2022-3782, CVE-2006-2753 — OUT OF SCOPE, for two different reasons. + + Both are encoding attacks, and neither is a Unicode attack. They are here so + the boundary is drawn on the page rather than left for a reader to discover + by trying. + """ + + def test_double_url_encoding_is_the_url_layers_job(self) -> None: + """CVE-2022-3782 (Keycloak, 9.1): `%252e` is `%2e` is `.`. + + disarm exposes ``percent_encode`` and no decoder at all, deliberately: + deciding how many times to decode is a property of the protocol stack, + and a library that guessed would create the very ambiguity the CVE is + about. + """ + assert canonicalize("%252e%252e%252fadmin") == "%252e%252e%252fadmin" + assert not hasattr(disarm, "percent_decode") + + def test_multibyte_escape_bypass_is_not_an_injection_defense(self) -> None: + """CVE-2006-2753 (MySQL, 7.5): a trailing byte swallows the escape. + + In SJIS/BIG5/GBK a multibyte character can end in 0x5C, so an escaping + routine that appends a backslash produces a valid character instead of + an escape. The fix is a charset-aware escaper or parameterized queries. + THREAT_MODEL.md: disarm performs no SQL quoting and never replaces one. + """ + payload = "\u00bf' OR 1=1" + assert canonicalize(payload) == payload + assert strip_obfuscation(payload) == payload + + # --------------------------------------------------------------------------- # The registry # --------------------------------------------------------------------------- @@ -1927,6 +2180,126 @@ def test_disarm_does_not_bound_input_length(self) -> None: probe=ZALGO_PILE, reference="https://nvd.nist.gov/vuln/detail/CVE-2017-20190", ), + CVE( + id="CVE-2024-46954", + title="Ghostscript — overlong UTF-8 decoded to a real ../ traversal", + cwe="CWE-22", + cvss=7.8, + cvss_version="v3.1", + dispositions=frozenset({NOT_AFFECTED}), + neutralizers=("decode_to_utf8",), + detectors=(), + probe="\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd", + reference="https://nvd.nist.gov/vuln/detail/CVE-2024-46954", + ), + CVE( + id="CVE-2026-44288", + title="protobufjs — overlong UTF-8 decoded to canonical characters", + cwe="CWE-176", + cvss=5.3, + cvss_version="v3.1", + dispositions=frozenset({NOT_AFFECTED}), + neutralizers=("decode_to_utf8",), + detectors=(), + probe="\ufffd\ufffd", + reference="https://nvd.nist.gov/vuln/detail/CVE-2026-44288", + ), + CVE( + id="CVE-2009-4142", + title="PHP htmlspecialchars — overlong UTF-8 and invalid Shift_JIS/EUC-JP", + cwe="CWE-79", + cvss=4.3, + cvss_version="v2.0", + dispositions=frozenset({NOT_AFFECTED}), + neutralizers=("decode_to_utf8",), + detectors=(), + probe="\ufffd\x00"), ] + +def _no_surrogate(text: str) -> bool: + return not any(0xD800 <= ord(ch) <= 0xDFFF for ch in text) + + #: (cve, attack, predicate) — handled when the primitive is gone. REMOVAL_VECTORS = [ ("CVE-2021-42574", TROJAN_C, lambda o: not any(c in o for c in BIDI_CONTROLS)), @@ -1988,6 +2367,8 @@ def test_disarm_does_not_bound_input_length(self) -> None: ("CVE-2023-43620", CROC_FILENAME, lambda o: not any(c in o for c in TERMINAL_CONTROLS)), ("CVE-2023-37275", AUTOGPT_OUTPUT, lambda o: not any(c in o for c in TERMINAL_CONTROLS)), ("CVE-2017-20190", ZALGO_PILE, lambda o: len(o) <= 4), + ("CVE-2022-31116", LONE_LOW_SURROGATE, _no_surrogate), + ("CVE-2025-64439", LONE_HIGH_SURROGATE, _no_surrogate), ] NEUTRALIZABLE = [c for c, _, _ in COLLAPSE_VECTORS] + [c for c, _, _ in REMOVAL_VECTORS] @@ -2172,6 +2553,7 @@ def test_the_narrow_presets_are_narrow(self) -> None: "CVE-2017-5383", "CVE-2017-20190", "CVE-2019-11721", + "CVE-2007-2688", }, sorted(missed) @@ -2202,6 +2584,18 @@ class TestDetectionHasNoSuperset: "CVE-2025-32711", # nor is the Tags block "CVE-2026-23950", # nor is a case-folding path collision "CVE-2023-46695", # nor is a long run of already-normalized characters + # The whole encoding class is silent too — see TestOverlongAndInvalid- + # Sequences and TestLoneSurrogates. Every one is neutralized and none + # is reported. + "CVE-2022-31116", + "CVE-2025-64439", + "CVE-2007-2688", + # The byte-level rows: not-affected, and nothing reports them either. + # `decode_to_utf8` returns `had_errors`, which is a return value rather + # than one of the panel predicates. + "CVE-2024-46954", + "CVE-2026-44288", + "CVE-2009-4142", } @staticmethod