diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b9a2ebc..eafd1be4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,42 @@ compatibility (see [RELEASING.md](RELEASING.md)). ### Added +- **`is_case_fold_stable` — ask whether a value is a stable identity key before you key a + table on it (#619).** Answers `fold_case(x) == x.lower()`. A `False` says some *other* + string folds to the same value, which is the precondition node-tar's `PathReservations` + guard missed in CVE-2026-23950: `groß.txt` and `gross.txt` are one path on a + case-insensitive filesystem. Available in Rust (`api::is_case_fold_stable`, and on + `DisarmStr`), Python (`is_case_fold_stable`, `Text.is_case_fold_stable`), Node + (`isCaseFoldStable`), Ruby (`Disarm.case_fold_stable?`), the C ABI and Java. + + **It states a fact about the string, not suspicion.** `groß` is an ordinary German word + and `file` an ordinary ligature, so the predicate reads `True` for ordinary text and is + deliberately kept out of `has_anomalies` and out of the CVE detector panel — folding it + in would flag ordinary German and every Greek word ending in sigma. What to do about a + `False` is the caller's decision: reserve both forms, reject the name, or key the table + on `fold_case` rather than `str.lower()`. + + **`str.lower()` is the comparison basis and `str.casefold()` is not.** Casefolding + performs the very transform under test, so a predicate written against it answers + `True` for every string in Unicode — the substitution #617 already made once, now + pinned by a test. + + **Not a per-character table, because a per-character table is wrong for Greek.** + `ΟΔΟΣ` ("street") lowercases to `οδος` and folds to `οδοσ`, yet `Σ` agrees with itself + in isolation; `U+03A3` is the only code point in Unicode whose lowercase mapping depends + on its neighbours, which a Tier-3 test asserts by enumeration rather than by assertion. + The implementation is allocation-free for anything that contains no capital sigma — + ASCII short-circuits, everything else scans the folding table in place — and falls back + to the exact string comparison for the rest, so it cannot drift from what `fold_case` + actually does. + + CVE-2026-23950's *Detected by* column stops reading `—`. The collision itself is still a + property of a pair of names and no single-string predicate can report it (#620 tracks + that); the precondition is what moved. **Measured limit:** the issue paired this with + CVE-2019-19844, and that half does not hold — that row's probe turns on `U+0131` DOTLESS + I, which folds *and* lowercases to itself and collides through `.upper()` instead, so + the predicate is silent on it. + - **Ten CVEs on encoding rather than code points, and the survey method behind them (36 → 46 rows).** Found by sweeping NVD across the operations disarm performs, then verified one ID at a time. diff --git a/bindings/cabi/disarm.h b/bindings/cabi/disarm.h index 31c71d1d..bd8adaa3 100644 --- a/bindings/cabi/disarm.h +++ b/bindings/cabi/disarm.h @@ -161,6 +161,14 @@ char * disarm_inspect_auto_lang ( char const * text); +/** \brief + * Whether case folding and simple lowercasing agree, so `text` is a stable + * identity key ("groß.txt" is not; "gross.txt" is). + */ +bool +disarm_is_case_fold_stable ( + char const * text); + /** \brief * Whether `text` mixes characters from more than one script. */ diff --git a/bindings/cabi/src/lib.rs b/bindings/cabi/src/lib.rs index 9fb8bc08..0e651d78 100644 --- a/bindings/cabi/src/lib.rs +++ b/bindings/cabi/src/lib.rs @@ -175,6 +175,13 @@ fn disarm_fold_case(text: char_p::Ref<'_>) -> char_p::Box { to_c(api::fold_case(text.to_str()).into_owned()) } +/// Whether case folding and simple lowercasing agree, so `text` is a stable +/// identity key ("groß.txt" is not; "gross.txt" is). +#[ffi_export] +fn disarm_is_case_fold_stable(text: char_p::Ref<'_>) -> bool { + api::is_case_fold_stable(text.to_str()) +} + /// Replace emoji with their plain names; `strip_modifiers` drops skin-tone marks. #[ffi_export] fn disarm_demojize(text: char_p::Ref<'_>, strip_modifiers: bool) -> char_p::Box { diff --git a/bindings/java/disarm-java/src/main/java/dev/disarm/Disarm.java b/bindings/java/disarm-java/src/main/java/dev/disarm/Disarm.java index fd47d82a..5220d870 100644 --- a/bindings/java/disarm-java/src/main/java/dev/disarm/Disarm.java +++ b/bindings/java/disarm-java/src/main/java/dev/disarm/Disarm.java @@ -90,6 +90,18 @@ public static String foldCase(String text) { return Native.foldCase(req(text)); } + /** + * Whether {@code text} is a stable identity key under case folding — whether + * {@link #foldCase} and {@code String.toLowerCase()} agree on it. + * + *

{@code false} means some other string folds to the same value, so a table keyed on + * this one can collide: {@code "groß.txt"} and {@code "gross.txt"} are the pair node-tar + * collided on (CVE-2026-23950). It states a fact about the string, not suspicion. + */ + public static boolean isCaseFoldStable(String text) { + return Native.isCaseFoldStable(req(text)); + } + /** Replace emoji with their plain names (skin-tone modifiers preserved). */ public static String demojize(String text) { return demojize(text, false); diff --git a/bindings/java/disarm-java/src/main/java/dev/disarm/internal/Native.java b/bindings/java/disarm-java/src/main/java/dev/disarm/internal/Native.java index c8e547ce..8e33a70c 100644 --- a/bindings/java/disarm-java/src/main/java/dev/disarm/internal/Native.java +++ b/bindings/java/disarm-java/src/main/java/dev/disarm/internal/Native.java @@ -48,6 +48,8 @@ public static native String normalizeConfusables( public static native String foldCase(String text); + public static native boolean isCaseFoldStable(String text); + public static native String demojize(String text, boolean stripModifiers); // ── Normalization ────────────────────────────────────────────────────────── diff --git a/bindings/java/disarm-java/src/test/java/dev/disarm/DisarmCoverageTest.java b/bindings/java/disarm-java/src/test/java/dev/disarm/DisarmCoverageTest.java index fb49abbf..ff1d3359 100644 --- a/bindings/java/disarm-java/src/test/java/dev/disarm/DisarmCoverageTest.java +++ b/bindings/java/disarm-java/src/test/java/dev/disarm/DisarmCoverageTest.java @@ -47,6 +47,13 @@ void stripPua() { assertEquals("ab", Disarm.stripPua("ab")); // Private Use Area } + @Test + void isCaseFoldStable() { + assertTrue(Disarm.isCaseFoldStable("gross.txt")); + assertFalse(Disarm.isCaseFoldStable("groß.txt")); // collides with gross.txt + assertThrows(NullPointerException.class, () -> Disarm.isCaseFoldStable(null)); + } + // ── Overloads with explicit optional arguments ────────────────────────────── @Test diff --git a/bindings/java/disarm-kotlin/src/main/kotlin/dev/disarm/kotlin/Disarm.kt b/bindings/java/disarm-kotlin/src/main/kotlin/dev/disarm/kotlin/Disarm.kt index 1dbeade0..e6ec7542 100644 --- a/bindings/java/disarm-kotlin/src/main/kotlin/dev/disarm/kotlin/Disarm.kt +++ b/bindings/java/disarm-kotlin/src/main/kotlin/dev/disarm/kotlin/Disarm.kt @@ -75,6 +75,14 @@ fun String.stripAccents(): String = JDisarm.stripAccents(this) fun String.foldCase(): String = JDisarm.foldCase(this) +/** + * Whether this value is a stable identity key under case folding — whether [foldCase] + * and `lowercase()` agree on it. `false` means some other string folds to the same + * value, the collision node-tar hit in CVE-2026-23950. A fact about the string, not + * suspicion: `groß` is an ordinary German word. + */ +fun String.isCaseFoldStable(): Boolean = JDisarm.isCaseFoldStable(this) + @JvmOverloads fun String.demojize(stripModifiers: Boolean = false): String = JDisarm.demojize(this, stripModifiers) diff --git a/bindings/java/disarm-kotlin/src/test/kotlin/dev/disarm/kotlin/DisarmKtTest.kt b/bindings/java/disarm-kotlin/src/test/kotlin/dev/disarm/kotlin/DisarmKtTest.kt index 619eed4a..9d254164 100644 --- a/bindings/java/disarm-kotlin/src/test/kotlin/dev/disarm/kotlin/DisarmKtTest.kt +++ b/bindings/java/disarm-kotlin/src/test/kotlin/dev/disarm/kotlin/DisarmKtTest.kt @@ -53,6 +53,8 @@ class DisarmKtTest { fun canonicalizationPrimitives() { assertEquals("cafe", "café".stripAccents()) assertEquals("hello", "Hello".foldCase()) + assertTrue("gross.txt".isCaseFoldStable()) + assertFalse("groß.txt".isCaseFoldStable()) assertFalse("😀".demojize().isBlank()) assertFalse("👍🏽".demojize(stripModifiers = true).isBlank()) } diff --git a/bindings/java/rust/src/lib.rs b/bindings/java/rust/src/lib.rs index 426a5fb8..b0a7073b 100644 --- a/bindings/java/rust/src/lib.rs +++ b/bindings/java/rust/src/lib.rs @@ -529,6 +529,21 @@ pub fn foldCase<'l>(env: EnvUnowned<'l>, _class: JClass<'l>, input: JString<'l>) map_str(env, input, |t| api::fold_case(t).into_owned()) } +/// Whether case folding and simple lowercasing agree, so `text` is a stable +/// identity key ("groß.txt" is not; "gross.txt" is). +#[jni_mangle("dev.disarm.internal.Native")] +pub fn isCaseFoldStable<'l>( + mut env: EnvUnowned<'l>, + _class: JClass<'l>, + input: JString<'l>, +) -> jboolean { + env.with_env(|env| -> JniResult { + let text = input.mutf8_chars(env)?.to_string(); + Ok(api::is_case_fold_stable(&text)) + }) + .resolve::() +} + /// Replace emoji with their plain names; `stripModifiers` drops skin-tone marks. #[jni_mangle("dev.disarm.internal.Native")] pub fn demojize<'l>( diff --git a/bindings/node/__test__/disarm.test.mjs b/bindings/node/__test__/disarm.test.mjs index 5c4a99f2..a6efbc3e 100644 --- a/bindings/node/__test__/disarm.test.mjs +++ b/bindings/node/__test__/disarm.test.mjs @@ -61,6 +61,14 @@ describe('slugify', () => { describe('canonicalization', () => { test('stripAccents', () => expect(disarm.stripAccents('café')).toBe('cafe')) test('foldCase', () => expect(disarm.foldCase('HELLO')).toBe('hello')) + test('isCaseFoldStable', () => { + expect(disarm.isCaseFoldStable('gross.txt')).toBe(true) + // The pair node-tar collided on: both sides reduce to gross.txt. + expect(disarm.isCaseFoldStable('groß.txt')).toBe(false) + // Greek final sigma — the whole-string answer, not a per-character one. + expect(disarm.isCaseFoldStable('ΟΔΟΣ')).toBe(false) + expect(disarm.isCaseFoldStable('ΣΑΒΒΑΤΟ')).toBe(true) + }) test('demojize', () => expect(disarm.demojize('hi 👍')).toBe('hi thumbs up')) }) diff --git a/bindings/node/index.ts b/bindings/node/index.ts index 48c42e56..0a22e9a2 100644 --- a/bindings/node/index.ts +++ b/bindings/node/index.ts @@ -254,6 +254,20 @@ export function foldCase(text: string): string { return native.foldCase(text) } +/** + * Whether `text` is a stable identity key under case folding — that is, whether + * {@link foldCase} and `String.toLowerCase()` agree on it (#619). + * + * `false` means some *other* string folds to the same value, so a table keyed on + * this one can collide: `'groß.txt'` and `'gross.txt'` are the pair node-tar + * collided on (CVE-2026-23950). It is a fact about the string and not an + * accusation — `groß` is an ordinary German word — so it is deliberately not + * folded into {@link hasAnomalies}. + */ +export function isCaseFoldStable(text: string): boolean { + return native.isCaseFoldStable(text) +} + /** Replace emoji with their plain names. `stripModifiers` drops skin-tone/variation marks. */ export function demojize(text: string, options: { stripModifiers?: boolean } = {}): string { return native.demojize(text, options.stripModifiers ?? false) diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index bdfc8223..d3796237 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -249,6 +249,13 @@ pub fn fold_case(text: String) -> String { api::fold_case(&text).into_owned() } +/// Whether case folding and simple lowercasing agree, so the value is a stable +/// identity key (#619). +#[napi] +pub fn is_case_fold_stable(text: String) -> bool { + api::is_case_fold_stable(&text) +} + /// Replace emoji with their plain names; `strip_modifiers` drops skin-tone marks. #[napi] pub fn demojize(text: String, strip_modifiers: bool) -> String { diff --git a/bindings/ruby/ext/disarm/src/lib.rs b/bindings/ruby/ext/disarm/src/lib.rs index 5e088f84..9f6a4e0f 100644 --- a/bindings/ruby/ext/disarm/src/lib.rs +++ b/bindings/ruby/ext/disarm/src/lib.rs @@ -255,6 +255,12 @@ fn fold_case(text: Wtf8Text) -> String { api::fold_case(&text).into_owned() } +/// `Disarm._is_case_fold_stable?(text)` — whether case folding and simple +/// lowercasing agree, so the value is a stable identity key (#619). +fn is_case_fold_stable(text: Wtf8Text) -> bool { + api::is_case_fold_stable(&text) +} + /// `Disarm._slugify(text, …)` — the full slug option surface, positional. The /// Ruby layer maps its keyword arguments (with the core's documented defaults) /// onto this order. `regex_pattern` and `replacements` are intentionally not @@ -779,6 +785,10 @@ fn init(ruby: &Ruby) -> Result<(), Error> { // keeping `rescue Disarm::Error` exhaustive across the whole public surface. module.define_singleton_method("_strip_accents", function!(strip_accents, 1))?; module.define_singleton_method("_fold_case", function!(fold_case, 1))?; + module.define_singleton_method( + "_is_case_fold_stable?", + function!(is_case_fold_stable, 1), + )?; module.define_singleton_method("_suspicious_hostname?", function!(suspicious_hostname, 1))?; module.define_singleton_method("_analyze_hostname", function!(analyze_hostname, 2))?; diff --git a/bindings/ruby/lib/disarm.rb b/bindings/ruby/lib/disarm.rb index 0390963f..c4a0a6f3 100644 --- a/bindings/ruby/lib/disarm.rb +++ b/bindings/ruby/lib/disarm.rb @@ -161,6 +161,15 @@ def fold_case(text) translate_errors { _fold_case(text) } end + # Whether `text` is a stable identity key under case folding — whether + # `fold_case` and `String#downcase` agree on it (#619). A false result means + # some other string folds to the same value, so a table keyed on this one can + # collide: "groß.txt" and "gross.txt" are the pair node-tar collided on + # (CVE-2026-23950). It is a fact about the string, not an accusation. + def case_fold_stable?(text) + translate_errors { _is_case_fold_stable?(text) } + end + # Whether the hostname looks like a mixed-script / confusable / bidi-reorder # IDN spoof. Flags a mixed-script label, a Latin confusable, or a # bidi-direction conflict (see #bidi_conflict?, the "BiDi Swap" precondition), diff --git a/bindings/ruby/spec/disarm_spec.rb b/bindings/ruby/spec/disarm_spec.rb index 464165c3..8535a2a1 100644 --- a/bindings/ruby/spec/disarm_spec.rb +++ b/bindings/ruby/spec/disarm_spec.rb @@ -120,6 +120,15 @@ it "case-folds" do expect(Disarm.fold_case("HELLO")).to eq("hello") end + + it "reports whether a value is a stable key under case folding" do + expect(Disarm.case_fold_stable?("gross.txt")).to be(true) + # The pair node-tar collided on (CVE-2026-23950). + expect(Disarm.case_fold_stable?("groß.txt")).to be(false) + # Greek final sigma — the whole-string answer, not a per-character one. + expect(Disarm.case_fold_stable?("ΟΔΟΣ")).to be(false) + expect(Disarm.case_fold_stable?("ΣΑΒΒΑΤΟ")).to be(true) + end end describe "security" do diff --git a/bindings/ruby/spec/surrogate_spec.rb b/bindings/ruby/spec/surrogate_spec.rb index 5337f77b..949aad4b 100644 --- a/bindings/ruby/spec/surrogate_spec.rb +++ b/bindings/ruby/spec/surrogate_spec.rb @@ -47,6 +47,7 @@ "transliterate" => ->(s) { Disarm.transliterate(s) }, "strip_accents" => ->(s) { Disarm.strip_accents(s) }, "fold_case" => ->(s) { Disarm.fold_case(s) }, + "case_fold_stable?" => ->(s) { Disarm.case_fold_stable?(s) }, "search_key" => ->(s) { Disarm.search_key(s) }, "sort_key" => ->(s) { Disarm.sort_key(s) }, "catalog_key" => ->(s) { Disarm.catalog_key(s) } diff --git a/docs/api/predicates.md b/docs/api/predicates.md index 2d1c35bf..3e7f832c 100644 --- a/docs/api/predicates.md +++ b/docs/api/predicates.md @@ -65,6 +65,30 @@ See [Language Detection](../user-guide/language-detection.md#inspecting-detectio --- +## is_case_fold_stable + +::: disarm.is_case_fold_stable + +```python +from disarm import is_case_fold_stable + +is_case_fold_stable("gross.txt") # True +is_case_fold_stable("groß.txt") # False — folds to gross.txt, so the two collide +is_case_fold_stable("file") # False — folds to file +is_case_fold_stable("ΟΔΟΣ") # False — lowercases to οδος, folds to οδοσ +``` + +Use it before a name becomes a key: a reservation table, a username registry, an +extraction path. `False` says the value shares its folded form with some other +string, which is the precondition node-tar's `PathReservations` guard missed in +CVE-2026-23950. It says nothing about intent, since `groß` is an ordinary German +word, so the predicate is kept out of +[anomaly detection](../user-guide/anomaly-detection.md) and the response is the +caller's to choose: reserve both forms, reject the name, or key the table on +[`fold_case`](transforms.md#fold_case) instead of `str.lower()`. + +--- + ## is_normalized ::: disarm.is_normalized diff --git a/docs/node/api.md b/docs/node/api.md index c81e92ce..00f2b265 100644 --- a/docs/node/api.md +++ b/docs/node/api.md @@ -112,6 +112,20 @@ foldCase('Straße') // => 'strasse' demojize('Café ☕') // => 'Café hot beverage' ``` +### `isCaseFoldStable(text)` + +Whether `foldCase` and `String.toLowerCase()` agree on `text`, so it is a stable +identity key. `false` means some other string folds to the same value, which is +the collision node-tar's `PathReservations` guard missed in CVE-2026-23950. Ask +it before a name becomes a key; a `false` is a fact about the string rather than +a report of an attack, since `groß` is an ordinary German word. + +```ts +isCaseFoldStable('gross.txt') // => true +isCaseFoldStable('groß.txt') // => false, folds to gross.txt +isCaseFoldStable('ΟΔΟΣ') // => false, lowercases to οδος and folds to οδοσ +``` + ## Normalization ### `normalize(text, options?)` · `isNormalized(text, options?)` diff --git a/docs/ruby/api.md b/docs/ruby/api.md index f52d7846..393ae497 100644 --- a/docs/ruby/api.md +++ b/docs/ruby/api.md @@ -101,6 +101,20 @@ Disarm.fold_case("HELLO") # => "hello" Disarm.fold_case("Straße") # => "strasse" ``` +### `Disarm.case_fold_stable?(text)` + +Whether `fold_case` and `String#downcase` agree on `text`, so it is a stable +identity key. `false` means some other string folds to the same value, which is +the collision node-tar's `PathReservations` guard missed in CVE-2026-23950. Ask +it before a name becomes a key; a `false` is a fact about the string rather than +a report of an attack, since `groß` is an ordinary German word. + +```ruby +Disarm.case_fold_stable?("gross.txt") # => true +Disarm.case_fold_stable?("groß.txt") # => false, folds to gross.txt +Disarm.case_fold_stable?("ΟΔΟΣ") # => false, downcases to οδος +``` + ### `Disarm.demojize(text, strip_modifiers: false)` Replace emoji with their plain names. `strip_modifiers:` drops skin-tone / diff --git a/docs/security/cve-validation.md b/docs/security/cve-validation.md index 11443dd7..4003464a 100644 --- a/docs/security/cve-validation.md +++ b/docs/security/cve-validation.md @@ -18,8 +18,11 @@ and CVE-2019-19844 is the clearest case: its neutralizers detect nothing, and its only detector rewrites nothing. The detector lists are derived rather than written down — the test suite runs -each row's vector through every detector and asserts the list matches what -actually fired. +each row's vector through the general-purpose detectors and asserts the list +matches what actually fired. Two detectors sit outside that panel because they +only answer for one shape of input or one kind of question: +`is_suspicious_hostname` and `is_case_fold_stable`. Each row that names one is +asserted against it directly, in the class for that CVE. **Out of scope** means disarm does not stop it and is not supposed to. Those rows are asserted as negatives, so a limit cannot quietly become a claim. @@ -79,7 +82,7 @@ ranking. | [CVE-2023-37275](https://nvd.nist.gov/vuln/detail/CVE-2023-37275) | Auto-GPT — console spoofing via ANSI relayed through an LLM | 4.3 (v3.1) | Neutralized + detected | `strip_log_injection`, `canonicalize` | `has_anomalies` | | [CVE-2019-11721](https://nvd.nist.gov/vuln/detail/CVE-2019-11721) | Firefox — Latin kra spoofing 'k' in the address bar | 6.5 (v3.1) | Neutralized + detected | `normalize_confusables`, `canonicalize`, `strip_obfuscation` | `is_confusable`, `is_suspicious_hostname` | | [CVE-2023-4399](https://nvd.nist.gov/vuln/detail/CVE-2023-4399) | Grafana — request deny list bypassed by punycode encoding | 7.2 (v3.1) | Detected only | — | `is_suspicious_hostname` | -| [CVE-2026-23950](https://nvd.nist.gov/vuln/detail/CVE-2026-23950) | node-tar — symlink poisoning via a Unicode path collision | 5.9 (v3.1) | Neutralized | `fold_case`, `search_key`, `catalog_key` | — | +| [CVE-2026-23950](https://nvd.nist.gov/vuln/detail/CVE-2026-23950) | node-tar — symlink poisoning via a Unicode path collision | 5.9 (v3.1) | Neutralized + detected | `fold_case`, `search_key`, `catalog_key` | `is_case_fold_stable` | | [CVE-2026-3276](https://nvd.nist.gov/vuln/detail/CVE-2026-3276) | CPython — unicodedata.normalize() CPU blowup on alternating-CCC runs | 6.3 (v4.0) | Not affected + detected | `normalize` | `is_zalgo`, `has_anomalies` | | [CVE-2023-46695](https://nvd.nist.gov/vuln/detail/CVE-2023-46695) | Django — NFKC normalization slow on Windows, DoS via UsernameField | 7.5 (v3.1) | Not affected | `normalize` | — | | [CVE-2017-20190](https://nvd.nist.gov/vuln/detail/CVE-2017-20190) | Windows — performance degradation from piled combining marks (Zalgo) | none (SSVC only) | Neutralized + detected | `strip_zalgo`, `canonicalize`, `strip_obfuscation` | `is_zalgo`, `has_anomalies` | @@ -166,21 +169,28 @@ them entirely: a leading NUL, the whole terminal-control class, and one byte-lev row whose probe is itself a NUL injection. The `control` anomaly kind (#612) closed all eight in one branch, and `has_anomalies` went from 11 rows to 19. +CVE-2026-23950 then left the list by a route this section had ruled out. Its +collision really is a property of a pair of names, and no single-string predicate +can say that `groß.txt` and `gross.txt` are the same path. But the *precondition* +is a single-string property, and it is the one a reservation table needs: +`is_case_fold_stable` (#619) answers whether full case folding and `str.lower()` +agree, so `groß.txt` reads `False` before anything has been extracted. It is not +in the detector panel and not in `has_anomalies`, because `groß` is an ordinary +German word and calling it suspicious would be both noisy and untrue. + What remains is a different shape, which is the useful part: | Vector | Why nothing flags it | |---|---| | CVE-2025-32711 | The Unicode Tags block is not an anomaly kind | -| CVE-2026-23950 | Nor is a case-folding path collision | | CVE-2023-46695 | Nor is a long run of already-normalized characters | | CVE-2022-31116, CVE-2025-64439, CVE-2007-2688 | Nor is a malformed encoding — by the time text reaches a detector it has already been decoded | | CVE-2024-46954, CVE-2026-44288 | Nor is a decode failure: `decode_to_utf8` returns `had_errors`, which is a return value rather than a panel predicate | -Not one of them is a character you can look for. Each needs a *comparison* — a -fold collision against another string, a length budget, a decode result — so no -number of additional character classes will close any of them. That is the line -this section is really drawing, and it is why the rule below follows from the -measurement rather than from taste: +Not one of them is a character you can look for. Each needs a *comparison*: a +length budget, a decode result. No number of additional character classes will +close any of them. That is the line this section is really drawing, and it is +why the rule below follows from the measurement rather than from taste: **Clean unconditionally. Use the detectors to decide whether to alert, never whether to clean.** Every one is still *neutralized*, so a pipeline that screens diff --git a/generated/parity.yaml b/generated/parity.yaml index 402d1f31..73f04299 100644 --- a/generated/parity.yaml +++ b/generated/parity.yaml @@ -163,6 +163,12 @@ operations: python: is_ascii ruby: null node: null + - id: is_case_fold_stable + names: + rust: is_case_fold_stable + python: is_case_fold_stable + ruby: case_fold_stable? + node: isCaseFoldStable - id: is_confusable names: rust: is_confusable diff --git a/python/disarm/__init__.py b/python/disarm/__init__.py index 364ee94a..92bdb1b8 100644 --- a/python/disarm/__init__.py +++ b/python/disarm/__init__.py @@ -37,6 +37,7 @@ inspect_anomalies, inspect_auto_lang, is_ascii, + is_case_fold_stable, is_confusable, is_mixed_script, is_normalized, @@ -310,6 +311,7 @@ "is_confusable", "unmapped_confusables", "is_ascii", + "is_case_fold_stable", "is_normalized", # Preset metadata "PRESETS", diff --git a/python/disarm/_api.py b/python/disarm/_api.py index c2390e18..4b9d7c73 100644 --- a/python/disarm/_api.py +++ b/python/disarm/_api.py @@ -49,6 +49,7 @@ _inspect_anomalies_lex, _inspect_auto_lang, _is_ascii, + _is_case_fold_stable, _is_confusable, _is_mixed_script, _is_normalized, @@ -908,6 +909,55 @@ def fold_case(text: str) -> str: casefold = fold_case +def is_case_fold_stable(text: str) -> bool: + """True if ``text`` is a stable identity key under case folding. + + Answers ``fold_case(text) == text.lower()``. A ``False`` result says some + *other* string folds to the same value, so a table keyed on this one can + collide — ``groß.txt`` and ``gross.txt`` are the pair node-tar collided on + (CVE-2026-23950), and ``ſtraße``/``straße`` and ``file``/``file`` are the same + shape. Roughly 2,000 code points behave this way, including every Latin + ligature, ``ẛ``, the micro sign, and all of Cherokee (whose fold direction + runs small→capital, so both cases move). + + **This is a fact about the string, not an accusation.** ``groß`` is an + ordinary German word, so a ``False`` here is not a report of an attack and + the predicate is deliberately kept out of :func:`has_anomalies`. What to do + about it is the caller's decision: reserve both forms, reject the name, or + key the table on :func:`fold_case` rather than ``str.lower()``. + + ``str.lower()`` is the correct comparison basis and ``str.casefold()`` is + not: casefolding performs the very transform under test, so a predicate + written against it is ``True`` everywhere. + + Answers about disarm's own folding table (Unicode 16.0), so it also reports + ``False`` for characters your Python's ``str.lower()`` knows about and that + table does not — which is a collision hazard for the same reason. + + A ``True`` result is **not** a uniqueness guarantee: two distinct stable + strings can still collide under some *other* normalization. + + Args: + text: Input string. + + Returns: + True if full case folding and simple lowercasing agree on ``text``. + + Examples: + >>> is_case_fold_stable("gross.txt") + True + >>> is_case_fold_stable("groß.txt") + False + >>> is_case_fold_stable("ΟΔΟΣ") # Greek final sigma: οδος vs οδοσ + False + """ + if not isinstance(text, str): + raise TypeError(f"is_case_fold_stable() expects str, got {type(text).__name__}") + if text.isascii(): + return True + return _is_case_fold_stable(text) + + def collapse_whitespace(text: str) -> str: """Fold all Unicode whitespace runs to single ASCII spaces, trimming the ends. diff --git a/python/disarm/_boundary.pyi b/python/disarm/_boundary.pyi index 9bbfb680..ef423942 100644 --- a/python/disarm/_boundary.pyi +++ b/python/disarm/_boundary.pyi @@ -116,6 +116,9 @@ from disarm._core import ( from disarm._core import ( _is_ascii as _is_ascii, ) +from disarm._core import ( + _is_case_fold_stable as _is_case_fold_stable, +) from disarm._core import ( _is_confusable as _is_confusable, ) diff --git a/python/disarm/_core.pyi b/python/disarm/_core.pyi index f604aa98..917e5045 100644 --- a/python/disarm/_core.pyi +++ b/python/disarm/_core.pyi @@ -208,6 +208,7 @@ def _sanitize_filename( ) -> str: ... def _strip_accents(text: str) -> str: ... def _fold_case(text: str) -> str: ... +def _is_case_fold_stable(text: str) -> bool: ... def _collapse_whitespace(text: str) -> str: ... def _strip_control_chars(text: str) -> str: ... def _strip_zero_width_chars(text: str) -> str: ... diff --git a/python/disarm/_text.py b/python/disarm/_text.py index 69a99bfe..89659103 100644 --- a/python/disarm/_text.py +++ b/python/disarm/_text.py @@ -373,6 +373,12 @@ def is_ascii(self) -> bool: """True if all characters are U+0000–U+007F.""" return self._t().is_ascii(self._value) + def is_case_fold_stable(self) -> bool: + """True if full case folding and ``str.lower()`` agree, so the value is a + stable identity key. ``False`` means another string folds to the same + thing (``groß.txt`` / ``gross.txt``) — a fact, not an accusation.""" + return self._t().is_case_fold_stable(self._value) + def is_normalized(self, *, form: NormalizationForm = "NFC") -> bool: """True if already in the specified normalization form.""" return self._t().is_normalized(self._value, form=form) diff --git a/python/disarm/_text.pyi b/python/disarm/_text.pyi index c8c91ce1..78856503 100644 --- a/python/disarm/_text.pyi +++ b/python/disarm/_text.pyi @@ -134,6 +134,9 @@ class Text: def is_ascii(self) -> bool: """True if all characters are U+0000–U+007F.""" ... + def is_case_fold_stable(self) -> bool: + """True if case folding and str.lower() agree, so the value is a stable key.""" + ... def is_normalized(self, *, form: NormalizationForm = "NFC") -> bool: """True if already in the specified normalization form.""" ... diff --git a/python/disarm/normalization.py b/python/disarm/normalization.py index eca5064b..54238c36 100644 --- a/python/disarm/normalization.py +++ b/python/disarm/normalization.py @@ -12,6 +12,7 @@ from disarm import ( collapse_whitespace, fold_case, + is_case_fold_stable, is_normalized, normalize, strip_accents, @@ -20,6 +21,7 @@ __all__ = [ "collapse_whitespace", "fold_case", + "is_case_fold_stable", "is_normalized", "normalize", "strip_accents", diff --git a/scripts/parity.py b/scripts/parity.py index a0359366..5a9f9007 100644 --- a/scripts/parity.py +++ b/scripts/parity.py @@ -63,6 +63,7 @@ def camel_to_snake(s): RUBY_PRED = { + "case_fold_stable": "is_case_fold_stable", "normalized": "is_normalized", "mixed_script": "is_mixed_script", "bidi_conflict": "has_bidi_conflict", diff --git a/src/api/mod.rs b/src/api/mod.rs index a0034139..b7c0538a 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -59,6 +59,11 @@ pub trait DisarmStr: AsRef { fn fold_case(&self) -> Cow<'_, str> { fold_case(self.as_ref()) } + /// See [`is_case_fold_stable`]. + #[must_use] + fn is_case_fold_stable(&self) -> bool { + is_case_fold_stable(self.as_ref()) + } /// See [`strip_accents`]. #[must_use] fn strip_accents(&self) -> Cow<'_, str> { diff --git a/src/api/text.rs b/src/api/text.rs index 4339f223..84529d99 100644 --- a/src/api/text.rs +++ b/src/api/text.rs @@ -121,6 +121,35 @@ pub fn fold_case(text: &str) -> Cow<'_, str> { crate::case_fold::fold_case_cow(text) } +/// True when `text` is a stable identity key under case folding — when +/// [`fold_case`] and `str::to_lowercase` agree on it (#619). +/// +/// `false` means some *other* string folds to the same value, so a table keyed +/// on this one can collide: `groß`/`gross` (CVE-2026-23950), `ſtraße`/`straße`, +/// `file`/`file`. Every member of that class is ordinary text in some language, +/// so this reports a fact about the string and not suspicion, and it is +/// deliberately **not** folded into [`crate::api::has_anomalies`]. +/// +/// It compares disarm's bundled CaseFolding table against the `to_lowercase` +/// compiled into the crate, and those two carry their own Unicode versions. The +/// answer turns on whether the two *results* differ, so a code point only one of +/// them has a mapping for reads `false` — the right answer for the same reason, +/// since two functions that disagree build two keys that disagree. A code point +/// neither has a mapping for is left alone by both and reads `true`. +/// +/// A `true` answer is not a promise the value is unique; two distinct stable +/// strings can still be equal after some *other* normalization step. +/// +/// ``` +/// use disarm::api; +/// assert!(api::is_case_fold_stable("gross.txt")); +/// assert!(!api::is_case_fold_stable("groß.txt")); +/// ``` +#[must_use] +pub fn is_case_fold_stable(text: &str) -> bool { + crate::case_fold::is_case_fold_stable_impl(text) +} + // ── Grapheme clusters (UAX #29) ────────────────────────────────────────────── /// Number of user-perceived characters (extended grapheme clusters): `"👩‍👩‍👧‍👦"` → 1. diff --git a/src/case_fold.rs b/src/case_fold.rs index df9f787a..d215aeb6 100644 --- a/src/case_fold.rs +++ b/src/case_fold.rs @@ -99,6 +99,63 @@ pub(crate) fn fold_case_into(text: &str, result: &mut String) { } } +/// True when full case folding and simple lowercasing agree on `text`, i.e. +/// `fold_case(text) == text.to_lowercase()`. +/// +/// A `false` answer says the value is **not a stable identity key**: some other +/// string folds to the same thing, so keying a table on it can collide. `groß` +/// and `gross` are the canonical pair (CVE-2026-23950), `ſtraße` and `straße` +/// the less obvious one. Nothing about a `false` is an accusation — `groß.txt` +/// is an ordinary German filename — which is why the question is phrased as a +/// property of the string rather than as suspicion, and why it stays out of +/// [`crate::api::has_anomalies`]. +/// +/// Comparing against `str::to_lowercase` is the point. Comparing against +/// `str::to_uppercase` answers a different question, and comparing against +/// `char::to_lowercase` per character is wrong for Greek (below); comparing +/// against a case *fold* is not a comparison at all, since that performs the +/// very transform under test and the predicate collapses to `true` everywhere. +/// +/// Three paths, in cost order: +/// 1. Pure ASCII is always stable — ASCII folds and lowercases identically. +/// 2. Per-character scan against the folding table, allocation-free. +/// 3. Exact whole-string comparison, reached only when `U+03A3` is present. +/// +/// Step 3 exists because `str::to_lowercase` applies the Final_Sigma context +/// rule and case folding has no context rule at all: `ΟΔΟΣ` lowercases to +/// `οδος` and folds to `οδοσ`, although `Σ` agrees with itself in isolation and +/// so passes step 2. `U+03A3` is the only code point in Unicode whose lowercase +/// mapping depends on its neighbours (asserted exhaustively by +/// `only_sigma_has_a_context_sensitive_lowercase`), so the allocating path is +/// reached only by text containing a capital sigma. Step 2 notes the sigma as it +/// passes rather than re-scanning for it, so the whole predicate is one pass. +pub(crate) fn is_case_fold_stable_impl(text: &str) -> bool { + // ASCII folds and lowercases identically, so no ASCII string can be + // unstable — pinned over all 128 by `ascii_folds_and_lowercases_identically`. + if text.is_ascii() { + return true; + } + + let mut saw_capital_sigma = false; + for ch in text.chars() { + saw_capital_sigma |= ch == '\u{03A3}'; + let agrees = match case_folding_data::lookup(ch) { + Some(folded) => folded.chars().eq(ch.to_lowercase()), + // Absent from the folding table ⇒ the character folds to itself. + None => std::iter::once(ch).eq(ch.to_lowercase()), + }; + if !agrees { + return false; + } + } + + if saw_capital_sigma { + let lowered = text.to_lowercase(); + return fold_case_cow(text).as_ref() == lowered.as_str(); + } + true +} + #[cfg(test)] mod tests { use super::*; @@ -427,6 +484,150 @@ mod tests { } } + // ── Fold stability (#619) ──────────────────────────────────────── + + #[test] + fn ordinary_text_is_stable() { + for s in [ + "", + "gross.txt", + "admin@example.com", + "café résumé naïve", + "Москва", + "你好世界", + "🎉 party", + "Σ", // capital sigma alone lowercases to σ, which is its fold + "ΣΑΒΒΑΤΟ", // …and medially too + "ΑΒΓΔ", + // MEASURED, and the counter-intuitive one: U+0130 is the textbook + // case-mapping oddity, but both sides expand it the same way — + // fold and lowercase agree on `i` + U+0307, so it is a stable key. + "İstanbul", + ] { + assert!(is_case_fold_stable_impl(s), "{s:?} reported unstable"); + } + } + + #[test] + fn the_collision_classes_are_unstable() { + for s in [ + "groß.txt", // CVE-2026-23950: collides with gross.txt + "ſtraße", // long s and eszett, one string, two collisions + "file", // ligature: collides with file + "ẛ", // U+1E9B folds to ṡ, lowercases to itself + "\u{13A0}", // Cherokee folds small→capital, so both cases move + "\u{AB70}", + "µ", // micro sign folds to Greek mu + ] { + assert!(!is_case_fold_stable_impl(s), "{s:?} reported stable"); + } + } + + #[test] + fn final_sigma_is_the_reason_the_answer_is_not_per_character() { + // Both words end in Σ, whose *lowercase* is context-sensitive (ς at the + // end of a word, σ elsewhere) while its *fold* is not. A per-character + // table would call these stable and under-report every Greek word + // ending in sigma — ΟΔΟΣ is Greek for "street". + assert_eq!(fold_case_impl("ΟΔΟΣ"), "οδοσ"); + assert_eq!("ΟΔΟΣ".to_lowercase(), "οδος"); + assert!(!is_case_fold_stable_impl("ΟΔΟΣ")); + assert!(!is_case_fold_stable_impl("ΣΟΦΟΣ")); + } + + #[test] + fn the_predicate_is_exactly_the_comparison_it_claims_to_be() { + // Not a reimplementation of the rule: the spelled-out comparison and the + // fast-path version must agree, or the fast paths have drifted. + for s in [ + "", + "abc", + "ABC", + "groß", + "gross", + "ΟΔΟΣ", + "ΣΑΒΒΑΤΟ", + "Σ", + "file", + "café", + "Ꭰꭰ", + "İstanbul", + "ΑΣΣΟΣ", + "aΣ", + "Σa", + "ß Σ", + ] { + assert_eq!( + is_case_fold_stable_impl(s), + fold_case_impl(s) == s.to_lowercase(), + "fast path disagrees with the definition on {s:?}" + ); + } + } + + /// The premise of the ASCII bypass, checked against the definition rather + /// than against the bypass. Asserting that the function returns `true` for + /// ASCII would only re-read the early return; what has to hold is that + /// folding and lowercasing genuinely agree on every ASCII code point, which + /// is what makes skipping the scan safe. Cheap enough to run in Tier 1. + #[test] + fn ascii_folds_and_lowercases_identically() { + for cp in 0u32..0x80 { + let s = char::from_u32(cp).unwrap().to_string(); + assert_eq!( + fold_case_impl(&s), + s.to_lowercase(), + "ASCII U+{cp:04X} folds and lowercases differently" + ); + assert!(is_case_fold_stable_impl(&s)); + } + } + + /// Tier-3 gate for the step-3 guard: `U+03A3` is the *only* code point whose + /// lowercase mapping depends on context, so it is the only one that can make + /// the whole-string answer differ from the per-character one. + /// + /// Anchored to the property rather than to the character: if a future Unicode + /// version gives a second code point a context-sensitive lowercase, this + /// fails rather than the predicate quietly under-reporting it. + #[test] + #[ignore = "exhaustive: every code point in three positions; run in Tier 3 / pre-release"] + fn only_sigma_has_a_context_sensitive_lowercase() { + let mut context_sensitive = Vec::new(); + for cp in 0u32..=0x0010_FFFF { + let Some(ch) = char::from_u32(cp) else { + continue; // surrogates + }; + let alone = ch.to_string().to_lowercase(); + let per_char: String = ch.to_lowercase().collect(); + let medial = format!("a{ch}a").to_lowercase(); + let last = format!("a{ch}").to_lowercase(); + if alone != per_char || medial != format!("a{alone}a") || last != format!("a{alone}") { + context_sensitive.push(format!("U+{cp:04X}")); + } + } + assert_eq!(context_sensitive, ["U+03A3"]); + } + + /// Tier-3 gate: the predicate agrees with its own definition over every code + /// point, so the ASCII bypass and the table scan cannot drift from + /// `fold_case(x) == x.to_lowercase()`. + #[test] + #[ignore = "exhaustive: every code point through is_case_fold_stable; run in Tier 3 / pre-release"] + fn exhaustive_agrees_with_the_definition() { + for cp in 0u32..=0x0010_FFFF { + let Some(ch) = char::from_u32(cp) else { + continue; // surrogates + }; + let s = ch.to_string(); + assert_eq!( + is_case_fold_stable_impl(&s), + fold_case_impl(&s) == s.to_lowercase(), + "disagreement on U+{cp:04X}" + ); + } + } + // ── Property-based tests ───────────────────────────────────────── mod proptest_properties { @@ -471,6 +672,24 @@ mod tests { ); } + /// The fast paths never disagree with the rule they optimize (#619). + #[test] + fn is_case_fold_stable_matches_the_definition(s in "\\PC*") { + prop_assert_eq!( + is_case_fold_stable_impl(&s), + fold_case_impl(&s) == s.to_lowercase() + ); + } + + /// A stable string's fold is its lowercase, which is what makes the + /// answer worth asking for: the caller can key on either. + #[test] + fn stable_means_the_two_keys_agree(s in "\\PC*") { + if is_case_fold_stable_impl(&s) { + prop_assert_eq!(fold_case_impl(&s), s.to_lowercase()); + } + } + /// Pure ASCII input stays pure ASCII after folding. #[test] fn fold_case_ascii_stays_ascii(s in "[\\x00-\\x7f]*") { diff --git a/src/lib.rs b/src/lib.rs index ba1e280f..cc3efb13 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -244,6 +244,7 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(py::encoders::_percent_encode, m)?)?; m.add_function(wrap_pyfunction!(py::filename::_sanitize_filename, m)?)?; m.add_function(wrap_pyfunction!(py::case_fold::_fold_case, m)?)?; + m.add_function(wrap_pyfunction!(py::case_fold::_is_case_fold_stable, m)?)?; m.add_function(wrap_pyfunction!(py::whitespace::_collapse_whitespace, m)?)?; m.add_function(wrap_pyfunction!(py::whitespace::_strip_control_chars, m)?)?; m.add_function(wrap_pyfunction!( diff --git a/src/py/case_fold.rs b/src/py/case_fold.rs index c5e334f6..b5b1d4cc 100644 --- a/src/py/case_fold.rs +++ b/src/py/case_fold.rs @@ -8,3 +8,10 @@ use pyo3::prelude::*; pub fn _fold_case(text: &str) -> String { crate::case_fold::fold_case_impl(text) } + +/// `is_case_fold_stable(text) -> bool` +#[pyfunction] +#[pyo3(signature = (text,))] +pub fn _is_case_fold_stable(text: &str) -> bool { + crate::case_fold::is_case_fold_stable_impl(text) +} diff --git a/tests/api_idioms.rs b/tests/api_idioms.rs index 84466b07..db92ee0e 100644 --- a/tests/api_idioms.rs +++ b/tests/api_idioms.rs @@ -83,6 +83,8 @@ fn disarm_str_extension_trait() { ); assert_eq!("café".strip_accents(), "cafe"); assert_eq!("HELLO".fold_case(), "hello"); + assert!("gross.txt".is_case_fold_stable()); + assert!(!"groß.txt".is_case_fold_stable()); assert_eq!("Москва".transliterate(), "Moskva"); assert!("p\u{0430}ypal.com".is_suspicious_hostname().suspicious); assert!("hello".strip_obfuscation().is_ok()); diff --git a/tests/api_pure_rust.rs b/tests/api_pure_rust.rs index a9549022..cfabc3b0 100644 --- a/tests/api_pure_rust.rs +++ b/tests/api_pure_rust.rs @@ -43,6 +43,8 @@ fn text_cleanup() { assert!(!api::is_zalgo("hi", 3)); assert_eq!(api::strip_zalgo("a", 2), "a"); assert_eq!(api::fold_case("ß"), "ss"); + assert!(api::is_case_fold_stable("gross.txt")); + assert!(!api::is_case_fold_stable("groß.txt")); } #[test] diff --git a/tests/test_api_stability.py b/tests/test_api_stability.py index c206e1e4..b1317d3a 100644 --- a/tests/test_api_stability.py +++ b/tests/test_api_stability.py @@ -107,6 +107,7 @@ "unmapped_confusables", "find_unmapped_confusables", "is_ascii", + "is_case_fold_stable", "is_normalized", # Pipeline management "PRESETS", diff --git a/tests/test_case_folding.py b/tests/test_case_folding.py index 34764dc2..bf0d1ed1 100644 --- a/tests/test_case_folding.py +++ b/tests/test_case_folding.py @@ -6,7 +6,7 @@ from hypothesis import given, settings from hypothesis import strategies as st -from disarm import fold_case +from disarm import fold_case, is_case_fold_stable from disarm._text import Text # ── ASCII fast path ────────────────────────────────────────────────── @@ -304,6 +304,122 @@ def test_fold_case_chained(self) -> None: def test_fold_case_ligatures(self) -> None: assert Text("find").fold_case().value == "find" + def test_is_case_fold_stable_method(self) -> None: + assert Text("gross.txt").is_case_fold_stable() is True + assert Text("groß.txt").is_case_fold_stable() is False + + +# ── Fold stability (#619) ──────────────────────────────────────────── + + +class TestCaseFoldStability: + """``is_case_fold_stable(x)`` answers ``fold_case(x) == x.lower()``. + + A ``False`` says some *other* string folds to the same value, so keying a + table on this one can collide. It is a statement about the string, not about + intent — every case below is ordinary text in some language. + """ + + @pytest.mark.parametrize( + "text", + [ + "", + "gross.txt", + "admin@example.com", + "café résumé naïve", + "Москва", + "你好世界", + "ΣΑΒΒΑΤΟ", # capital sigma, but not word-final + "İstanbul", # fold and lower both give i + U+0307 + ], + ) + def test_ordinary_text_is_stable(self, text: str) -> None: + assert is_case_fold_stable(text) is True + + @pytest.mark.parametrize( + ("text", "why"), + [ + ("groß.txt", "CVE-2026-23950: collides with gross.txt"), + ("ſtraße", "long s and eszett"), + ("file", "ligature: collides with file"), + ("ẛ", "U+1E9B folds to ṡ and lowercases to itself"), + ("µm", "micro sign folds to Greek mu"), + ("Ꭰ", "Cherokee folds small→capital, so the capital moves too"), + ("ꭰ", "…and so does the small form"), + ("ΟΔΟΣ", "Greek final sigma: lowercases to οδος, folds to οδοσ"), + ], + ) + def test_the_collision_classes_are_unstable(self, text: str, why: str) -> None: + assert is_case_fold_stable(text) is False, why + + def test_the_answer_is_about_the_string_not_its_characters(self) -> None: + """Why a per-character lookup table would be wrong, measured. + + ``Σ`` agrees with itself in isolation — its fold and its lowercase are + both ``σ`` — so a per-character table calls every one of these stable. + The whole-string answer is the correct one, because ``str.lower()`` + applies the Final_Sigma context rule and case folding has none. + """ + assert fold_case("ΟΔΟΣ") == "οδοσ" + assert "ΟΔΟΣ".lower() == "οδος" + assert is_case_fold_stable("Σ") is True + assert is_case_fold_stable("ΟΔΟΣ") is False + + def test_str_casefold_is_the_wrong_comparison_basis(self) -> None: + """The trap the docstring names, pinned. + + ``str.casefold()`` performs the very transform under test, so a + predicate written against it answers ``stable`` for everything. disarm's + fold agrees with CPython's casefold on all of these, which is exactly + what makes the substitution silent. + """ + for text in ["groß.txt", "ſtraße", "file", "gross.txt"]: + assert fold_case(text) == text.casefold(), text + assert is_case_fold_stable(text) == (fold_case(text) == text.lower()) + + def test_rejects_non_str(self) -> None: + with pytest.raises(TypeError, match="expects str"): + is_case_fold_stable(42) # type: ignore[arg-type] + + def test_a_lone_surrogate_answers_for_its_scrubbed_form(self) -> None: + """#469's boundary contract, applied here. + + A lone surrogate has no UTF-8 encoding, so it never reaches Rust: the + boundary replaces it with U+FFFD and the predicate answers for that. + ``fold_case(x) == x.lower()`` computed in Python answers for the + surrogate instead and disagrees, which is a decode artifact rather than + a fold property. + """ + assert is_case_fold_stable("a\ud800b") is True + assert fold_case("a\ud800b") == "a�b" + + def test_disagreements_with_the_python_one_liner_are_data_version_gaps(self) -> None: + """MEASURED, and the caveat the docstring states. + + disarm answers with its own CaseFolding table against the lowercase + mapping compiled into the crate. ``fold_case(x) == x.lower()`` written in + Python substitutes *CPython's* Unicode version for the second half, and + the two Unicode versions are rarely the same — this file's + ``TestAgainstPythonCasefold`` already navigates the same skew from the + other side. + + The invariant that survives a version bump in either direction: wherever + the two disagree, one of them sees no case mapping for that code point at + all. There is no code point where both know a mapping and they still + disagree — that would be a real defect rather than a data gap. + """ + contested = [] + for cp in range(0x80, 0x110000): + if 0xD800 <= cp <= 0xDFFF: + continue # surrogates cannot cross the boundary + ch = chr(cp) + if is_case_fold_stable(ch) == (fold_case(ch) == ch.lower()): + continue + if ch.lower() == ch or fold_case(ch) == ch: + continue # one side has no mapping: a data-version gap + contested.append(f"U+{cp:04X}") + assert not contested, contested + # ── Casefold correctness against Python's str.casefold() ───────────── diff --git a/tests/test_cve_vectors.py b/tests/test_cve_vectors.py index 70b8ced0..c2e1d88e 100644 --- a/tests/test_cve_vectors.py +++ b/tests/test_cve_vectors.py @@ -64,6 +64,7 @@ has_anomalies, has_bidi_conflict, inspect_anomalies, + is_case_fold_stable, is_confusable, is_mixed_script, is_suspicious_hostname, @@ -325,17 +326,10 @@ def test_canonical_forms_agree(self, defense) -> None: left for the reset token to be delivered to.""" assert defense(ATTACKER_EMAIL) == defense(VICTIM_EMAIL) - def test_upper_collision_class_is_closed(self) -> None: - """Exhaustive: the whole Unicode space, not a sample. - - The CVE's collision class is exactly *"non-ASCII code points whose - ``.upper()`` is pure ASCII"*. There are ten of them. ``fold_case`` - composed with ``canonicalize_strict`` maps every one to the same ASCII - its uppercase form implies, so the class is closed with no residue. - - Runs in ~0.2s, which buys an exhaustive gate for the price of a - sampled one. - """ + @staticmethod + def _upper_collision_class() -> list[tuple[str, str]]: + """The CVE's collision class: non-ASCII code points whose ``.upper()`` is + pure ASCII. Exhaustive over the whole Unicode space, ~0.2s.""" ascii_upper = set("ABCDEFGHIJKLMNOPQRSTUVWXYZ") collisions = [] for cp in range(0x80, 0x110000): @@ -343,6 +337,19 @@ def test_upper_collision_class_is_closed(self) -> None: up = ch.upper() if up and up != ch and all(c in ascii_upper for c in up): collisions.append((ch, up)) + return collisions + + def test_upper_collision_class_is_closed(self) -> None: + """Exhaustive: the whole Unicode space, not a sample. + + There are ten members. ``fold_case`` composed with + ``canonicalize_strict`` maps every one to the same ASCII its uppercase + form implies, so the class is closed with no residue. + + Runs in ~0.2s, which buys an exhaustive gate for the price of a + sampled one. + """ + collisions = self._upper_collision_class() # Pinned: a Unicode data bump that changes this count should be seen. assert len(collisions) == 10, [f"U+{ord(c):04X}" for c, _ in collisions] @@ -369,6 +376,24 @@ def test_confusable_folding_alone_leaves_sharp_s(self) -> None: assert fold_case("ß") == "ss" assert search_key("ß@example.com") == search_key("ss@example.com") + def test_fold_stability_does_not_report_this_row(self) -> None: + """MEASURED LIMIT, and a correction to how #619 was framed. + + The issue pairs this CVE with CVE-2026-23950 as two rows turning on one + precondition. Nine of the ten sources in the collision class above are + fold-unstable, so for those the pairing holds — but this row's probe uses + the tenth, ``U+0131`` DOTLESS I, which folds to itself *and* lowercases + to itself. It collides through ``.upper()`` instead, which is a different + question, so ``is_case_fold_stable`` is silent here and the row's + detector stays ``is_confusable`` alone. + """ + assert is_case_fold_stable(ATTACKER_EMAIL) is True + assert fold_case("ı") == "ı" == "ı".lower() + assert "ı".upper() == "I" + + unstable = [ch for ch, _ in self._upper_collision_class() if not is_case_fold_stable(ch)] + assert len(unstable) == 9, [f"U+{ord(c):04X}" for c in unstable] + # --------------------------------------------------------------------------- # CVE-2014-9390 — git .git path equivalence @@ -1383,6 +1408,53 @@ def test_the_canonicalizers_deliberately_do_not(self) -> None: assert canonicalize_strict("groß.txt") == "groß.txt" assert strip_obfuscation("groß.txt") == "groß.txt" + def test_the_precondition_is_reportable_even_though_the_collision_is_not(self) -> None: + """#619: this row's *Detected by* column, and the shape of what it claims. + + The collision is a property of a **pair** of names, and every disarm + detector is a single-string predicate, so nothing here can say + ``groß.txt`` collides with ``gross.txt`` — that is #620's job. What is a + single-string property, and what node-tar's ``PathReservations`` guard + needed, is the *precondition*: this name does not fold to its own + lowercase, so some other name may fold to the same key. + """ + assert is_case_fold_stable("groß.txt") is False + assert is_case_fold_stable("gross.txt") is True + + def test_the_predicate_is_a_fact_and_not_an_accusation(self) -> None: + """Why it stays out of ``DETECTOR_PANEL`` and out of ``has_anomalies``. + + ``groß.txt`` is an ordinary German filename. Folding the predicate into + the anomaly report would flag ordinary German, ordinary Greek and every + Latin ligature as suspicious, which would be both noisy and the wrong + claim. The panel is silent on this row, and that stays measured rather + than assumed. + """ + assert not has_anomalies("groß.txt") + assert not any(pred("groß.txt") for pred in DETECTOR_PANEL.values()) + + def test_the_class_is_wider_than_the_eszett(self) -> None: + """The reason it is worth a function rather than a one-liner per caller. + + ``ß`` is the member everyone knows. The class it belongs to also holds + the long s, every Latin ligature, the micro sign and the whole of + Cherokee (whose fold runs small→capital, so both cases move), and a + caller has to know all of that before the one-liner is worth writing. + """ + for name in ["groß", "ſtraße", "file", "µm", "Ꭰ", "ꭰ"]: + assert not is_case_fold_stable(name), name + + def test_str_casefold_is_the_trap_the_comparison_avoids(self) -> None: + """MEASURED: ``str.casefold()`` is the wrong basis, and silently so. + + Casefolding performs the very transform under test, so a predicate + written against it answers ``stable`` for every string in Unicode. #617 + walked into this exact substitution in the comparator harness. + """ + for name in ["groß.txt", "ſtraße", "file", "gross.txt"]: + assert fold_case(name) == name.casefold(), name + assert is_case_fold_stable("groß.txt") != (fold_case("groß.txt") == "groß.txt".casefold()) + # --------------------------------------------------------------------------- # Normalization cost — CVE-2026-3276, CVE-2023-46695, CVE-2017-20190 @@ -2174,9 +2246,12 @@ def test_multibyte_escape_bypass_is_not_an_injection_defense(self) -> None: cwe="CWE-176", cvss=5.9, cvss_version="v3.1", - dispositions=frozenset({NEUTRALIZED}), + dispositions=frozenset({NEUTRALIZED, DETECTED}), neutralizers=("fold_case", "search_key", "catalog_key"), - detectors=(), + # Outside DETECTOR_PANEL on purpose (#619): the panel members report an + # anomaly, and this one reports a property of an ordinary German + # filename. Asserted in TestTarPathCollision instead. + detectors=("is_case_fold_stable",), probe="groß.txt", reference="https://nvd.nist.gov/vuln/detail/CVE-2026-23950", ), @@ -2631,10 +2706,10 @@ class TestDetectionHasNoSuperset: Neutralization has a safe default. Detection does not — and not because one predicate is weaker than another: **no combination of them** covers the - matrix. Five vectors are silent to every detector disarm exposes. + matrix. Seven vectors are silent to every detector disarm exposes. So a pipeline that screens first and only cleans what it flagged forwards - those five untouched. Clean unconditionally; use the detectors to decide + those seven untouched. Clean unconditionally; use the detectors to decide whether to *alert*, never whether to *clean*. """ @@ -2644,7 +2719,6 @@ class TestDetectionHasNoSuperset: #: than as a silent improvement nobody notices. UNDETECTED_IN_SCOPE = { "CVE-2025-32711", # the Tags block is not an anomaly kind - "CVE-2026-23950", # nor is a case-folding path collision "CVE-2023-46695", # nor is a long run of already-normalized characters # The whole encoding class is silent too — see TestOverlongAndInvalid- # Sequences and TestLoneSurrogates. Every one is neutralized and none @@ -2658,6 +2732,16 @@ class TestDetectionHasNoSuperset: "CVE-2024-46954", "CVE-2026-44288", } + #: Closed by ``is_case_fold_stable`` (#619), by a route this class had ruled + #: out. The standing claim is that each remaining row needs a *comparison* + #: between two strings, and for the collision itself that is still true — + #: nothing here can say ``groß.txt`` collides with ``gross.txt``. What was + #: wrong is treating the comparison as the only reportable part: the + #: *precondition* is a single-string property, and it is the part a + #: reservation table needs. Whether the length-budget and decode-result rows + #: have a reportable precondition too is open. + CLOSED_BY_FOLD_STABILITY = {"CVE-2026-23950"} + #: Closed by the ``control`` anomaly kind (#612). Kept as a record, because the #: shape of what remains is the interesting part: every row still undetected #: needs a *comparison* — a fold collision, a length budget, a decode result — @@ -2752,6 +2836,22 @@ def test_every_terminal_control_row_is_now_reported(self) -> None: for cve_id in sorted(terminal): assert self._fires(BY_ID[cve_id]), cve_id + def test_the_fold_collision_row_is_now_reported(self) -> None: + """Inverted by #619, the same way and for a different reason. + + The panel is still silent on it and should be — ``is_case_fold_stable`` + is not an anomaly claim — so this row is reported by a named + single-string detector rather than by the panel, exactly as + CVE-2023-4399 is reported by the hostname screen. + """ + assert self.CLOSED_BY_FOLD_STABILITY == {"CVE-2026-23950"} + for cve_id in sorted(self.CLOSED_BY_FOLD_STABILITY): + cve = BY_ID[cve_id] + assert cve_id not in self.UNDETECTED_IN_SCOPE + assert not self._fires(cve), f"{cve_id} should stay out of the panel" + assert cve.detectors == ("is_case_fold_stable",) + assert not is_case_fold_stable(cve.probe) + def test_nfkc_unmasking_is_silent_too(self) -> None: """CVE-2019-9636 is out of scope *and* undetected, which is the worst pair. @@ -2764,8 +2864,9 @@ def test_nfkc_unmasking_is_silent_too(self) -> None: def test_stripping_covers_what_detection_misses(self) -> None: """The payoff: every vector no detector sees is still neutralized.""" measurable = self.UNDETECTED_IN_SCOPE & set(NEUTRALIZABLE) - # Not every undetected row has a collapse/removal vector — CVE-2026-23950 - # is neutralized by a key builder, which the _handles rule does not model. + # Not every undetected row has a collapse/removal vector — the encoding + # rows are neutralized at decode time, which the _handles rule does not + # model, so the intersection is narrower than the set. assert measurable, "no undetected row is measurable any more" for cve in sorted(measurable): assert _handles(canonicalize, cve), cve @@ -3034,15 +3135,22 @@ class TestDocsMatrixDrift: DOC = ROOT / "docs" / "security" / "cve-validation.md" THREAT_MODEL = ROOT / "THREAT_MODEL.md" - #: "| [CVE-x](url) | title | 9.8 (v3.1) | Neutralized | `f`, `g` |" + #: "| [CVE-x](url) | title | 9.8 (v3.1) | Neutralized | `f`, `g` | `h` |" ROW = re.compile( r"^\|\s*\[(?PCVE-\d{4}-\d+)\][^|]*\|" r"[^|]*\|" r"\s*(?:(?P[\d.]+)\s*\((?Pv[\d.]+)\)|none \(SSVC only\))\s*\|" - r"\s*(?P[^|]+?)\s*\|", + r"\s*(?P[^|]+?)\s*\|" + r"\s*(?P[^|]*?)\s*\|" + r"\s*(?P[^|]*?)\s*\|", flags=re.MULTILINE, ) + @staticmethod + def _cell(text: str) -> tuple[str, ...]: + """The entry points named in one cell. An em dash means none.""" + return tuple(re.findall(r"`([^`]+)`", text)) + def _rows(self) -> dict[str, re.Match[str]]: text = self.DOC.read_text(encoding="utf-8") return {m.group("id"): m for m in self.ROW.finditer(text)} @@ -3112,3 +3220,24 @@ def test_documented_disposition_matches_the_registry(self) -> None: if m.group("disposition").strip("* ") != BY_ID[cve_id].rendered ] assert not mismatches, mismatches + + def test_documented_entry_points_match_the_registry(self) -> None: + """The two right-hand columns are the answer to "what do I call?". + + The disposition check above already stops a row being softened, but it + says nothing about the function names beside it — a typo, a stale name + after a rename, or a detector added to the registry and not to the page + all passed. That is the same gap `test_documented_cvss_matches_the_registry` + exists to close one column to the left, so it gets the same treatment. + """ + mismatches = [] + for cve_id, match in self._rows().items(): + cve = BY_ID[cve_id] + for column, claimed in ( + ("neutralizers", cve.neutralizers), + ("detectors", cve.detectors), + ): + documented = self._cell(match.group(column)) + if documented != claimed: + mismatches.append((cve_id, column, documented, claimed)) + assert not mismatches, mismatches