fix: close the two CVE rows behind "no single call" (#614, #615)

CHANGELOG.md

@@ -493,6 +493,59 @@ compatibility (see [RELEASING.md](RELEASING.md)).
 
 ### Fixed
 
+- **The two CVE rows behind "no single call" are closed, and they had to close together
+  (#614, #615).** Each was one of the exactly two vectors that made
+  `docs/security/cve-validation.md` say no entry point cleared everything, so fixing one
+  alone would have left the other failing and forced the guidance to be rewritten twice.
+
+  **#614 — `strip_obfuscation` named confusables instead of folding them.** 49 code
+  points appear in both `emoji_single.tsv` and `confusables_to_latin.tsv`, and most are
+  not emoji: typographic punctuation, currency, math operators, CJK brackets. They reach
+  the emoji table from CLDR `annotationsDerived`, which names non-emoji characters.
+  `strip_obfuscation("€xample.com")` produced `"euro xample.com"`, so the spoof and the
+  genuine host stopped being equal rather than becoming equal — CVE-2017-5383 surviving a
+  preset documented as maximum-strength deobfuscation.
+
+  **Not fixed the way the issue proposed.** Reordering the confusable fold before
+  `demojize` would break idempotency: punctuation inside emoji *names* (the `’` in
+  "woman’s hat") has to be folded by the confusable pass. That ordering is documented
+  three times and pinned by `tests/test_presets.py`. Instead the overlap is derived at
+  build time as an intersection of the two tables — so it cannot drift the way a curated
+  list would — and `demojize` skips those rows inside comparison presets only. Standalone
+  `demojize("I ❤ €5")` still returns `"I red heart euro 5"`, which is what that function
+  is for. `build.rs` asserts the count is 49, so a table refresh that claims another
+  confusable source fails the build instead of widening the gap silently.
+
+  **#615 — `canonicalize` cannot cap its way out of an eclipsing mark.** The anti-zalgo
+  step is a *count*, and by count one Arabic shadda is indistinguishable from one acute
+  accent, so no threshold removes CVE-2017-7833's spoof and keeps `café`. The
+  discriminator that works was already in disarm's script data: strip a combining mark
+  whose own Script is a *specific* script differing from its base's, and keep `Inherited`
+  marks, which attach to anything. That is UTS #39's mixed-script reasoning applied per
+  grapheme rather than per string.
+
+  It runs in `canonicalize_strict` **only**. The rule is destructive for scholarly
+  transliteration, IPA and linguistic transcription, where marks from one script
+  legitimately sit on bases of another — the corpus least able to notice. `canonicalize`
+  is deliberately still one short, and there is a test asserting that rather than leaving
+  it implied. Verified against nine legitimate samples in five scripts, including Arabic
+  *with* its own vowel marks: all pass through completely unchanged.
+
+  The step sits **after** the confusable fold, and that ordering is load-bearing. Placed
+  before it, `а` (Cyrillic) + `U+0489` (Cyrillic mark) agrees on the first pass, then the
+  fold rewrites the base to Latin `a` and the next pass strips the mark — `f(f(x)) !=
+  f(x)`. The property test `canonicalize_strict_idempotent` caught it; deciding against
+  the *final* base script is the only stable point.
+
+  `canonicalize_strict` and `strip_obfuscation` now each clear the whole matrix, so
+  `TestOneCall`'s guard is inverted rather than deleted: it asserted that closing a gap
+  should fail loudly, it did, and it now asserts the two sufficient entry points stay
+  sufficient while every other one stays short. The published advice is unchanged and its
+  reason has moved — from "nothing suffices" to "the two that suffice are the two most
+  destructive ones", which is the same conclusion for a caller who has to forward the
+  text they cleaned.
+
+
 - **`is_suspicious_hostname()` now catches tags, variation selectors, noncharacters
   and PUA, and stops reporting a noncharacter as Arabic (#610).** Third in the
   sequence after #603 (bidi controls) and #605 (zero-width). 17 of 18 sampled

build.rs

@@ -9,7 +9,7 @@
 //!   - str→str maps:  `key\tvalue`
 //!   - char sets:      `HEXCODEPOINT`
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::env;
 use std::fmt::Write as _;
 use std::fs;
@@ -288,6 +288,56 @@ fn main() {
         "EMOJI_SINGLE",
         "pub",
     );
+    // --- Emoji rows the TR39 confusable table also claims (#614) ---
+    // 49 code points appear in BOTH emoji_single.tsv and confusables_to_latin.tsv, and
+    // they are mostly not emoji at all: typographic punctuation (`2010 hyphen`, the
+    // apostrophes and quotes, `2026 ellipsis`), currency (`20AC euro`), math operators
+    // and the CJK tortoise-shell brackets. They reach the emoji table from CLDR
+    // `annotationsDerived`, which names non-emoji characters.
+    //
+    // Both entries are legitimate. `demojize("I ❤ €5")` -> "I red heart euro 5" is
+    // exactly what that function is for. What is wrong is which table wins inside a
+    // COMPARISON preset: `strip_obfuscation("€xample.com")` named the euro sign instead
+    // of folding it, so the spoof and the genuine string stopped being equal rather than
+    // becoming equal, and CVE-2017-5383 survived a preset documented as maximum-strength
+    // deobfuscation.
+    //
+    // Derived as an INTERSECTION rather than read from a curated file, so it cannot drift
+    // out of date the way a hand-written override list would. The count is asserted: a
+    // future emoji-table refresh that claims another confusable source fails the build
+    // with this message instead of silently widening the gap.
+    {
+        let emoji = read_char_str_tsv(&data_dir.join("emoji_single.tsv"));
+        let confusable = read_char_str_tsv(&data_dir.join("confusables_to_latin.tsv"));
+        let overlap: BTreeSet<u32> = emoji
+            .keys()
+            .filter(|cp| confusable.contains_key(cp))
+            .copied()
+            .collect();
+        assert_eq!(
+            overlap.len(),
+            49,
+            "emoji_single.tsv ∩ confusables_to_latin.tsv changed: expected the 49 rows \
+             reviewed in #614, found {}. A new row means a confusable source is now \
+             named instead of folded inside strip_obfuscation. Review it, then update \
+             this count.",
+            overlap.len()
+        );
+        let mut code = String::from(
+            "/// Code points claimed by BOTH the emoji name table and the TR39 confusable\n\
+             /// table (#614). Skipped by `demojize` inside comparison presets so the fold\n\
+             /// wins; standalone `demojize` still names them.\n\
+             pub(crate) static EMOJI_ROWS_TR39_ALSO_CLAIMS: phf::Set<u32> = ",
+        );
+        let mut set = phf_codegen::Set::new();
+        for cp in &overlap {
+            set.entry(*cp);
+        }
+        code.push_str(&set.build().to_string());
+        code.push_str(";\n");
+        fs::write(out_dir.join("emoji_tr39_overlap_phf.rs"), code).unwrap();
+    }
+
     // Production matcher (#242 item 4): compact code-point trie.
     generate_emoji_trie(
         &data_dir.join("emoji_multi.tsv"),

docs/security/cve-validation.md

@@ -106,7 +106,7 @@ so the claim passed its own gate. CVE-2017-7833 is the vector that breaks it —
 a single Arabic vowel mark riding a Latin letter:
 
 ```python
-from disarm import canonicalize, strip_obfuscation, strip_zalgo, is_zalgo, catalog_key
+from disarm import canonicalize, canonicalize_strict, strip_obfuscation, is_zalgo
 
 eclipsed = "exaّmple.com"   # U+0651 ARABIC SHADDA over the "a"
 
@@ -116,43 +116,41 @@ assert is_zalgo(eclipsed) is False
 assert canonicalize(eclipsed) != canonicalize("example.com")
 ```
 
-The mirror image is just as real. `strip_obfuscation` removes the mark but
-*names* punctuation confusables instead of folding them, so U+2010 HYPHEN comes
-out as the word "hyphen" and never collapses onto ASCII `-`:
+The mirror image was just as real. `strip_obfuscation` removed the mark but
+*named* punctuation confusables instead of folding them, so U+2010 HYPHEN came
+out as the word "hyphen" and never collapsed onto ASCII `-`. Neither preset
+dominated the other, and they failed on different inputs.
 
-```python
-assert strip_obfuscation(eclipsed) == strip_obfuscation("example.com")   # mark: yes
-assert strip_obfuscation("ex‐ample.com") == "ex hyphen ample.com"        # hyphen: no
-```
-
-Neither preset dominates the other, and they fail on different inputs. Reading
-that as "5/6 each, pick either" would be the wrong lesson.
-
-### There is no single call
-
-Measured across the whole matrix plus both vectors above, **no entry point
-clears everything.** `catalog_key` comes closest — it is the only one carrying
-both a confusable step and `strip_accents`, so it handles the mark *and* the
-hyphen — and it has no format-stripping step, so the Unicode Tags block of
-CVE-2025-32711 passes straight through it.
+### Two calls now suffice, and the pairing is not a coincidence
 
-The answer is a composition:
+Both gaps are closed (#614, #615), and they had to close together: each was one
+of the exactly two vectors behind the claim that nothing sufficed, so fixing one
+alone would have left the other still failing.
 
 ```python
-def canonical(text):
-    return canonicalize(strip_zalgo(text, max_marks=0))
+assert strip_obfuscation(eclipsed) == strip_obfuscation("example.com")
+assert strip_obfuscation("ex‐ample.com") == strip_obfuscation("ex-ample.com")
 
-assert canonical(eclipsed) == canonical("example.com")
-assert canonical("ex‐ample.com") == canonical("ex-ample.com")
-assert canonical(".g‌it/config") == ".git/config"
+assert canonicalize_strict(eclipsed) == canonicalize_strict("example.com")
+assert canonicalize_strict("ex‐ample.com") == canonicalize_strict("ex-ample.com")
 ```
 
-`strip_zalgo(max_marks=0)` supplies the step `canonicalize` lacks;
-`canonicalize` supplies the confusable folding and format stripping that
-`strip_zalgo` and `catalog_key` lack. That pair clears every vector on this
-page, and `TestOneCall` gates both halves of the claim — including a test that
-fails if any single entry point ever *does* become sufficient, so the guidance
-gets revisited rather than silently left stale.
+Measured across the whole matrix, `canonicalize_strict` and `strip_obfuscation`
+each clear every row. What they share is the thing that matters: a confusable
+fold **and** something that removes a combining mark. `catalog_key` misses only
+the Unicode Tags block; `canonicalize` misses only the eclipsing mark.
+
+**`canonicalize` is deliberately still short by one.** The rule that closes
+CVE-2017-7833 keys on the mark's own Script, and that is destructive for
+scholarly transliteration, IPA and linguistic transcription, where marks from one
+script legitimately sit on bases of another. So it lives in `canonicalize_strict`,
+where a caller has already accepted a stricter contract, and not in the preset
+meant for text that gets forwarded.
+
+That also means the advice below has not changed, only its reason. It used to
+rest on "nothing suffices". It now rests on "the two that suffice are the two
+most destructive ones", which is the same conclusion for a caller who has to
+forward the text they cleaned.
 
 These 25 vectors are a spot check, not a measurement of the confusable space.
 Treat this as "no vector here needs a third call", not as coverage.

proptest-regressions/presets.txt

@@ -11,3 +11,4 @@ cc da36583310da1e7223530863bfb32f55a617bf2ef42b31a6f08d71d60f8ae38f # shrinks to
 cc 88f3c26133409f858f19ddb38641040bc1c50665eb05233ddeab4c0ff8440776 # shrinks to s = "£"
 cc ddb34e6d319f0e4347521cb55a6c0a8582b5cfe49206664033e878768e51c551 # shrinks to s = "𐏈"
 cc e311270910ff3cb8ed5b4e5c5945101b01a02f59ab600fccedfb473bf498d899 # shrinks to s = "\u{80}"
+cc 40dcc66bceeb4cda9984a876398f234fd85cc67e51771690a7306c0517a52526 # shrinks to s = "а\u{489}"

src/emoji.rs

@@ -319,13 +319,30 @@ pub(crate) fn pad_emoji_replacement(result: &mut String, text: &str) {
 /// materialising the full input for non-ASCII text.
 pub fn demojize_rust(text: &str, strip_modifiers: bool) -> String {
     let mut out = String::new();
-    demojize_rust_into(text, strip_modifiers, &mut out);
+    demojize_rust_into(text, strip_modifiers, false, &mut out);
     out
 }
 
 /// In-place form of [`demojize_rust`] writing into `result` (cleared first), so
 /// the pipeline can reuse one buffer across steps (#236 item 7).
-pub fn demojize_rust_into(text: &str, strip_modifiers: bool, result: &mut String) {
+/// `skip_tr39_claimed` (#614): leave the 49 code points the TR39 confusable table
+/// also claims for the confusable step to fold, instead of naming them here.
+///
+/// Set only by comparison presets. `strip_obfuscation("\u{20AC}xample.com")` named the
+/// euro sign — "euro xample.com" — so the spoof and the genuine string stopped being
+/// equal rather than becoming equal, and CVE-2017-5383 survived a preset documented as
+/// maximum-strength deobfuscation. Standalone `demojize` still names them, because
+/// `demojize("I \u{2764} \u{20AC}5")` -> "I red heart euro 5" is what that function is for.
+///
+/// Skipping here rather than reordering the steps is deliberate: `normalize_confusables`
+/// runs *after* `demojize` so typographic punctuation inside emoji names (the `\u{2019}`
+/// in "woman\u{2019}s hat") is folded too, and swapping them would break idempotency.
+pub fn demojize_rust_into(
+    text: &str,
+    strip_modifiers: bool,
+    skip_tr39_claimed: bool,
+    result: &mut String,
+) {
     result.clear();
     // Fast path: pure-ASCII text cannot contain emoji.
     if text.is_ascii() {
@@ -343,6 +360,27 @@ pub fn demojize_rust_into(text: &str, strip_modifiers: bool, result: &mut String
             continue;
         }
 
+        // #614: hand this code point to the confusable fold instead of naming it.
+        // Emitted verbatim, so the later `confusables` step sees it.
+        if skip_tr39_claimed && crate::tables::is_tr39_claimed_emoji_row(ch) {
+            // The separator decision has to look at what this character will BECOME,
+            // not what it is. `\u{20AC}` is not alphanumeric, but TR39 folds it to `e`,
+            // so emitting it bare after an emoji name produced `"woman's hat"` + `"e"`
+            // -> `"woman's hate"` once the fold ran: a word that was in neither the
+            // input nor any name. `\u{2211}` -> `s` and `\u{2200}` -> `a` do the same.
+            // Punctuation targets (`\u{2010}` -> `-`) still take no separator, matching
+            // how every other non-alphanumeric is emitted here.
+            let becomes_alphanumeric = crate::tables::lookup_confusable(ch, "latin")
+                .is_some_and(|t| t.starts_with(char::is_alphanumeric));
+            if last_was_emoji && (ch.is_alphanumeric() || becomes_alphanumeric) {
+                result.push(' ');
+            }
+            result.push(ch);
+            last_was_emoji = false;
+            win.advance(1);
+            continue;
+        }
+
         if let Some((name, consumed)) = match_emoji_at(win.as_slice()) {
             let replacement = strip_modifier_suffix(name, strip_modifiers);
             pad_emoji_replacement(result, replacement);

src/pipeline.rs

@@ -262,7 +262,7 @@ impl Pipeline {
             crate::presets::strip_bidi_into(input, out);
             Ok(true)
         } else if step == PipelineSteps::DEMOJIZE {
-            emoji::demojize_rust_into(input, false, out);
+            emoji::demojize_rust_into(input, false, false, out);
             Ok(true)
         } else if step == PipelineSteps::STRIP_ACCENTS {
             transliterate::strip_accents_into(input, out);