diff --git a/CHANGELOG.md b/CHANGELOG.md index ea8414ff..cd5f00fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -493,6 +493,32 @@ compatibility (see [RELEASING.md](RELEASING.md)). ### Fixed +- **Python can now call `strip_control_chars` and `strip_zero_width_chars` directly + (#616).** They already existed in the Rust core (`disarm::api`) and in the C ABI, + Java/Kotlin, Node and Ruby bindings. Python was the only surface without them, so + control-stripping there meant constructing a `TextPipeline` rather than calling a + function — unlike the ten sibling `strip_*` operations, four of which + (`strip_tags`, `strip_pua`, `strip_noncharacters`, `strip_variation_selectors`) are + narrower and are plain functions. Both are now exported, and `Text` gains the + matching fluent methods. + + The parity matrix recorded the gap as deliberate and named the substitute as + `collapse_whitespace(strip_control=True)` — a signature that has never existed; + `collapse_whitespace` takes only `text`. That record lived in `PROVIDED_VIA` in + `scripts/parity.py`, so anyone consulting the matrix for the Python equivalent was + sent to a `TypeError`. Both entries are removed and the matrix regenerated. + +- **`collapse_whitespace` gains a property test covering control characters.** The + existing `no_leading_trailing_whitespace` property draws from `\PC*`, which + excludes controls, so the trim invariant was never tested against them. It holds: + measured exhaustively over the cross product of whitespace, controls and letters + for lengths 1–4, and over 200,000 random strings, with zero cases where the output + starts or ends with whitespace. Reported as a trim bug in #612; that report was + wrong and is retracted there. What looked like a defeated trim is the space + *between* a leading control and the word, which is interior by the same rule that + makes `"a\u{0}b"` keep both of its spaces. No behaviour change — the test closes + the coverage gap that made the question open. + - **`is_suspicious_hostname()` now catches zero-width and invisible characters, and no longer reports a phantom script for them (#605).** Sibling of #603, for the characters that carry no direction at all — `U+200B`–`U+200D`, `U+2060`–`U+2064`, `U+FEFF` and diff --git a/generated/parity.yaml b/generated/parity.yaml index 2c493f23..402d1f31 100644 --- a/generated/parity.yaml +++ b/generated/parity.yaml @@ -370,7 +370,7 @@ operations: - id: strip_control_chars names: rust: strip_control_chars - python: { provided_via: "collapse_whitespace(strip_control=True) / get_pipeline()" } + python: strip_control_chars ruby: strip_control_chars node: stripControlChars - id: strip_format @@ -424,7 +424,7 @@ operations: - id: strip_zero_width_chars names: rust: strip_zero_width_chars - python: { provided_via: "collapse_whitespace(strip_zero_width=True) / get_pipeline()" } + python: strip_zero_width_chars ruby: strip_zero_width_chars node: stripZeroWidthChars - id: terminal_width diff --git a/python/disarm/__init__.py b/python/disarm/__init__.py index 2ab7a2f3..364ee94a 100644 --- a/python/disarm/__init__.py +++ b/python/disarm/__init__.py @@ -61,7 +61,9 @@ set_emoji_provider, slugify, strip_accents, + strip_control_chars, strip_log_injection, + strip_zero_width_chars, terminal_width, transliterate, unmapped_confusables, @@ -249,6 +251,7 @@ "normalize_confusables", "sanitize_filename", "strip_accents", + "strip_control_chars", "fold_case", "collapse_whitespace", "demojize", @@ -292,6 +295,7 @@ "escape_html", "percent_encode", "strip_log_injection", + "strip_zero_width_chars", "HostnameAnalysis", # Reverse transliteration "reverse_langs", diff --git a/python/disarm/_api.py b/python/disarm/_api.py index bd435248..81551e86 100644 --- a/python/disarm/_api.py +++ b/python/disarm/_api.py @@ -80,7 +80,9 @@ _slugify_batch, _strip_accents, _strip_accents_batch, + _strip_control_chars, _strip_log_injection, + _strip_zero_width_chars, # #476: the surrogate-boundary guard, for the class-based entrypoints (the module # loop in _boundary wraps only free functions, not class methods). _surrogate_safe, @@ -912,10 +914,10 @@ def collapse_whitespace(text: str) -> str: Folds **whitespace only** (#433): the line controls (TAB/LF/VT/FF/CR), the information separators (U+001C–U+001F), NEL, the ``Zs``/``Zl``/``Zp`` spaces, and the blank-rendering set (Braille blank, the Hangul fillers) each fold to a - single space. It does **not** delete control or zero-width characters — to do - that, run a :class:`TextPipeline` with the ``strip_control`` / - ``strip_zero_width`` steps (the ``canonicalize`` / ``canonicalize_strict`` - presets already do). + single space. It does **not** delete control or zero-width characters — for + that, call :func:`strip_control_chars` / :func:`strip_zero_width_chars`, or + use a preset that sequences them ahead of the fold (``canonicalize`` and + ``canonicalize_strict`` both do). Folding the line controls (rather than deleting them) means a carriage return between two tokens becomes a space, never a silent join: ``"a\\rb"`` → @@ -940,6 +942,63 @@ def collapse_whitespace(text: str) -> str: return _collapse_whitespace(text) +def strip_control_chars(text: str) -> str: + """Remove control characters that are **not** whitespace (#433). + + Deletes every C0/C1 control (NUL, BEL, ESC, DEL, the C1 block) *except* the + ones :func:`collapse_whitespace` folds — TAB, LF, VT, FF, CR, the information + separators ``U+001C``–``U+001F``, and NEL. Those are preserved here so the + fold can turn them into a space; deleting them would join the tokens either + side, which is the invisible-join hazard the split exists to avoid. + + Pair it with :func:`collapse_whitespace` when you want both, in that order. + + Args: + text: Input string. + + Returns: + String with non-whitespace controls removed. + + Examples: + >>> strip_control_chars("a\\x00b\\x07c") + 'abc' + >>> strip_control_chars("a\\rb") # CR preserved for the fold to handle + 'a\\rb' + """ + if not isinstance(text, str): + raise TypeError(f"strip_control_chars() expects str, got {type(text).__name__}") + return _strip_control_chars(text) + + +def strip_zero_width_chars(text: str) -> str: + """Remove zero-width characters. + + Deletes the zero-width set, which renders as nothing and is used to fragment a + token so it evades a denylist while looking unchanged. The set is exactly: + + - ``U+200B``–``U+200D`` — ZWSP, ZWNJ, ZWJ + - ``U+2060``–``U+2064`` — word joiner and the invisible operators + - ``U+FEFF`` — BOM / zero-width no-break space + - ``U+180E`` — Mongolian vowel separator (reclassified ``Zs`` → ``Cf`` in + Unicode 6.3, so it is a format character despite the name) + + Args: + text: Input string. + + Returns: + String with zero-width characters removed. + + Examples: + >>> strip_zero_width_chars("pay\\u200bpal") + 'paypal' + >>> strip_zero_width_chars("a\\ufeffb") + 'ab' + """ + if not isinstance(text, str): + raise TypeError(f"strip_zero_width_chars() expects str, got {type(text).__name__}") + return _strip_zero_width_chars(text) + + def demojize( text: str, *, diff --git a/python/disarm/_boundary.pyi b/python/disarm/_boundary.pyi index 899ad1b6..9bbfb680 100644 --- a/python/disarm/_boundary.pyi +++ b/python/disarm/_boundary.pyi @@ -206,6 +206,9 @@ from disarm._core import ( from disarm._core import ( _strip_bidi as _strip_bidi, ) +from disarm._core import ( + _strip_control_chars as _strip_control_chars, +) from disarm._core import ( _strip_format as _strip_format, ) @@ -230,6 +233,9 @@ from disarm._core import ( from disarm._core import ( _strip_zalgo as _strip_zalgo, ) +from disarm._core import ( + _strip_zero_width_chars as _strip_zero_width_chars, +) from disarm._core import ( _terminal_width as _terminal_width, ) diff --git a/python/disarm/_core.pyi b/python/disarm/_core.pyi index 0296d457..f604aa98 100644 --- a/python/disarm/_core.pyi +++ b/python/disarm/_core.pyi @@ -209,6 +209,8 @@ def _sanitize_filename( def _strip_accents(text: str) -> str: ... def _fold_case(text: str) -> str: ... def _collapse_whitespace(text: str) -> str: ... +def _strip_control_chars(text: str) -> str: ... +def _strip_zero_width_chars(text: str) -> str: ... def _demojize( text: str, *, diff --git a/python/disarm/_text.py b/python/disarm/_text.py index b482841b..69a99bfe 100644 --- a/python/disarm/_text.py +++ b/python/disarm/_text.py @@ -180,6 +180,18 @@ def collapse_whitespace(self) -> Text: than being deleted, so ``"a\\rb"`` becomes ``"a b"``.""" return Text(self._t().collapse_whitespace(self._value)) + def strip_control_chars(self) -> Text: + """Remove control characters that are not whitespace (#433). + + The controls :meth:`collapse_whitespace` folds — TAB, LF, VT, FF, CR, the + information separators and NEL — are preserved so the fold can turn them + into a space; deleting them would join the tokens either side.""" + return Text(self._t().strip_control_chars(self._value)) + + def strip_zero_width_chars(self) -> Text: + """Remove zero-width characters (ZWSP, ZWNJ, ZWJ, word joiner, BOM, …).""" + return Text(self._t().strip_zero_width_chars(self._value)) + def slugify( self, *, diff --git a/python/disarm/_text.pyi b/python/disarm/_text.pyi index fb07520a..c8c91ce1 100644 --- a/python/disarm/_text.pyi +++ b/python/disarm/_text.pyi @@ -50,6 +50,12 @@ class Text: def collapse_whitespace(self) -> Text: """Fold whitespace runs to single ASCII spaces (fold-only, #433).""" ... + def strip_control_chars(self) -> Text: + """Remove control characters that are not whitespace (#433).""" + ... + def strip_zero_width_chars(self) -> Text: + """Remove zero-width characters (ZWSP, ZWNJ, ZWJ, word joiner, BOM).""" + ... def slugify( self, *, diff --git a/scripts/parity.py b/scripts/parity.py index 738c3d3f..a0359366 100644 --- a/scripts/parity.py +++ b/scripts/parity.py @@ -8,8 +8,11 @@ (v0 ignored re-exports -> has_anomalies/inspect_anomalies false nulls). * Ruby surface = real `def` lines in bindings/ruby/lib/disarm.rb. * Schema gains `alias_of` and `provided_via` so folded/aliased ops are not - mislabelled as gaps (reverse_transliterate via transliterate(target=...), - strip_control_chars/strip_zero_width_chars via the pipeline, etc.). + mislabelled as gaps (reverse_transliterate via transliterate(target=...)). + Use `provided_via` only for a route that is real and callable: the entries + for strip_control_chars/strip_zero_width_chars named + `collapse_whitespace(strip_control=True)`, a signature that never existed, + which hid the gap until #616 exported both from Python. Caveat: Python+Rust are verified against the real public surface; Ruby is parsed from source defs (reliable) and Node from `export function` (reliable), but neither is runtime-introspected (no toolchain) — finalize with @@ -100,10 +103,6 @@ def canon(name, lang): # bindings expose a nullary accessor because a native module cannot export a static. "confusables_version": {"python": "disarm.CONFUSABLES_VERSION"}, "reverse_transliterate": {"python": "transliterate(target=…)"}, - "strip_control_chars": {"python": "collapse_whitespace(strip_control=True) / get_pipeline()"}, - "strip_zero_width_chars": { - "python": "collapse_whitespace(strip_zero_width=True) / get_pipeline()" - }, } # Deliberate scope decisions for Ruby/Node — not blind backfill: # * registration mutates process-global state; encoders are sink-context tools; diff --git a/src/lib.rs b/src/lib.rs index 1762e037..ba1e280f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -245,6 +245,11 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(py::filename::_sanitize_filename, m)?)?; m.add_function(wrap_pyfunction!(py::case_fold::_fold_case, m)?)?; m.add_function(wrap_pyfunction!(py::whitespace::_collapse_whitespace, m)?)?; + m.add_function(wrap_pyfunction!(py::whitespace::_strip_control_chars, m)?)?; + m.add_function(wrap_pyfunction!( + py::whitespace::_strip_zero_width_chars, + m + )?)?; m.add_function(wrap_pyfunction!(py::scripts::_detect_scripts, m)?)?; m.add_function(wrap_pyfunction!(py::scripts::_is_mixed_script, m)?)?; m.add_function(wrap_pyfunction!(py::scripts::_has_bidi_conflict, m)?)?; diff --git a/src/py/whitespace.rs b/src/py/whitespace.rs index 7bd7469b..f0d5e11d 100644 --- a/src/py/whitespace.rs +++ b/src/py/whitespace.rs @@ -8,3 +8,17 @@ use pyo3::prelude::*; pub fn _collapse_whitespace(text: &str) -> String { crate::whitespace::collapse_whitespace(text) } + +/// `strip_control_chars(text) -> str` (#616). +#[pyfunction] +#[pyo3(signature = (text,))] +pub fn _strip_control_chars(text: &str) -> String { + crate::api::strip_control_chars(text) +} + +/// `strip_zero_width_chars(text) -> str` (#616). +#[pyfunction] +#[pyo3(signature = (text,))] +pub fn _strip_zero_width_chars(text: &str) -> String { + crate::api::strip_zero_width_chars(text) +} diff --git a/src/whitespace.rs b/src/whitespace.rs index 98ca5ba5..e5c77c24 100644 --- a/src/whitespace.rs +++ b/src/whitespace.rs @@ -169,6 +169,14 @@ pub(crate) fn is_zero_width(ch: char) -> bool { mod tests { use super::*; + /// Controls are still never DELETED — that is `strip_control_chars`' job, and + /// deleting here would join the tokens either side. + #[test] + fn controls_are_preserved_not_deleted() { + assert_eq!(collapse_whitespace("a\u{0}b"), "a\u{0}b"); + assert_eq!(collapse_whitespace("a \u{0} b"), "a \u{0} b"); + } + #[test] fn test_collapse_whitespace() { assert_eq!(collapse_whitespace("hello world"), "hello world"); @@ -356,6 +364,26 @@ mod tests { let twice = collapse_whitespace(&once); prop_assert_eq!(&once, &twice); } + + /// The trim invariant holds when CONTROLS are in the mix too. + /// + /// `no_leading_trailing_whitespace` above draws from `\PC*`, which + /// excludes controls, so it never covered this. A control is content + /// for run-collapsing — `"a \u{0} b"` keeps both spaces — and the + /// question that leaves open is whether one at an edge can strand + /// whitespace outside it. It cannot: the leading run is dropped before + /// the control is emitted, and the trailing truncate runs after. + /// Measured over the full cross product of these classes (#612). + #[test] + fn no_edge_whitespace_even_with_controls( + s in r"[ab\u{00e9}\x20\x09\x0a\u{00a0}\u{3000}\x00\x07\x1b\x7f\u{0080}\u{009f}]{0,16}" + ) { + let result = collapse_whitespace(&s); + if !result.is_empty() { + prop_assert_ne!(result.chars().next().unwrap(), ' '); + prop_assert_ne!(result.chars().next_back().unwrap(), ' '); + } + } } } } diff --git a/tests/test_api_stability.py b/tests/test_api_stability.py index 9a5d902b..c206e1e4 100644 --- a/tests/test_api_stability.py +++ b/tests/test_api_stability.py @@ -46,6 +46,8 @@ "normalize_confusables", "sanitize_filename", "strip_accents", + "strip_control_chars", + "strip_zero_width_chars", "fold_case", "collapse_whitespace", "demojize", @@ -350,6 +352,8 @@ def _param_kinds(fn) -> dict[str, str]: "strip_accents": ["text"], "fold_case": ["text"], "collapse_whitespace": ["text"], + "strip_control_chars": ["text"], + "strip_zero_width_chars": ["text"], "demojize": [ "text", "strip_modifiers", @@ -487,6 +491,8 @@ def test_first_param_is_positional(self, name: str): ], "fold_case": [], "collapse_whitespace": [], + "strip_control_chars": [], + "strip_zero_width_chars": [], "slugify": [ "separator", "lowercase", diff --git a/tests/test_form_invariance_audit.py b/tests/test_form_invariance_audit.py index 09fe0abc..dffd7953 100644 --- a/tests/test_form_invariance_audit.py +++ b/tests/test_form_invariance_audit.py @@ -49,6 +49,12 @@ "strip_variation_selectors", "strip_noncharacters", "collapse_whitespace", + # #616: same category as the strips above — each deletes one character class + # and touches nothing else, so `ї` stays decomposed if it arrived decomposed. + # Neither is a recovery entrypoint; compose them with a preset when you want + # boundary normalization as well. + "strip_control_chars", + "strip_zero_width_chars", "escape_html", "strip_log_injection", "demojize", diff --git a/tests/test_presets.py b/tests/test_presets.py index 6bee8e00..d660ad44 100644 --- a/tests/test_presets.py +++ b/tests/test_presets.py @@ -6,15 +6,19 @@ from disarm import ( DisarmError, + Text, canonicalize, canonicalize_strict, catalog_key, + collapse_whitespace, ml_normalize, search_key, sort_key, strip_bidi, + strip_control_chars, strip_format, strip_obfuscation, + strip_zero_width_chars, ) # ===== canonicalize ===== @@ -718,3 +722,68 @@ def check(text): assert canonicalize(once) == once, f"not idempotent on {text!r}: {once!r}" check() + + +class TestStripControlAndZeroWidthBehaviour: + """#616: the two newly-exported entrypoints, checked by behaviour. + + They were covered only by the API-surface audits, which assert a name and a + signature and nothing about what the function does. These pin the character + sets and, more importantly, the *split* between them — the reason there are two + functions rather than one. + """ + + #: Removed by strip_control_chars: every control that is not a separator. + CONTROLS_REMOVED = ["\x00", "\x07", "\x08", "\x1b", "\x7f", "\x80", "\x9f"] + #: Preserved by it: the whitespace-class controls collapse_whitespace folds. + CONTROLS_KEPT = ["\t", "\n", "\x0b", "\x0c", "\r", "\x1c", "\x1f", "\x85"] + #: The whole zero-width set, U+180E included. + ZERO_WIDTH = [ + "\u200b", + "\u200c", + "\u200d", + "\u2060", + "\u2061", + "\u2062", + "\u2063", + "\u2064", + "\ufeff", + "\u180e", + ] + + @pytest.mark.parametrize("ch", CONTROLS_REMOVED) + def test_non_whitespace_controls_are_removed(self, ch): + assert strip_control_chars(f"a{ch}b") == "ab" + + @pytest.mark.parametrize("ch", CONTROLS_KEPT) + def test_whitespace_controls_are_preserved(self, ch): + """Deleting these would join the tokens either side — the #433 contract.""" + assert strip_control_chars(f"a{ch}b") == f"a{ch}b" + + @pytest.mark.parametrize("ch", ZERO_WIDTH) + def test_zero_width_characters_are_removed(self, ch): + assert strip_zero_width_chars(f"pay{ch}pal") == "paypal" + + def test_the_two_functions_do_not_overlap(self): + """Each leaves the other's set alone, which is why both exist.""" + for ch in self.ZERO_WIDTH: + assert strip_control_chars(f"a{ch}b") == f"a{ch}b", ch + for ch in self.CONTROLS_REMOVED: + assert strip_zero_width_chars(f"a{ch}b") == f"a{ch}b", ch + + def test_ordinary_text_is_untouched(self): + for s in ("hello world", "Café déjà vu", "Привет мир", "a\tb\nc"): + assert strip_control_chars(s) == s + assert strip_zero_width_chars(s) == s + + def test_composition_matches_canonicalize(self): + """The documented recipe: strip controls, then fold whitespace.""" + messy = " \x00 hello \x7f " + assert collapse_whitespace(strip_control_chars(messy)) == "hello" + assert canonicalize(messy) == "hello" + + def test_text_methods_mirror_the_functions(self): + assert str(Text("a\x00b").strip_control_chars()) == "ab" + assert str(Text("pay\u200bpal").strip_zero_width_chars()) == "paypal" + # Fluent chaining is the point of the Text wrapper. + assert str(Text(" \x00 hi \x7f ").strip_control_chars().collapse_whitespace()) == "hi"