From f186fd17e3804fa6e652df361fdef0648c92d3eb Mon Sep 17 00:00:00 2001 From: Richard Quinn Date: Wed, 26 Aug 2026 19:15:01 +0200 Subject: [PATCH 1/2] feat: export strip_control_chars and strip_zero_width_chars from Python (#616) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They already existed in the Rust core (disarm::api) and in the C ABI, Java/Kotlin, Node and Ruby bindings. Python was the only surface without them, so control-stripping there meant constructing a TextPipeline instead of calling a function — unlike the ten sibling strip_* operations, four of which (strip_tags, strip_pua, strip_noncharacters, strip_variation_selectors) are narrower and are plain functions. Both are now exported, with matching Text methods. The parity matrix recorded the gap as deliberate and named the substitute as collapse_whitespace(strip_control=True) — a signature that has never existed; collapse_whitespace takes only `text`. That record lived in PROVIDED_VIA in scripts/parity.py, so anyone consulting the matrix for the Python equivalent was sent to a TypeError. Both entries are removed and generated/parity.yaml regenerated; it now names a real function on all six surfaces. Two gates did their job and are worth recording. test_api_stability caught the missing __all__ entries. test_form_invariance_audit auto-discovered both new entrypoints and failed them, because a new str->str name is in scope for boundary normalization unless someone deliberately classifies it — these are targeted strips in the same category as strip_bidi/strip_tags/strip_pua, so they join the reviewed FORM_PRESERVING allowlist rather than being silently exempted. Also closes a coverage gap in collapse_whitespace's property tests. The existing no_leading_trailing_whitespace property draws from \PC*, which excludes controls, so the trim invariant was never tested against them. Added no_edge_whitespace_even_with_controls, which draws from a whitespace-plus-controls alphabet. The invariant holds. #612 reported a trim bug here; that report was wrong and is retracted on the issue. Measured exhaustively over the cross product of whitespace, controls and letters for lengths 1-4, plus 200,000 random strings: zero cases where the output starts or ends with whitespace. What looked like a defeated trim is the space BETWEEN a leading control and the word, which is interior by exactly the rule that makes "a \0 b" keep both of its spaces. No behaviour change here — only the test that makes the answer checkable. Signed-off-by: Richard Quinn Assisted-by: Claude Code:claude-opus-5 --- CHANGELOG.md | 26 ++++++++++++ generated/parity.yaml | 4 +- python/disarm/__init__.py | 4 ++ python/disarm/_api.py | 62 +++++++++++++++++++++++++++-- python/disarm/_boundary.pyi | 6 +++ python/disarm/_core.pyi | 2 + python/disarm/_text.py | 12 ++++++ python/disarm/_text.pyi | 6 +++ scripts/parity.py | 11 +++-- src/lib.rs | 5 +++ src/py/whitespace.rs | 14 +++++++ src/whitespace.rs | 28 +++++++++++++ tests/test_api_stability.py | 6 +++ tests/test_form_invariance_audit.py | 6 +++ 14 files changed, 180 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 693b7b06..2289d156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -460,6 +460,32 @@ compatibility (see [RELEASING.md](RELEASING.md)). ### Fixed +- **Python can now call `strip_control_chars` and `strip_zero_width_chars` directly + (#616).** They already existed in the Rust core (`disarm::api`) and in the C ABI, + Java/Kotlin, Node and Ruby bindings. Python was the only surface without them, so + control-stripping there meant constructing a `TextPipeline` rather than calling a + function — unlike the ten sibling `strip_*` operations, four of which + (`strip_tags`, `strip_pua`, `strip_noncharacters`, `strip_variation_selectors`) are + narrower and are plain functions. Both are now exported, and `Text` gains the + matching fluent methods. + + The parity matrix recorded the gap as deliberate and named the substitute as + `collapse_whitespace(strip_control=True)` — a signature that has never existed; + `collapse_whitespace` takes only `text`. That record lived in `PROVIDED_VIA` in + `scripts/parity.py`, so anyone consulting the matrix for the Python equivalent was + sent to a `TypeError`. Both entries are removed and the matrix regenerated. + +- **`collapse_whitespace` gains a property test covering control characters.** The + existing `no_leading_trailing_whitespace` property draws from `\PC*`, which + excludes controls, so the trim invariant was never tested against them. It holds: + measured exhaustively over the cross product of whitespace, controls and letters + for lengths 1–4, and over 200,000 random strings, with zero cases where the output + starts or ends with whitespace. Reported as a trim bug in #612; that report was + wrong and is retracted there. What looked like a defeated trim is the space + *between* a leading control and the word, which is interior by the same rule that + makes `"a\u{0}b"` keep both of its spaces. No behaviour change — the test closes + the coverage gap that made the question open. + - **`is_suspicious_hostname()` now catches zero-width and invisible characters, and no longer reports a phantom script for them (#605).** Sibling of #603, for the characters that carry no direction at all — `U+200B`–`U+200D`, `U+2060`–`U+2064`, `U+FEFF` and diff --git a/generated/parity.yaml b/generated/parity.yaml index 2c493f23..402d1f31 100644 --- a/generated/parity.yaml +++ b/generated/parity.yaml @@ -370,7 +370,7 @@ operations: - id: strip_control_chars names: rust: strip_control_chars - python: { provided_via: "collapse_whitespace(strip_control=True) / get_pipeline()" } + python: strip_control_chars ruby: strip_control_chars node: stripControlChars - id: strip_format @@ -424,7 +424,7 @@ operations: - id: strip_zero_width_chars names: rust: strip_zero_width_chars - python: { provided_via: "collapse_whitespace(strip_zero_width=True) / get_pipeline()" } + python: strip_zero_width_chars ruby: strip_zero_width_chars node: stripZeroWidthChars - id: terminal_width diff --git a/python/disarm/__init__.py b/python/disarm/__init__.py index 2ab7a2f3..364ee94a 100644 --- a/python/disarm/__init__.py +++ b/python/disarm/__init__.py @@ -61,7 +61,9 @@ set_emoji_provider, slugify, strip_accents, + strip_control_chars, strip_log_injection, + strip_zero_width_chars, terminal_width, transliterate, unmapped_confusables, @@ -249,6 +251,7 @@ "normalize_confusables", "sanitize_filename", "strip_accents", + "strip_control_chars", "fold_case", "collapse_whitespace", "demojize", @@ -292,6 +295,7 @@ "escape_html", "percent_encode", "strip_log_injection", + "strip_zero_width_chars", "HostnameAnalysis", # Reverse transliteration "reverse_langs", diff --git a/python/disarm/_api.py b/python/disarm/_api.py index bd435248..8bb310fa 100644 --- a/python/disarm/_api.py +++ b/python/disarm/_api.py @@ -80,7 +80,9 @@ _slugify_batch, _strip_accents, _strip_accents_batch, + _strip_control_chars, _strip_log_injection, + _strip_zero_width_chars, # #476: the surrogate-boundary guard, for the class-based entrypoints (the module # loop in _boundary wraps only free functions, not class methods). _surrogate_safe, @@ -912,10 +914,10 @@ def collapse_whitespace(text: str) -> str: Folds **whitespace only** (#433): the line controls (TAB/LF/VT/FF/CR), the information separators (U+001C–U+001F), NEL, the ``Zs``/``Zl``/``Zp`` spaces, and the blank-rendering set (Braille blank, the Hangul fillers) each fold to a - single space. It does **not** delete control or zero-width characters — to do - that, run a :class:`TextPipeline` with the ``strip_control`` / - ``strip_zero_width`` steps (the ``canonicalize`` / ``canonicalize_strict`` - presets already do). + single space. It does **not** delete control or zero-width characters — for + that, call :func:`strip_control_chars` / :func:`strip_zero_width_chars`, or + use a preset that sequences them ahead of the fold (``canonicalize`` and + ``canonicalize_strict`` both do). Folding the line controls (rather than deleting them) means a carriage return between two tokens becomes a space, never a silent join: ``"a\\rb"`` → @@ -940,6 +942,58 @@ def collapse_whitespace(text: str) -> str: return _collapse_whitespace(text) +def strip_control_chars(text: str) -> str: + """Remove control characters that are **not** whitespace (#433). + + Deletes every C0/C1 control (NUL, BEL, ESC, DEL, the C1 block) *except* the + ones :func:`collapse_whitespace` folds — TAB, LF, VT, FF, CR, the information + separators ``U+001C``–``U+001F``, and NEL. Those are preserved here so the + fold can turn them into a space; deleting them would join the tokens either + side, which is the invisible-join hazard the split exists to avoid. + + Pair it with :func:`collapse_whitespace` when you want both, in that order. + + Args: + text: Input string. + + Returns: + String with non-whitespace controls removed. + + Examples: + >>> strip_control_chars("a\\x00b\\x07c") + 'abc' + >>> strip_control_chars("a\\rb") # CR preserved for the fold to handle + 'a\\rb' + """ + if not isinstance(text, str): + raise TypeError(f"strip_control_chars() expects str, got {type(text).__name__}") + return _strip_control_chars(text) + + +def strip_zero_width_chars(text: str) -> str: + """Remove zero-width characters. + + Deletes the zero-width set — ZWSP, ZWNJ, ZWJ, the word joiner, the invisible + operators and the BOM — which render as nothing and are used to fragment a + token so it evades a denylist while looking unchanged. + + Args: + text: Input string. + + Returns: + String with zero-width characters removed. + + Examples: + >>> strip_zero_width_chars("pay\\u200bpal") + 'paypal' + >>> strip_zero_width_chars("a\\ufeffb") + 'ab' + """ + if not isinstance(text, str): + raise TypeError(f"strip_zero_width_chars() expects str, got {type(text).__name__}") + return _strip_zero_width_chars(text) + + def demojize( text: str, *, diff --git a/python/disarm/_boundary.pyi b/python/disarm/_boundary.pyi index 899ad1b6..9bbfb680 100644 --- a/python/disarm/_boundary.pyi +++ b/python/disarm/_boundary.pyi @@ -206,6 +206,9 @@ from disarm._core import ( from disarm._core import ( _strip_bidi as _strip_bidi, ) +from disarm._core import ( + _strip_control_chars as _strip_control_chars, +) from disarm._core import ( _strip_format as _strip_format, ) @@ -230,6 +233,9 @@ from disarm._core import ( from disarm._core import ( _strip_zalgo as _strip_zalgo, ) +from disarm._core import ( + _strip_zero_width_chars as _strip_zero_width_chars, +) from disarm._core import ( _terminal_width as _terminal_width, ) diff --git a/python/disarm/_core.pyi b/python/disarm/_core.pyi index 0296d457..f604aa98 100644 --- a/python/disarm/_core.pyi +++ b/python/disarm/_core.pyi @@ -209,6 +209,8 @@ def _sanitize_filename( def _strip_accents(text: str) -> str: ... def _fold_case(text: str) -> str: ... def _collapse_whitespace(text: str) -> str: ... +def _strip_control_chars(text: str) -> str: ... +def _strip_zero_width_chars(text: str) -> str: ... def _demojize( text: str, *, diff --git a/python/disarm/_text.py b/python/disarm/_text.py index b482841b..69a99bfe 100644 --- a/python/disarm/_text.py +++ b/python/disarm/_text.py @@ -180,6 +180,18 @@ def collapse_whitespace(self) -> Text: than being deleted, so ``"a\\rb"`` becomes ``"a b"``.""" return Text(self._t().collapse_whitespace(self._value)) + def strip_control_chars(self) -> Text: + """Remove control characters that are not whitespace (#433). + + The controls :meth:`collapse_whitespace` folds — TAB, LF, VT, FF, CR, the + information separators and NEL — are preserved so the fold can turn them + into a space; deleting them would join the tokens either side.""" + return Text(self._t().strip_control_chars(self._value)) + + def strip_zero_width_chars(self) -> Text: + """Remove zero-width characters (ZWSP, ZWNJ, ZWJ, word joiner, BOM, …).""" + return Text(self._t().strip_zero_width_chars(self._value)) + def slugify( self, *, diff --git a/python/disarm/_text.pyi b/python/disarm/_text.pyi index fb07520a..c8c91ce1 100644 --- a/python/disarm/_text.pyi +++ b/python/disarm/_text.pyi @@ -50,6 +50,12 @@ class Text: def collapse_whitespace(self) -> Text: """Fold whitespace runs to single ASCII spaces (fold-only, #433).""" ... + def strip_control_chars(self) -> Text: + """Remove control characters that are not whitespace (#433).""" + ... + def strip_zero_width_chars(self) -> Text: + """Remove zero-width characters (ZWSP, ZWNJ, ZWJ, word joiner, BOM).""" + ... def slugify( self, *, diff --git a/scripts/parity.py b/scripts/parity.py index 738c3d3f..a0359366 100644 --- a/scripts/parity.py +++ b/scripts/parity.py @@ -8,8 +8,11 @@ (v0 ignored re-exports -> has_anomalies/inspect_anomalies false nulls). * Ruby surface = real `def` lines in bindings/ruby/lib/disarm.rb. * Schema gains `alias_of` and `provided_via` so folded/aliased ops are not - mislabelled as gaps (reverse_transliterate via transliterate(target=...), - strip_control_chars/strip_zero_width_chars via the pipeline, etc.). + mislabelled as gaps (reverse_transliterate via transliterate(target=...)). + Use `provided_via` only for a route that is real and callable: the entries + for strip_control_chars/strip_zero_width_chars named + `collapse_whitespace(strip_control=True)`, a signature that never existed, + which hid the gap until #616 exported both from Python. Caveat: Python+Rust are verified against the real public surface; Ruby is parsed from source defs (reliable) and Node from `export function` (reliable), but neither is runtime-introspected (no toolchain) — finalize with @@ -100,10 +103,6 @@ def canon(name, lang): # bindings expose a nullary accessor because a native module cannot export a static. "confusables_version": {"python": "disarm.CONFUSABLES_VERSION"}, "reverse_transliterate": {"python": "transliterate(target=…)"}, - "strip_control_chars": {"python": "collapse_whitespace(strip_control=True) / get_pipeline()"}, - "strip_zero_width_chars": { - "python": "collapse_whitespace(strip_zero_width=True) / get_pipeline()" - }, } # Deliberate scope decisions for Ruby/Node — not blind backfill: # * registration mutates process-global state; encoders are sink-context tools; diff --git a/src/lib.rs b/src/lib.rs index 1762e037..ba1e280f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -245,6 +245,11 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(py::filename::_sanitize_filename, m)?)?; m.add_function(wrap_pyfunction!(py::case_fold::_fold_case, m)?)?; m.add_function(wrap_pyfunction!(py::whitespace::_collapse_whitespace, m)?)?; + m.add_function(wrap_pyfunction!(py::whitespace::_strip_control_chars, m)?)?; + m.add_function(wrap_pyfunction!( + py::whitespace::_strip_zero_width_chars, + m + )?)?; m.add_function(wrap_pyfunction!(py::scripts::_detect_scripts, m)?)?; m.add_function(wrap_pyfunction!(py::scripts::_is_mixed_script, m)?)?; m.add_function(wrap_pyfunction!(py::scripts::_has_bidi_conflict, m)?)?; diff --git a/src/py/whitespace.rs b/src/py/whitespace.rs index 7bd7469b..f0d5e11d 100644 --- a/src/py/whitespace.rs +++ b/src/py/whitespace.rs @@ -8,3 +8,17 @@ use pyo3::prelude::*; pub fn _collapse_whitespace(text: &str) -> String { crate::whitespace::collapse_whitespace(text) } + +/// `strip_control_chars(text) -> str` (#616). +#[pyfunction] +#[pyo3(signature = (text,))] +pub fn _strip_control_chars(text: &str) -> String { + crate::api::strip_control_chars(text) +} + +/// `strip_zero_width_chars(text) -> str` (#616). +#[pyfunction] +#[pyo3(signature = (text,))] +pub fn _strip_zero_width_chars(text: &str) -> String { + crate::api::strip_zero_width_chars(text) +} diff --git a/src/whitespace.rs b/src/whitespace.rs index 98ca5ba5..e5c77c24 100644 --- a/src/whitespace.rs +++ b/src/whitespace.rs @@ -169,6 +169,14 @@ pub(crate) fn is_zero_width(ch: char) -> bool { mod tests { use super::*; + /// Controls are still never DELETED — that is `strip_control_chars`' job, and + /// deleting here would join the tokens either side. + #[test] + fn controls_are_preserved_not_deleted() { + assert_eq!(collapse_whitespace("a\u{0}b"), "a\u{0}b"); + assert_eq!(collapse_whitespace("a \u{0} b"), "a \u{0} b"); + } + #[test] fn test_collapse_whitespace() { assert_eq!(collapse_whitespace("hello world"), "hello world"); @@ -356,6 +364,26 @@ mod tests { let twice = collapse_whitespace(&once); prop_assert_eq!(&once, &twice); } + + /// The trim invariant holds when CONTROLS are in the mix too. + /// + /// `no_leading_trailing_whitespace` above draws from `\PC*`, which + /// excludes controls, so it never covered this. A control is content + /// for run-collapsing — `"a \u{0} b"` keeps both spaces — and the + /// question that leaves open is whether one at an edge can strand + /// whitespace outside it. It cannot: the leading run is dropped before + /// the control is emitted, and the trailing truncate runs after. + /// Measured over the full cross product of these classes (#612). + #[test] + fn no_edge_whitespace_even_with_controls( + s in r"[ab\u{00e9}\x20\x09\x0a\u{00a0}\u{3000}\x00\x07\x1b\x7f\u{0080}\u{009f}]{0,16}" + ) { + let result = collapse_whitespace(&s); + if !result.is_empty() { + prop_assert_ne!(result.chars().next().unwrap(), ' '); + prop_assert_ne!(result.chars().next_back().unwrap(), ' '); + } + } } } } diff --git a/tests/test_api_stability.py b/tests/test_api_stability.py index 9a5d902b..c206e1e4 100644 --- a/tests/test_api_stability.py +++ b/tests/test_api_stability.py @@ -46,6 +46,8 @@ "normalize_confusables", "sanitize_filename", "strip_accents", + "strip_control_chars", + "strip_zero_width_chars", "fold_case", "collapse_whitespace", "demojize", @@ -350,6 +352,8 @@ def _param_kinds(fn) -> dict[str, str]: "strip_accents": ["text"], "fold_case": ["text"], "collapse_whitespace": ["text"], + "strip_control_chars": ["text"], + "strip_zero_width_chars": ["text"], "demojize": [ "text", "strip_modifiers", @@ -487,6 +491,8 @@ def test_first_param_is_positional(self, name: str): ], "fold_case": [], "collapse_whitespace": [], + "strip_control_chars": [], + "strip_zero_width_chars": [], "slugify": [ "separator", "lowercase", diff --git a/tests/test_form_invariance_audit.py b/tests/test_form_invariance_audit.py index 09fe0abc..dffd7953 100644 --- a/tests/test_form_invariance_audit.py +++ b/tests/test_form_invariance_audit.py @@ -49,6 +49,12 @@ "strip_variation_selectors", "strip_noncharacters", "collapse_whitespace", + # #616: same category as the strips above — each deletes one character class + # and touches nothing else, so `ї` stays decomposed if it arrived decomposed. + # Neither is a recovery entrypoint; compose them with a preset when you want + # boundary normalization as well. + "strip_control_chars", + "strip_zero_width_chars", "escape_html", "strip_log_injection", "demojize", From 1cf747d637983140b1cd317e6072b2e9e335a07a Mon Sep 17 00:00:00 2001 From: Richard Quinn Date: Wed, 26 Aug 2026 20:52:57 +0200 Subject: [PATCH 2/2] test: pin the behaviour of the two new strip entrypoints, and name U+180E (#616) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both from review on #623. The docstring listed the zero-width set as prose and omitted U+180E, which whitespace::is_zero_width does remove — the Mongolian vowel separator, reclassified Zs -> Cf in Unicode 6.3, so a format character despite the name. The set is now enumerated exactly. The two entrypoints were covered only by the API-surface audits, which assert a name and a signature and nothing about what the function does. 29 cases now pin the character sets, ordinary text passing through untouched, the documented composition matching canonicalize, and the Text methods. The one worth keeping is test_the_two_functions_do_not_overlap: each leaves the other's set alone, which is the reason there are two functions rather than one and the thing a regression in the binding glue would break first. Signed-off-by: Richard Quinn Assisted-by: Claude Code:claude-opus-5 --- python/disarm/_api.py | 11 +++++-- tests/test_presets.py | 69 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/python/disarm/_api.py b/python/disarm/_api.py index 8bb310fa..81551e86 100644 --- a/python/disarm/_api.py +++ b/python/disarm/_api.py @@ -973,9 +973,14 @@ def strip_control_chars(text: str) -> str: def strip_zero_width_chars(text: str) -> str: """Remove zero-width characters. - Deletes the zero-width set — ZWSP, ZWNJ, ZWJ, the word joiner, the invisible - operators and the BOM — which render as nothing and are used to fragment a - token so it evades a denylist while looking unchanged. + Deletes the zero-width set, which renders as nothing and is used to fragment a + token so it evades a denylist while looking unchanged. The set is exactly: + + - ``U+200B``–``U+200D`` — ZWSP, ZWNJ, ZWJ + - ``U+2060``–``U+2064`` — word joiner and the invisible operators + - ``U+FEFF`` — BOM / zero-width no-break space + - ``U+180E`` — Mongolian vowel separator (reclassified ``Zs`` → ``Cf`` in + Unicode 6.3, so it is a format character despite the name) Args: text: Input string. diff --git a/tests/test_presets.py b/tests/test_presets.py index 6bee8e00..d660ad44 100644 --- a/tests/test_presets.py +++ b/tests/test_presets.py @@ -6,15 +6,19 @@ from disarm import ( DisarmError, + Text, canonicalize, canonicalize_strict, catalog_key, + collapse_whitespace, ml_normalize, search_key, sort_key, strip_bidi, + strip_control_chars, strip_format, strip_obfuscation, + strip_zero_width_chars, ) # ===== canonicalize ===== @@ -718,3 +722,68 @@ def check(text): assert canonicalize(once) == once, f"not idempotent on {text!r}: {once!r}" check() + + +class TestStripControlAndZeroWidthBehaviour: + """#616: the two newly-exported entrypoints, checked by behaviour. + + They were covered only by the API-surface audits, which assert a name and a + signature and nothing about what the function does. These pin the character + sets and, more importantly, the *split* between them — the reason there are two + functions rather than one. + """ + + #: Removed by strip_control_chars: every control that is not a separator. + CONTROLS_REMOVED = ["\x00", "\x07", "\x08", "\x1b", "\x7f", "\x80", "\x9f"] + #: Preserved by it: the whitespace-class controls collapse_whitespace folds. + CONTROLS_KEPT = ["\t", "\n", "\x0b", "\x0c", "\r", "\x1c", "\x1f", "\x85"] + #: The whole zero-width set, U+180E included. + ZERO_WIDTH = [ + "\u200b", + "\u200c", + "\u200d", + "\u2060", + "\u2061", + "\u2062", + "\u2063", + "\u2064", + "\ufeff", + "\u180e", + ] + + @pytest.mark.parametrize("ch", CONTROLS_REMOVED) + def test_non_whitespace_controls_are_removed(self, ch): + assert strip_control_chars(f"a{ch}b") == "ab" + + @pytest.mark.parametrize("ch", CONTROLS_KEPT) + def test_whitespace_controls_are_preserved(self, ch): + """Deleting these would join the tokens either side — the #433 contract.""" + assert strip_control_chars(f"a{ch}b") == f"a{ch}b" + + @pytest.mark.parametrize("ch", ZERO_WIDTH) + def test_zero_width_characters_are_removed(self, ch): + assert strip_zero_width_chars(f"pay{ch}pal") == "paypal" + + def test_the_two_functions_do_not_overlap(self): + """Each leaves the other's set alone, which is why both exist.""" + for ch in self.ZERO_WIDTH: + assert strip_control_chars(f"a{ch}b") == f"a{ch}b", ch + for ch in self.CONTROLS_REMOVED: + assert strip_zero_width_chars(f"a{ch}b") == f"a{ch}b", ch + + def test_ordinary_text_is_untouched(self): + for s in ("hello world", "Café déjà vu", "Привет мир", "a\tb\nc"): + assert strip_control_chars(s) == s + assert strip_zero_width_chars(s) == s + + def test_composition_matches_canonicalize(self): + """The documented recipe: strip controls, then fold whitespace.""" + messy = " \x00 hello \x7f " + assert collapse_whitespace(strip_control_chars(messy)) == "hello" + assert canonicalize(messy) == "hello" + + def test_text_methods_mirror_the_functions(self): + assert str(Text("a\x00b").strip_control_chars()) == "ab" + assert str(Text("pay\u200bpal").strip_zero_width_chars()) == "paypal" + # Fluent chaining is the point of the Text wrapper. + assert str(Text(" \x00 hi \x7f ").strip_control_chars().collapse_whitespace()) == "hi"