feat: export strip_control_chars and strip_zero_width_chars from Python (#616)

CHANGELOG.md

@@ -493,6 +493,32 @@ compatibility (see [RELEASING.md](RELEASING.md)).
 
 ### Fixed
 
+- **Python can now call `strip_control_chars` and `strip_zero_width_chars` directly
+  (#616).** They already existed in the Rust core (`disarm::api`) and in the C ABI,
+  Java/Kotlin, Node and Ruby bindings. Python was the only surface without them, so
+  control-stripping there meant constructing a `TextPipeline` rather than calling a
+  function — unlike the ten sibling `strip_*` operations, four of which
+  (`strip_tags`, `strip_pua`, `strip_noncharacters`, `strip_variation_selectors`) are
+  narrower and are plain functions. Both are now exported, and `Text` gains the
+  matching fluent methods.
+
+  The parity matrix recorded the gap as deliberate and named the substitute as
+  `collapse_whitespace(strip_control=True)` — a signature that has never existed;
+  `collapse_whitespace` takes only `text`. That record lived in `PROVIDED_VIA` in
+  `scripts/parity.py`, so anyone consulting the matrix for the Python equivalent was
+  sent to a `TypeError`. Both entries are removed and the matrix regenerated.
+
+- **`collapse_whitespace` gains a property test covering control characters.** The
+  existing `no_leading_trailing_whitespace` property draws from `\PC*`, which
+  excludes controls, so the trim invariant was never tested against them. It holds:
+  measured exhaustively over the cross product of whitespace, controls and letters
+  for lengths 1–4, and over 200,000 random strings, with zero cases where the output
+  starts or ends with whitespace. Reported as a trim bug in #612; that report was
+  wrong and is retracted there. What looked like a defeated trim is the space
+  *between* a leading control and the word, which is interior by the same rule that
+  makes `"a\u{0}b"` keep both of its spaces. No behaviour change — the test closes
+  the coverage gap that made the question open.
+
 - **`is_suspicious_hostname()` now catches zero-width and invisible characters, and no
   longer reports a phantom script for them (#605).** Sibling of #603, for the characters
   that carry no direction at all — `U+200B`–`U+200D`, `U+2060`–`U+2064`, `U+FEFF` and

generated/parity.yaml

@@ -370,7 +370,7 @@ operations:
   - id: strip_control_chars
     names:
       rust: strip_control_chars
-      python: { provided_via: "collapse_whitespace(strip_control=True) / get_pipeline()" }
+      python: strip_control_chars
       ruby: strip_control_chars
       node: stripControlChars
   - id: strip_format
@@ -424,7 +424,7 @@ operations:
   - id: strip_zero_width_chars
     names:
       rust: strip_zero_width_chars
-      python: { provided_via: "collapse_whitespace(strip_zero_width=True) / get_pipeline()" }
+      python: strip_zero_width_chars
       ruby: strip_zero_width_chars
       node: stripZeroWidthChars
   - id: terminal_width

python/disarm/__init__.py

@@ -61,7 +61,9 @@
     set_emoji_provider,
     slugify,
     strip_accents,
+    strip_control_chars,
     strip_log_injection,
+    strip_zero_width_chars,
     terminal_width,
     transliterate,
     unmapped_confusables,
@@ -249,6 +251,7 @@
     "normalize_confusables",
     "sanitize_filename",
     "strip_accents",
+    "strip_control_chars",
     "fold_case",
     "collapse_whitespace",
     "demojize",
@@ -292,6 +295,7 @@
     "escape_html",
     "percent_encode",
     "strip_log_injection",
+    "strip_zero_width_chars",
     "HostnameAnalysis",
     # Reverse transliteration
     "reverse_langs",

python/disarm/_api.py

@@ -80,7 +80,9 @@
     _slugify_batch,
     _strip_accents,
     _strip_accents_batch,
+    _strip_control_chars,
     _strip_log_injection,
+    _strip_zero_width_chars,
     # #476: the surrogate-boundary guard, for the class-based entrypoints (the module
     # loop in _boundary wraps only free functions, not class methods).
     _surrogate_safe,
@@ -912,10 +914,10 @@ def collapse_whitespace(text: str) -> str:
     Folds **whitespace only** (#433): the line controls (TAB/LF/VT/FF/CR), the
     information separators (U+001C–U+001F), NEL, the ``Zs``/``Zl``/``Zp`` spaces,
     and the blank-rendering set (Braille blank, the Hangul fillers) each fold to a
-    single space. It does **not** delete control or zero-width characters — to do
-    that, run a :class:`TextPipeline` with the ``strip_control`` /
-    ``strip_zero_width`` steps (the ``canonicalize`` / ``canonicalize_strict``
-    presets already do).
+    single space. It does **not** delete control or zero-width characters — for
+    that, call :func:`strip_control_chars` / :func:`strip_zero_width_chars`, or
+    use a preset that sequences them ahead of the fold (``canonicalize`` and
+    ``canonicalize_strict`` both do).
 
     Folding the line controls (rather than deleting them) means a carriage return
     between two tokens becomes a space, never a silent join: ``"a\\rb"`` →
@@ -940,6 +942,63 @@ def collapse_whitespace(text: str) -> str:
     return _collapse_whitespace(text)
 
 
+def strip_control_chars(text: str) -> str:
+    """Remove control characters that are **not** whitespace (#433).
+
+    Deletes every C0/C1 control (NUL, BEL, ESC, DEL, the C1 block) *except* the
+    ones :func:`collapse_whitespace` folds — TAB, LF, VT, FF, CR, the information
+    separators ``U+001C``–``U+001F``, and NEL. Those are preserved here so the
+    fold can turn them into a space; deleting them would join the tokens either
+    side, which is the invisible-join hazard the split exists to avoid.
+
+    Pair it with :func:`collapse_whitespace` when you want both, in that order.
+
+    Args:
+        text: Input string.
+
+    Returns:
+        String with non-whitespace controls removed.
+
+    Examples:
+        >>> strip_control_chars("a\\x00b\\x07c")
+        'abc'
+        >>> strip_control_chars("a\\rb")  # CR preserved for the fold to handle
+        'a\\rb'
+    """
+    if not isinstance(text, str):
+        raise TypeError(f"strip_control_chars() expects str, got {type(text).__name__}")
+    return _strip_control_chars(text)
+
+
+def strip_zero_width_chars(text: str) -> str:
+    """Remove zero-width characters.
+
+    Deletes the zero-width set, which renders as nothing and is used to fragment a
+    token so it evades a denylist while looking unchanged. The set is exactly:
+
+    - ``U+200B``–``U+200D`` — ZWSP, ZWNJ, ZWJ
+    - ``U+2060``–``U+2064`` — word joiner and the invisible operators
+    - ``U+FEFF`` — BOM / zero-width no-break space
+    - ``U+180E`` — Mongolian vowel separator (reclassified ``Zs`` → ``Cf`` in
+      Unicode 6.3, so it is a format character despite the name)
+
+    Args:
+        text: Input string.
+
+    Returns:
+        String with zero-width characters removed.
+
+    Examples:
+        >>> strip_zero_width_chars("pay\\u200bpal")
+        'paypal'
+        >>> strip_zero_width_chars("a\\ufeffb")
+        'ab'
+    """
+    if not isinstance(text, str):
+        raise TypeError(f"strip_zero_width_chars() expects str, got {type(text).__name__}")
+    return _strip_zero_width_chars(text)
+
+
 def demojize(
     text: str,
     *,

python/disarm/_boundary.pyi

@@ -206,6 +206,9 @@ from disarm._core import (
 from disarm._core import (
     _strip_bidi as _strip_bidi,
 )
+from disarm._core import (
+    _strip_control_chars as _strip_control_chars,
+)
 from disarm._core import (
     _strip_format as _strip_format,
 )
@@ -230,6 +233,9 @@ from disarm._core import (
 from disarm._core import (
     _strip_zalgo as _strip_zalgo,
 )
+from disarm._core import (
+    _strip_zero_width_chars as _strip_zero_width_chars,
+)
 from disarm._core import (
     _terminal_width as _terminal_width,
 )

python/disarm/_core.pyi

@@ -209,6 +209,8 @@ def _sanitize_filename(
 def _strip_accents(text: str) -> str: ...
 def _fold_case(text: str) -> str: ...
 def _collapse_whitespace(text: str) -> str: ...
+def _strip_control_chars(text: str) -> str: ...
+def _strip_zero_width_chars(text: str) -> str: ...
 def _demojize(
     text: str,
     *,

python/disarm/_text.py

@@ -180,6 +180,18 @@ def collapse_whitespace(self) -> Text:
         than being deleted, so ``"a\\rb"`` becomes ``"a b"``."""
         return Text(self._t().collapse_whitespace(self._value))
 
+    def strip_control_chars(self) -> Text:
+        """Remove control characters that are not whitespace (#433).
+
+        The controls :meth:`collapse_whitespace` folds — TAB, LF, VT, FF, CR, the
+        information separators and NEL — are preserved so the fold can turn them
+        into a space; deleting them would join the tokens either side."""
+        return Text(self._t().strip_control_chars(self._value))
+
+    def strip_zero_width_chars(self) -> Text:
+        """Remove zero-width characters (ZWSP, ZWNJ, ZWJ, word joiner, BOM, …)."""
+        return Text(self._t().strip_zero_width_chars(self._value))
+
     def slugify(
         self,
         *,

python/disarm/_text.pyi

@@ -50,6 +50,12 @@ class Text:
     def collapse_whitespace(self) -> Text:
         """Fold whitespace runs to single ASCII spaces (fold-only, #433)."""
         ...
+    def strip_control_chars(self) -> Text:
+        """Remove control characters that are not whitespace (#433)."""
+        ...
+    def strip_zero_width_chars(self) -> Text:
+        """Remove zero-width characters (ZWSP, ZWNJ, ZWJ, word joiner, BOM)."""
+        ...
     def slugify(
         self,
         *,

scripts/parity.py

@@ -8,8 +8,11 @@
     (v0 ignored re-exports -> has_anomalies/inspect_anomalies false nulls).
   * Ruby surface  = real `def` lines in bindings/ruby/lib/disarm.rb.
   * Schema gains `alias_of` and `provided_via` so folded/aliased ops are not
-    mislabelled as gaps (reverse_transliterate via transliterate(target=...),
-    strip_control_chars/strip_zero_width_chars via the pipeline, etc.).
+    mislabelled as gaps (reverse_transliterate via transliterate(target=...)).
+    Use `provided_via` only for a route that is real and callable: the entries
+    for strip_control_chars/strip_zero_width_chars named
+    `collapse_whitespace(strip_control=True)`, a signature that never existed,
+    which hid the gap until #616 exported both from Python.
 Caveat: Python+Rust are verified against the real public surface; Ruby is parsed
 from source defs (reliable) and Node from `export function` (reliable), but
 neither is runtime-introspected (no toolchain) — finalize with
@@ -100,10 +103,6 @@ def canon(name, lang):
     # bindings expose a nullary accessor because a native module cannot export a static.
     "confusables_version": {"python": "disarm.CONFUSABLES_VERSION"},
     "reverse_transliterate": {"python": "transliterate(target=…)"},
-    "strip_control_chars": {"python": "collapse_whitespace(strip_control=True) / get_pipeline()"},
-    "strip_zero_width_chars": {
-        "python": "collapse_whitespace(strip_zero_width=True) / get_pipeline()"
-    },
 }
 # Deliberate scope decisions for Ruby/Node — not blind backfill:
 #  * registration mutates process-global state; encoders are sink-context tools;

src/lib.rs

@@ -245,6 +245,11 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(py::filename::_sanitize_filename, m)?)?;
     m.add_function(wrap_pyfunction!(py::case_fold::_fold_case, m)?)?;
     m.add_function(wrap_pyfunction!(py::whitespace::_collapse_whitespace, m)?)?;
+    m.add_function(wrap_pyfunction!(py::whitespace::_strip_control_chars, m)?)?;
+    m.add_function(wrap_pyfunction!(
+        py::whitespace::_strip_zero_width_chars,
+        m
+    )?)?;
     m.add_function(wrap_pyfunction!(py::scripts::_detect_scripts, m)?)?;
     m.add_function(wrap_pyfunction!(py::scripts::_is_mixed_script, m)?)?;
     m.add_function(wrap_pyfunction!(py::scripts::_has_bidi_conflict, m)?)?;

src/py/whitespace.rs

@@ -8,3 +8,17 @@ use pyo3::prelude::*;
 pub fn _collapse_whitespace(text: &str) -> String {
     crate::whitespace::collapse_whitespace(text)
 }
+
+/// `strip_control_chars(text) -> str` (#616).
+#[pyfunction]
+#[pyo3(signature = (text,))]
+pub fn _strip_control_chars(text: &str) -> String {
+    crate::api::strip_control_chars(text)
+}
+
+/// `strip_zero_width_chars(text) -> str` (#616).
+#[pyfunction]
+#[pyo3(signature = (text,))]
+pub fn _strip_zero_width_chars(text: &str) -> String {
+    crate::api::strip_zero_width_chars(text)
+}

src/whitespace.rs

@@ -169,6 +169,14 @@ pub(crate) fn is_zero_width(ch: char) -> bool {
 mod tests {
     use super::*;
 
+    /// Controls are still never DELETED — that is `strip_control_chars`' job, and
+    /// deleting here would join the tokens either side.
+    #[test]
+    fn controls_are_preserved_not_deleted() {
+        assert_eq!(collapse_whitespace("a\u{0}b"), "a\u{0}b");
+        assert_eq!(collapse_whitespace("a \u{0} b"), "a \u{0} b");
+    }
+
     #[test]
     fn test_collapse_whitespace() {
         assert_eq!(collapse_whitespace("hello   world"), "hello world");
@@ -356,6 +364,26 @@ mod tests {
                 let twice = collapse_whitespace(&once);
                 prop_assert_eq!(&once, &twice);
             }
+
+            /// The trim invariant holds when CONTROLS are in the mix too.
+            ///
+            /// `no_leading_trailing_whitespace` above draws from `\PC*`, which
+            /// excludes controls, so it never covered this. A control is content
+            /// for run-collapsing — `"a \u{0} b"` keeps both spaces — and the
+            /// question that leaves open is whether one at an edge can strand
+            /// whitespace outside it. It cannot: the leading run is dropped before
+            /// the control is emitted, and the trailing truncate runs after.
+            /// Measured over the full cross product of these classes (#612).
+            #[test]
+            fn no_edge_whitespace_even_with_controls(
+                s in r"[ab\u{00e9}\x20\x09\x0a\u{00a0}\u{3000}\x00\x07\x1b\x7f\u{0080}\u{009f}]{0,16}"
+            ) {
+                let result = collapse_whitespace(&s);
+                if !result.is_empty() {
+                    prop_assert_ne!(result.chars().next().unwrap(), ' ');
+                    prop_assert_ne!(result.chars().next_back().unwrap(), ' ');
+                }
+            }
         }
     }
 }

tests/test_api_stability.py

@@ -46,6 +46,8 @@
     "normalize_confusables",
     "sanitize_filename",
     "strip_accents",
+    "strip_control_chars",
+    "strip_zero_width_chars",
     "fold_case",
     "collapse_whitespace",
     "demojize",
@@ -350,6 +352,8 @@ def _param_kinds(fn) -> dict[str, str]:
     "strip_accents": ["text"],
     "fold_case": ["text"],
     "collapse_whitespace": ["text"],
+    "strip_control_chars": ["text"],
+    "strip_zero_width_chars": ["text"],
     "demojize": [
         "text",
         "strip_modifiers",
@@ -487,6 +491,8 @@ def test_first_param_is_positional(self, name: str):
         ],
         "fold_case": [],
         "collapse_whitespace": [],
+        "strip_control_chars": [],
+        "strip_zero_width_chars": [],
         "slugify": [
             "separator",
             "lowercase",

tests/test_form_invariance_audit.py

@@ -49,6 +49,12 @@
     "strip_variation_selectors",
     "strip_noncharacters",
     "collapse_whitespace",
+    # #616: same category as the strips above — each deletes one character class
+    # and touches nothing else, so `ї` stays decomposed if it arrived decomposed.
+    # Neither is a recovery entrypoint; compose them with a preset when you want
+    # boundary normalization as well.
+    "strip_control_chars",
+    "strip_zero_width_chars",
     "escape_html",
     "strip_log_injection",
     "demojize",