fix(search): preserve malformed names (no collapse) + --normalize-malformed switch

crates/uffs-client/src/protocol/cli_args.rs

@@ -51,6 +51,7 @@ impl SearchParams {
                 "--dirs-only" => raw.dirs_only = true,
                 "--hide-system" => raw.hide_system = true,
                 "--hide-ads" => raw.hide_ads = true,
+                "--normalize-malformed" => raw.normalize_malformed = true,
                 // WI-4.4 forensic filters: find ill-formed (non-UTF-8) names.
                 "--malformed" => raw.malformed = Some(true),
                 "--well-formed" => raw.malformed = Some(false),
@@ -306,6 +307,7 @@ struct RawCliArgs {
     dirs_only: bool,
     hide_system: bool,
     hide_ads: bool,
+    normalize_malformed: bool,
     /// WI-4.4: `Some(true)` from `--malformed`, `Some(false)` from
     /// `--well-formed`, `None` if neither (no filter).
     malformed: Option<bool>,
@@ -719,6 +721,7 @@ impl RawCliArgs {
             // Misc
             hide_system: self.hide_system,
             hide_ads: self.hide_ads,
+            normalize_malformed: self.normalize_malformed,
             // Profiling
             profile: self.profile || self.benchmark,
             aggregations,

crates/uffs-client/src/protocol/mod.rs

@@ -428,6 +428,11 @@ pub struct SearchParams {
     /// Hide NTFS Alternate Data Streams from results.
     #[serde(default)]
     pub hide_ads: bool,
+    /// Render ill-formed (surrogate-bearing) names with greppable `<BAD:HHHH>`
+    /// markers instead of the default U+FFFD (`�`). Display-only — does not
+    /// filter results (use the malformed filter for that).
+    #[serde(default)]
+    pub normalize_malformed: bool,
 
     // ── Profiling ──────────────────────────────────────────────────
     /// Request detailed timing breakdown from the daemon.
@@ -598,6 +603,7 @@ impl Default for SearchParams {
             malformed_path: None,
             hide_system: false,
             hide_ads: false,
+            normalize_malformed: false,
             profile: false,
             aggregations: vec![],
             include_rows: true,

crates/uffs-client/src/protocol/tests.rs

@@ -69,6 +69,45 @@ fn search_params_round_trip() {
     assert_eq!(parsed.response_mode, Some(SearchResponseMode::Json));
 }
 
+/// `normalize_malformed` round-trips, and an older payload that omits it
+/// deserializes as `false` (backward-compatible via `#[serde(default)]`).
+#[test]
+fn search_params_normalize_malformed_round_trip_and_default() {
+    let params = SearchParams {
+        pattern: "*".to_owned(),
+        normalize_malformed: true,
+        ..Default::default()
+    };
+    let json = serde_json::to_value(&params).expect("serialize");
+    let parsed: SearchParams = serde_json::from_value(json).expect("deserialize");
+    assert!(
+        parsed.normalize_malformed,
+        "the flag must survive the JSON-RPC round trip"
+    );
+
+    // A payload from an older client that never knew the field.
+    let legacy: SearchParams =
+        serde_json::from_value(serde_json::json!({ "pattern": "*" })).expect("legacy deserialize");
+    assert!(
+        !legacy.normalize_malformed,
+        "omitted field defaults off (wire backward-compat)"
+    );
+}
+
+/// The CLI surface: `--normalize-malformed` sets the param; absent → off.
+#[test]
+fn from_cli_args_normalize_malformed_flag() {
+    let on = SearchParams::from_cli_args(&["*.tmp".to_owned(), "--normalize-malformed".to_owned()])
+        .expect("parse with flag");
+    assert!(
+        on.normalize_malformed,
+        "--normalize-malformed must set the flag"
+    );
+
+    let off = SearchParams::from_cli_args(&["*.tmp".to_owned()]).expect("parse without flag");
+    assert!(!off.normalize_malformed, "absent flag defaults off");
+}
+
 /// Canonical helpers preserve legacy single-flag sort semantics.
 ///
 /// First field: ascending by default (no `--sort-desc`).

crates/uffs-core/src/compact.rs

@@ -49,7 +49,7 @@ pub use delta::IndexDelta;
 pub use extension::ExtensionIndex;
 pub(crate) use path_len::{PathChange, compute_path_lengths, update_path_lengths_incremental};
 pub(crate) use record::NTFS_METAFILE_NAMES;
-pub use record::{CompactRecord, is_ntfs_metafile_name};
+pub use record::{CompactRecord, MalformedRender, is_ntfs_metafile_name};
 
 /// Touched-record count (adds + tombstones since the last compaction) above
 /// which [`DriveCompactIndex::apply_index_delta`] folds the delta back into

crates/uffs-core/src/compact/record.rs

@@ -124,6 +124,92 @@ pub fn is_ntfs_metafile_name(name: &str) -> bool {
         .any(|reserved| name.eq_ignore_ascii_case(reserved))
 }
 
+/// How an **ill-formed** (surrogate-bearing) NTFS name renders for display.
+///
+/// Used by [`CompactRecord::name_display_with`]. Well-formed names are returned
+/// verbatim under either variant, so this only matters for the rare malformed
+/// case and never touches the well-formed hot path.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum MalformedRender {
+    /// Replace each ill-formed run with U+FFFD (`�`) — the default, matching
+    /// the reference C++ tool and ordinary file managers.
+    #[default]
+    Lossy,
+    /// Replace each offending UTF-16 code unit with a greppable, reversible
+    /// sentinel `<BAD:HHHH>` (HHHH = the code unit, e.g. an unpaired low
+    /// surrogate → `<BAD:DCFF>`). `<` and `>` are invalid in NTFS names, so the
+    /// marker can never collide with a real one; the hex is unique per distinct
+    /// bad unit (so two malformed siblings stay distinct) and lossless (the
+    /// true name is recoverable). Enabled by `--normalize-malformed`.
+    Normalized,
+}
+
+/// Render WTF-8 `bytes` one **UTF-16 code unit** at a time, replacing each
+/// offending unit (an unpaired surrogate) with a marker; well-formed runs pass
+/// through unchanged. Only called on the rare ill-formed path (the caller has
+/// already seen `from_utf8` fail).
+///
+/// Crucially this is **per code unit**, not per byte: a WTF-8 lone surrogate is
+/// the 3-byte sequence `0xED 0xA0..=0xBF 0x80..=0xBF` decoding to a single
+/// U+D800..=U+DFFF unit, so it yields ONE marker (matching the reference C++
+/// tool and ordinary file managers) rather than `from_utf8_lossy`'s three. The
+/// marker is U+FFFD (`�`) for [`MalformedRender::Lossy`] or `<BAD:HHHH>` for
+/// [`MalformedRender::Normalized`]. Any other invalid byte (not expected from a
+/// faithful NTFS name) is treated as one offending position.
+fn render_malformed(bytes: &[u8], render: MalformedRender) -> String {
+    let mut out = String::with_capacity(bytes.len().saturating_add(8));
+    let mut rest = bytes;
+    while !rest.is_empty() {
+        match core::str::from_utf8(rest) {
+            Ok(valid) => {
+                out.push_str(valid);
+                break;
+            }
+            Err(err) => {
+                let good = err.valid_up_to();
+                let (head, bad) = rest.split_at(good);
+                // `head` is valid UTF-8 by construction (`valid_up_to`).
+                out.push_str(core::str::from_utf8(head).unwrap_or(""));
+                if let [0xED, b1 @ 0xA0..=0xBF, b2 @ 0x80..=0xBF, tail @ ..] = bad {
+                    match render {
+                        MalformedRender::Lossy => out.push('\u{FFFD}'),
+                        MalformedRender::Normalized => {
+                            // Decode the WTF-8 lone surrogate back to its 16-bit
+                            // code unit (lead nibble is the fixed 0xD of 0xED),
+                            // then emit `<BAD:HHHH>`. `wrapping_sh*` + bit-ops +
+                            // `char::from_digit` are all panic-free here (values
+                            // masked to ≤ 0xF, base 16).
+                            let unit: u32 = 0xD000
+                                | (u32::from(*b1) & 0x3F).wrapping_shl(6)
+                                | (u32::from(*b2) & 0x3F);
+                            out.push_str("<BAD:");
+                            for shift in [12_u32, 8, 4, 0] {
+                                let nibble = unit.wrapping_shr(shift) & 0xF;
+                                out.push(
+                                    char::from_digit(nibble, 16)
+                                        .unwrap_or('0')
+                                        .to_ascii_uppercase(),
+                                );
+                            }
+                            out.push('>');
+                        }
+                    }
+                    rest = tail;
+                } else {
+                    let adv = err.error_len().unwrap_or(1).min(bad.len());
+                    let (_, after) = bad.split_at(adv);
+                    match render {
+                        MalformedRender::Lossy => out.push('\u{FFFD}'),
+                        MalformedRender::Normalized => out.push_str("<BAD>"),
+                    }
+                    rest = after;
+                }
+            }
+        }
+    }
+    out
+}
+
 impl CompactRecord {
     /// Directory flag bit in raw NTFS `FILE_ATTRIBUTE_DIRECTORY`.
     const DIRECTORY_BIT: u32 = 0x0010;
@@ -156,16 +242,70 @@ impl CompactRecord {
     /// Get the name from a names blob as a **lossy `&str` view**.
     ///
     /// Valid-UTF-8 names (the common case) are returned verbatim; an ill-formed
-    /// (surrogate-bearing) name stored as WTF-8 returns `""` for display. Use
-    /// [`Self::name_bytes`] for the lossless bytes that exact/substring search
-    /// matches against, so a file with an ill-formed name stays findable
-    /// (WI-4.4).
+    /// (surrogate-bearing) name stored as WTF-8 returns `""`. This is the
+    /// cheap borrow used by internal hot paths (metafile check, fold/search
+    /// keys); for anything **user-visible** prefer [`Self::name_display`],
+    /// which renders ill-formed names lossily instead of emptying them (an
+    /// empty segment collapses reconstructed paths). Use
+    /// [`Self::name_bytes`] for the lossless bytes that exact/substring
+    /// search matches against, so a file with an ill-formed name stays
+    /// findable (WI-4.4).
     #[inline]
     #[must_use]
     pub fn name<'a>(&self, names: &'a [u8]) -> &'a str {
         core::str::from_utf8(self.name_bytes(names)).unwrap_or("")
     }
 
+    /// Get the name as a **lossy display string** for paths and the name
+    /// column.
+    ///
+    /// Valid-UTF-8 names (the overwhelming common case) are returned as a
+    /// zero-cost `Cow::Borrowed`; an ill-formed (surrogate-bearing) name is
+    /// rendered with U+FFFD (`�`) via [`String::from_utf8_lossy`] — preserving
+    /// the entry's **position** instead of collapsing it to `""`, so a file
+    /// under a malformed-named directory keeps its true path and stays visible
+    /// (WI-4.4). Only the rare malformed case allocates, so the well-formed hot
+    /// path is unaffected. Filtering malformed entries stays on the `malformed`
+    /// flag (a real folder named `�` is not the same as a corrupt one); this
+    /// accessor is purely for rendering.
+    #[inline]
+    #[must_use]
+    pub fn name_display<'a>(&self, names: &'a [u8]) -> alloc::borrow::Cow<'a, str> {
+        self.name_display_with(names, MalformedRender::Lossy)
+    }
+
+    /// Whether the stored name is ill-formed UTF-16 (its WTF-8 bytes are not
+    /// valid UTF-8). Cheap; used to gate the lossy render and the malformed
+    /// flag.
+    #[inline]
+    #[must_use]
+    pub fn is_name_malformed(&self, names: &[u8]) -> bool {
+        core::str::from_utf8(self.name_bytes(names)).is_err()
+    }
+
+    /// Like [`Self::name_display`] but choosing how an ill-formed name renders:
+    /// [`MalformedRender::Lossy`] (U+FFFD) or [`MalformedRender::Normalized`]
+    /// (`<BAD:HHHH>` markers, for `--normalize-malformed`). A valid name is a
+    /// zero-cost `Cow::Borrowed` under either mode; only the rare malformed
+    /// name allocates, so the well-formed hot path is identical to
+    /// `name_display`.
+    #[inline]
+    #[must_use]
+    pub fn name_display_with<'a>(
+        &self,
+        names: &'a [u8],
+        render: MalformedRender,
+    ) -> alloc::borrow::Cow<'a, str> {
+        let bytes = self.name_bytes(names);
+        // Well-formed (the overwhelming common case) → zero-cost borrow; an
+        // ill-formed name renders one marker per offending code unit (U+FFFD or
+        // `<BAD:HHHH>`), matching the reference tool's per-unit replacement.
+        core::str::from_utf8(bytes).map_or_else(
+            |_| alloc::borrow::Cow::Owned(render_malformed(bytes, render)),
+            alloc::borrow::Cow::Borrowed,
+        )
+    }
+
     /// Get the name's **raw bytes** (WTF-8) from a names blob — the lossless
     /// accessor.
     ///