From 98c125b60c6968a9c29d43f5b877cc9afa7e8a2c Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Sun, 28 Jun 2026 05:57:43 -0700 Subject: [PATCH 1/3] fix(search): preserve malformed-named entries in resolved paths (no collapse) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A directory whose NTFS name is ill-formed (unpaired UTF-16 surrogate) is stored byte-faithfully and flagged `malformed`, but the lossy `name()` accessor returned "" for it. The path resolver pushed that empty segment, so `…\evil�.exe\report.txt` collapsed to `…\report.txt` — re-parenting children to the volume root and emitting duplicate parent rows. That is the bulk of the G-drive parity mismatch vs the reference C++ tool, which renders the lossy name. Add `CompactRecord::name_display() -> Cow`: valid names borrow at zero cost; an ill-formed name renders lossily (U+FFFD, like C++) instead of emptying. Repoint both path resolvers to push `name_display`, and fix `resolve_path_inner` to terminate the parent walk on the lossless bytes (it was breaking on the empty lossy name, truncating everything beneath a crooked directory). Hot path unaffected: well-formed names take `Cow::Borrowed`; only the rare malformed case allocates. Filtering stays on the `malformed` flag. Tests: a crooked directory segment is preserved in the resolved path (both resolvers agree, path flagged malformed), and a crooked-leaf file is still enumerated in search results with its lossy path — the two cases that collapsed / went missing on the G drive. Co-Authored-By: Claude Opus 4.8 (The WI-4.4 malformed tests are extracted into a `compact_tests/malformed.rs` submodule so `compact_tests.rs` stays under the 800-LOC policy.) --- crates/uffs-core/src/compact/record.rs | 30 +- crates/uffs-core/src/compact_tests.rs | 149 +-------- .../uffs-core/src/compact_tests/malformed.rs | 291 ++++++++++++++++++ crates/uffs-core/src/search/tree.rs | 47 +-- 4 files changed, 352 insertions(+), 165 deletions(-) create mode 100644 crates/uffs-core/src/compact_tests/malformed.rs diff --git a/crates/uffs-core/src/compact/record.rs b/crates/uffs-core/src/compact/record.rs index 8fcb19638..fe48ac80c 100644 --- a/crates/uffs-core/src/compact/record.rs +++ b/crates/uffs-core/src/compact/record.rs @@ -156,16 +156,38 @@ impl CompactRecord { /// Get the name from a names blob as a **lossy `&str` view**. /// /// Valid-UTF-8 names (the common case) are returned verbatim; an ill-formed - /// (surrogate-bearing) name stored as WTF-8 returns `""` for display. Use - /// [`Self::name_bytes`] for the lossless bytes that exact/substring search - /// matches against, so a file with an ill-formed name stays findable - /// (WI-4.4). + /// (surrogate-bearing) name stored as WTF-8 returns `""`. This is the + /// cheap borrow used by internal hot paths (metafile check, fold/search + /// keys); for anything **user-visible** prefer [`Self::name_display`], + /// which renders ill-formed names lossily instead of emptying them (an + /// empty segment collapses reconstructed paths). Use + /// [`Self::name_bytes`] for the lossless bytes that exact/substring + /// search matches against, so a file with an ill-formed name stays + /// findable (WI-4.4). #[inline] #[must_use] pub fn name<'a>(&self, names: &'a [u8]) -> &'a str { core::str::from_utf8(self.name_bytes(names)).unwrap_or("") } + /// Get the name as a **lossy display string** for paths and the name + /// column. + /// + /// Valid-UTF-8 names (the overwhelming common case) are returned as a + /// zero-cost `Cow::Borrowed`; an ill-formed (surrogate-bearing) name is + /// rendered with U+FFFD (`�`) via [`String::from_utf8_lossy`] — preserving + /// the entry's **position** instead of collapsing it to `""`, so a file + /// under a malformed-named directory keeps its true path and stays visible + /// (WI-4.4). Only the rare malformed case allocates, so the well-formed hot + /// path is unaffected. Filtering malformed entries stays on the `malformed` + /// flag (a real folder named `�` is not the same as a corrupt one); this + /// accessor is purely for rendering. + #[inline] + #[must_use] + pub fn name_display<'a>(&self, names: &'a [u8]) -> alloc::borrow::Cow<'a, str> { + String::from_utf8_lossy(self.name_bytes(names)) + } + /// Get the name's **raw bytes** (WTF-8) from a names blob — the lossless /// accessor. /// diff --git a/crates/uffs-core/src/compact_tests.rs b/crates/uffs-core/src/compact_tests.rs index 66d875ca2..ebe69256e 100644 --- a/crates/uffs-core/src/compact_tests.rs +++ b/crates/uffs-core/src/compact_tests.rs @@ -7,6 +7,13 @@ use uffs_mft::index::{ use super::*; +// WI-4.4 malformed-name tests live in a sibling file (shared fixtures via +// `super`) to keep this file under the 800-LOC policy. Explicit `#[path]` +// because this module is itself `#[path]`-loaded (`compact.rs` → `tests`), +// so plain `mod malformed;` would resolve against `src/`, not here. +#[path = "compact_tests/malformed.rs"] +mod malformed; + // ── helpers ────────────────────────────────────────────────────── /// Push a name into the index and return its `IndexNameRef`. @@ -573,148 +580,6 @@ fn ads_on_directory_strips_directory_flag() { ); } -// ── WI-4.4: a crooked (surrogate-named) file cannot hide from the search -// layer. Build the compact/search index from an MftIndex holding an -// ill-formed name and prove it is enumerated and byte-recoverable. ── - -/// WTF-8 of `evil` + lone-high-surrogate(U+D800) + `.exe` -/// (`0xD800` → 3-byte WTF-8 `ED A0 80`). Not valid UTF-8. -const CROOKED_NAME_WTF8: &[u8] = &[ - b'e', b'v', b'i', b'l', 0xED, 0xA0, 0x80, b'.', b'e', b'x', b'e', -]; - -/// Like `push_name`, but stores raw WTF-8 bytes (the lossless ingestion path) -/// for an ill-formed NTFS name. -fn push_name_bytes(index: &mut MftIndex, bytes: &[u8]) -> IndexNameRef { - let offset = index.add_name_bytes(bytes); - let len = u16::try_from(bytes.len()).expect("test name too long"); - // Ill-formed → not ASCII, no extension (the `.exe` here is decorative). - IndexNameRef::new(offset, len, false, 0) -} - -#[test] -fn crooked_surrogate_name_is_visible_in_compact_index() { - let mut idx = fixture_index(); - - // Plant a file whose name contains an unpaired surrogate under root. - let crooked = push_name_bytes(&mut idx, CROOKED_NAME_WTF8); - let rec = idx.get_or_create(909.into()); - rec.first_name.name = crooked; - rec.first_name.parent_frs = Into::into(ROOT_FRS); - rec.first_stream.size = SizeInfo { - length: 1337, - allocated: 1536, - }; - - let (drive, _, _) = build_compact_index(uffs_mft::platform::DriveLetter::C, &idx); - - // The crooked file must appear in the compact index — found by its TRUE - // bytes via the lossless `name_bytes` accessor. This is the "cannot hide" - // guarantee: a malicious ill-formed name is still enumerated. - let found = drive - .records - .iter() - .find(|cr| cr.name_bytes(&drive.names) == CROOKED_NAME_WTF8) - .expect( - "crooked surrogate-named file must be present + byte-recoverable in the search index", - ); - - assert_eq!( - found.size, 1337, - "the crooked file's metadata transfers too" - ); - // Its lossy &str view is empty (not valid UTF-8) — display degrades, but - // the file is NOT hidden (it is enumerated above). - assert_eq!(found.name(&drive.names), ""); - // And the true bytes carry no U+FFFD replacement — nothing was lost. - assert!( - !found - .name_bytes(&drive.names) - .windows(3) - .any(|win| win == [0xEF, 0xBF, 0xBD]), - "the search index must hold the true bytes, not a lossy replacement" - ); -} - -// ── WI-4.4: forensic facts on the resolved DisplayRow ──────────────────── -// A CLEAN-named file living under a CROOKED (surrogate-named) directory: -// its own leaf is well-formed (malformed=false) but its PATH is poisoned -// (malformed_path=true). This is the "clean file under a crooked dir is -// still flagged" guarantee, and it exercises the real search → resolve → -// row-forensics path end to end. - -#[test] -fn clean_child_under_crooked_dir_is_path_malformed_not_leaf_malformed() { - let mut idx = MftIndex::new(uffs_mft::platform::DriveLetter::C); - - // Root. - let root_name = push_name(&mut idx, "."); - let root = idx.get_or_create(ROOT_FRS.into()); - root.stdinfo.set_directory(true); - root.first_name.name = root_name; - root.first_name.parent_frs = Into::into(ROOT_FRS); - - // A directory whose NAME is ill-formed (lone surrogate), under root. - let crooked_dir = push_name_bytes(&mut idx, CROOKED_NAME_WTF8); - let dir = idx.get_or_create(500.into()); - dir.stdinfo.set_directory(true); - dir.first_name.name = crooked_dir; - dir.first_name.parent_frs = Into::into(ROOT_FRS); - - // A CLEAN-named child file under the crooked directory. - let child_name = push_name(&mut idx, "report.txt"); - let child = idx.get_or_create(501.into()); - child.first_name.name = child_name; - child.first_name.parent_frs = Into::into(500); - child.first_stream.size = SizeInfo { - length: 42, - allocated: 64, - }; - - let (drive, _, _) = build_compact_index(uffs_mft::platform::DriveLetter::C, &idx); - - // Match-all enumeration through the real search/resolve/forensics path. - let drives = vec![drive]; - let mut filters = crate::search::filters::SearchFilters::default(); - let (rows, _) = crate::search::query::collect_global_top_n( - &drives, - 100, - crate::search::field::FieldId::Name, - false, - crate::search::backend::FilterMode::All, - &mut filters, - ); - - // The CLEAN child: leaf well-formed, but PATH malformed (ancestor crooked). - let child_row = rows - .iter() - .find(|row| row.name() == "report.txt") - .expect("clean-named child must be enumerated"); - assert!( - !child_row.malformed, - "the child's own leaf name is well-formed" - ); - assert!( - child_row.malformed_path, - "a clean file under a crooked directory must be flagged malformed_path" - ); - // A clean leaf carries no name_hex evidence (only ill-formed leaves do). - assert!(child_row.name_hex.is_none()); - - // The crooked DIRECTORY itself: leaf malformed + name_hex evidence present. - // Its lossy `&str` view is empty, so match it by its true bytes via name_hex. - let dir_row = rows - .iter() - .find(|row| row.malformed) - .expect("the crooked directory must itself be enumerated and flagged"); - assert!(dir_row.malformed_path); - assert_eq!( - dir_row.name_hex.as_deref(), - Some("6576696ceda0802e657865"), - "name_hex is the lowercase hex of the true WTF-8 bytes of `evil.exe`" - ); -} - // ── is_ntfs_metafile_name: exact-allowlist classifier ────────────── // // 2026-06-11: `--hide-system` used to hide every name starting with `$`, diff --git a/crates/uffs-core/src/compact_tests/malformed.rs b/crates/uffs-core/src/compact_tests/malformed.rs new file mode 100644 index 000000000..2c43f9aa0 --- /dev/null +++ b/crates/uffs-core/src/compact_tests/malformed.rs @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! WI-4.4 malformed-name tests: an ill-formed (surrogate-bearing) NTFS name +//! must be stored byte-faithfully, preserved at its true position (not +//! collapsed), rendered lossily (U+FFFD) for display, and still enumerated by +//! its true bytes. +//! +//! Split out of the parent `compact_tests` to keep that file under the 800-LOC +//! policy; the synthetic-index fixtures (`fixture_index`, `push_name`) are +//! shared via `super`. + +use uffs_mft::index::{IndexNameRef, MftIndex, ROOT_FRS, SizeInfo}; + +use super::{fixture_index, push_name}; +use crate::compact::build_compact_index; + +// ── WI-4.4: a crooked (surrogate-named) file cannot hide from the search +// layer. Build the compact/search index from an MftIndex holding an +// ill-formed name and prove it is enumerated and byte-recoverable. ── + +/// WTF-8 of `evil` + lone-high-surrogate(U+D800) + `.exe` +/// (`0xD800` → 3-byte WTF-8 `ED A0 80`). Not valid UTF-8. +const CROOKED_NAME_WTF8: &[u8] = &[ + b'e', b'v', b'i', b'l', 0xED, 0xA0, 0x80, b'.', b'e', b'x', b'e', +]; + +/// Like `push_name`, but stores raw WTF-8 bytes (the lossless ingestion path) +/// for an ill-formed NTFS name. +fn push_name_bytes(index: &mut MftIndex, bytes: &[u8]) -> IndexNameRef { + let offset = index.add_name_bytes(bytes); + let len = u16::try_from(bytes.len()).expect("test name too long"); + // Ill-formed → not ASCII, no extension (the `.exe` here is decorative). + IndexNameRef::new(offset, len, false, 0) +} + +#[test] +fn crooked_surrogate_name_is_visible_in_compact_index() { + let mut idx = fixture_index(); + + // Plant a file whose name contains an unpaired surrogate under root. + let crooked = push_name_bytes(&mut idx, CROOKED_NAME_WTF8); + let rec = idx.get_or_create(909.into()); + rec.first_name.name = crooked; + rec.first_name.parent_frs = Into::into(ROOT_FRS); + rec.first_stream.size = SizeInfo { + length: 1337, + allocated: 1536, + }; + + let (drive, _, _) = build_compact_index(uffs_mft::platform::DriveLetter::C, &idx); + + // The crooked file must appear in the compact index — found by its TRUE + // bytes via the lossless `name_bytes` accessor. This is the "cannot hide" + // guarantee: a malicious ill-formed name is still enumerated. + let found = drive + .records + .iter() + .find(|cr| cr.name_bytes(&drive.names) == CROOKED_NAME_WTF8) + .expect( + "crooked surrogate-named file must be present + byte-recoverable in the search index", + ); + + assert_eq!( + found.size, 1337, + "the crooked file's metadata transfers too" + ); + // Its lossy &str view is empty (not valid UTF-8) — display degrades, but + // the file is NOT hidden (it is enumerated above). + assert_eq!(found.name(&drive.names), ""); + // And the true bytes carry no U+FFFD replacement — nothing was lost. + assert!( + !found + .name_bytes(&drive.names) + .windows(3) + .any(|win| win == [0xEF, 0xBF, 0xBD]), + "the search index must hold the true bytes, not a lossy replacement" + ); +} + +// ── WI-4.4: forensic facts on the resolved DisplayRow ──────────────────── +// A CLEAN-named file living under a CROOKED (surrogate-named) directory: +// its own leaf is well-formed (malformed=false) but its PATH is poisoned +// (malformed_path=true). This is the "clean file under a crooked dir is +// still flagged" guarantee, and it exercises the real search → resolve → +// row-forensics path end to end. + +#[test] +fn clean_child_under_crooked_dir_is_path_malformed_not_leaf_malformed() { + let mut idx = MftIndex::new(uffs_mft::platform::DriveLetter::C); + + // Root. + let root_name = push_name(&mut idx, "."); + let root = idx.get_or_create(ROOT_FRS.into()); + root.stdinfo.set_directory(true); + root.first_name.name = root_name; + root.first_name.parent_frs = Into::into(ROOT_FRS); + + // A directory whose NAME is ill-formed (lone surrogate), under root. + let crooked_dir = push_name_bytes(&mut idx, CROOKED_NAME_WTF8); + let dir = idx.get_or_create(500.into()); + dir.stdinfo.set_directory(true); + dir.first_name.name = crooked_dir; + dir.first_name.parent_frs = Into::into(ROOT_FRS); + + // A CLEAN-named child file under the crooked directory. + let child_name = push_name(&mut idx, "report.txt"); + let child = idx.get_or_create(501.into()); + child.first_name.name = child_name; + child.first_name.parent_frs = Into::into(500); + child.first_stream.size = SizeInfo { + length: 42, + allocated: 64, + }; + + let (drive, _, _) = build_compact_index(uffs_mft::platform::DriveLetter::C, &idx); + + // Match-all enumeration through the real search/resolve/forensics path. + let drives = vec![drive]; + let mut filters = crate::search::filters::SearchFilters::default(); + let (rows, _) = crate::search::query::collect_global_top_n( + &drives, + 100, + crate::search::field::FieldId::Name, + false, + crate::search::backend::FilterMode::All, + &mut filters, + ); + + // The CLEAN child: leaf well-formed, but PATH malformed (ancestor crooked). + let child_row = rows + .iter() + .find(|row| row.name() == "report.txt") + .expect("clean-named child must be enumerated"); + assert!( + !child_row.malformed, + "the child's own leaf name is well-formed" + ); + assert!( + child_row.malformed_path, + "a clean file under a crooked directory must be flagged malformed_path" + ); + // A clean leaf carries no name_hex evidence (only ill-formed leaves do). + assert!(child_row.name_hex.is_none()); + + // The crooked DIRECTORY itself: leaf malformed + name_hex evidence present. + // Its lossy `&str` view is empty, so match it by its true bytes via name_hex. + let dir_row = rows + .iter() + .find(|row| row.malformed) + .expect("the crooked directory must itself be enumerated and flagged"); + assert!(dir_row.malformed_path); + assert_eq!( + dir_row.name_hex.as_deref(), + Some("6576696ceda0802e657865"), + "name_hex is the lowercase hex of the true WTF-8 bytes of `evil.exe`" + ); +} + +// ── WI-4.4 regression: a malformed directory must NOT collapse the path ── +// This is the bug behind the G-drive parity mismatch: the lossy `name()` +// returned "" for a surrogate-named directory, so the path resolver pushed +// an empty segment and `…\evil�.exe\report.txt` collapsed to `…\report.txt` +// (re-parenting the child to the volume root + producing duplicate parent +// rows). `name_display()` renders the segment lossily (U+FFFD) so its +// position is preserved — matching the reference C++ tool. + +#[test] +fn crooked_dir_segment_is_preserved_in_resolved_path() { + use crate::search::tree; + + // root → → report.txt (FRS 501) + let mut idx = MftIndex::new(uffs_mft::platform::DriveLetter::C); + let root_name = push_name(&mut idx, "."); + let root = idx.get_or_create(ROOT_FRS.into()); + root.stdinfo.set_directory(true); + root.first_name.name = root_name; + root.first_name.parent_frs = Into::into(ROOT_FRS); + + let crooked_dir = push_name_bytes(&mut idx, CROOKED_NAME_WTF8); + let dir = idx.get_or_create(500.into()); + dir.stdinfo.set_directory(true); + dir.first_name.name = crooked_dir; + dir.first_name.parent_frs = Into::into(ROOT_FRS); + + let child_name = push_name(&mut idx, "report.txt"); + let child = idx.get_or_create(501.into()); + child.first_name.name = child_name; + child.first_name.parent_frs = Into::into(500); + + let (drive, _, _) = build_compact_index(uffs_mft::platform::DriveLetter::C, &idx); + + // The crooked directory's lossy display: `evil` + U+FFFD + `.exe`. + let crooked_display = String::from_utf8_lossy(CROOKED_NAME_WTF8); + assert!( + crooked_display.contains('\u{FFFD}'), + "fixture must render lossily (got {crooked_display:?})" + ); + + // `name_display` preserves the segment where `name` empties it. + let dir_rec = drive + .records + .iter() + .find(|cr| cr.name_bytes(&drive.names) == CROOKED_NAME_WTF8) + .expect("crooked dir present"); + assert_eq!( + dir_rec.name(&drive.names), + "", + "lossy &str view still empties" + ); + assert_eq!( + dir_rec.name_display(&drive.names), + crooked_display, + "name_display renders the surrogate as U+FFFD, not empty" + ); + + let child_idx = drive + .records + .iter() + .position(|cr| cr.name_bytes(&drive.names) == b"report.txt") + .expect("child present"); + + // Both resolvers must keep the child UNDER the crooked dir (not at root). + let expected = format!("C:\\{crooked_display}\\report.txt"); + let plain = tree::resolve_path(&drive, child_idx, "C:"); + assert_eq!( + plain, expected, + "plain resolver must preserve the crooked segment" + ); + assert_ne!( + plain, "C:\\report.txt", + "regression: the crooked segment must not collapse to the parent" + ); + + let mut dir_cache = tree::DirCache::default(); + let mut mal_cache = tree::MalformedCache::default(); + let (mpath, malformed) = tree::resolve_path_cached_with_malformed( + &drive, + child_idx, + "C:", + &mut dir_cache, + &mut mal_cache, + ); + assert_eq!( + mpath, expected, + "malformed-aware resolver agrees on the preserved path" + ); + assert!( + malformed, + "the path is flagged malformed (crooked ancestor)" + ); +} + +#[test] +fn crooked_leaf_file_is_not_dropped_from_results() { + // A file whose own LEAF name is ill-formed must still be enumerated in + // search results (it was being dropped by a lossy-`name().is_empty()` + // guard — the reason the crooked *files* went missing on the G drive). + let mut idx = fixture_index(); + let crooked = push_name_bytes(&mut idx, CROOKED_NAME_WTF8); + let rec = idx.get_or_create(909.into()); + rec.first_name.name = crooked; + rec.first_name.parent_frs = Into::into(ROOT_FRS); + rec.first_stream.size = SizeInfo { + length: 1337, + allocated: 1536, + }; + + let (drive, _, _) = build_compact_index(uffs_mft::platform::DriveLetter::C, &idx); + let drives = vec![drive]; + let mut filters = crate::search::filters::SearchFilters::default(); + let (rows, _) = crate::search::query::collect_global_top_n( + &drives, + 1000, + crate::search::field::FieldId::Name, + false, + crate::search::backend::FilterMode::All, + &mut filters, + ); + + let crooked_display = String::from_utf8_lossy(CROOKED_NAME_WTF8); + let found = rows + .iter() + .find(|row| row.malformed) + .expect("the crooked-leaf file must be enumerated in search results, not silently dropped"); + assert!( + found.path.contains(crooked_display.as_ref()), + "its path must carry the lossy leaf name, got {:?}", + found.path + ); +} diff --git a/crates/uffs-core/src/search/tree.rs b/crates/uffs-core/src/search/tree.rs index 33ea9ddcc..0a4a10f1f 100644 --- a/crates/uffs-core/src/search/tree.rs +++ b/crates/uffs-core/src/search/tree.rs @@ -153,9 +153,10 @@ pub fn resolve_path_cached_with_malformed( if bytes.is_empty() || bytes == b"." { None } else { - // `name()` (lossy) is what is pushed into the displayed path; - // reserve for its length, which may differ from `bytes.len()`. - Some(1 + rec.name(&drive.names).len()) + // `name_display()` (lossy, U+FFFD for ill-formed) is what is + // pushed into the displayed path; reserve for its length, which + // may differ from `bytes.len()`. + Some(1 + rec.name_display(&drive.names).len()) } }) .sum(); @@ -178,20 +179,22 @@ pub fn resolve_path_cached_with_malformed( // The single byte-level check: the lossless name bytes are not UTF-8. let component_malformed = core::str::from_utf8(bytes).is_err(); malformed |= component_malformed; - // Displayed component is the lossy view (ill-formed → empty segment), - // but the path STRUCTURE and the malformed bit reflect the true name. - let name = rec.name(&drive.names); + // Displayed component is the lossy view: ill-formed names render as + // U+FFFD (`�`) rather than an empty segment, so the malformed directory + // keeps its place in the path instead of collapsing everything beneath + // it onto its parent. + let name = rec.name_display(&drive.names); if !path.ends_with('\\') && !path.is_empty() { path.push('\\'); } - path.push_str(name); + path.push_str(&name); // Build the cacheable directory prefix + malformed bit in lockstep. if !dir_path.ends_with('\\') && !dir_path.is_empty() { dir_path.push('\\'); } - dir_path.push_str(name); + dir_path.push_str(&name); dir_malformed |= component_malformed; if rec.is_directory() { let key = uffs_mft::len_to_u32(idx); @@ -233,8 +236,12 @@ fn resolve_path_inner( break; }; - let name = record.name(&drive.names); - if name.is_empty() || name == "." { + // Terminate on the LOSSLESS bytes, not the lossy `&str`: an ill-formed + // (surrogate) directory name has non-empty bytes but an empty `&str` + // view, and must NOT truncate the chain — otherwise a crooked directory + // would hide the path of everything beneath it. + let bytes = record.name_bytes(&drive.names); + if bytes.is_empty() || bytes == b"." { break; } @@ -255,11 +262,11 @@ fn resolve_path_inner( .iter() .filter_map(|&idx| { let rec = drive.records.get(idx)?; - let name = rec.name(&drive.names); - if name.is_empty() || name == "." { + let bytes = rec.name_bytes(&drive.names); + if bytes.is_empty() || bytes == b"." { None } else { - Some(1 + name.len()) + Some(1 + rec.name_display(&drive.names).len()) } }) .sum(); @@ -268,12 +275,13 @@ fn resolve_path_inner( path.push_str(prefix); for &idx in chain.iter().rev() { if let Some(rec) = drive.records.get(idx) { - let name = rec.name(&drive.names); - if !name.is_empty() && name != "." { + let bytes = rec.name_bytes(&drive.names); + if !bytes.is_empty() && bytes != b"." { + let name = rec.name_display(&drive.names); if !path.ends_with('\\') && !path.is_empty() { path.push('\\'); } - path.push_str(name); + path.push_str(&name); } } } @@ -285,14 +293,15 @@ fn resolve_path_inner( let mut dir_path = String::from(prefix); for &idx in chain.iter().rev() { if let Some(rec) = drive.records.get(idx) { - let name = rec.name(&drive.names); - if name.is_empty() || name == "." { + let bytes = rec.name_bytes(&drive.names); + if bytes.is_empty() || bytes == b"." { continue; } + let name = rec.name_display(&drive.names); if !dir_path.ends_with('\\') && !dir_path.is_empty() { dir_path.push('\\'); } - dir_path.push_str(name); + dir_path.push_str(&name); // Only cache directories — files won't be looked up as parents. if rec.is_directory() { cache From 5ac5c29798b031ffa266ccbb4e9897a3a03e3500 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Sun, 28 Jun 2026 06:27:58 -0700 Subject: [PATCH 2/3] feat(search): render malformed names one marker per code unit (Lossy/Normalized) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `name_display`'s lossy path used `from_utf8_lossy`, which emits one U+FFFD per BYTE — so a single ill-formed UTF-16 code unit (a 3-byte WTF-8 surrogate) showed as `���` where the reference C++ tool, Everything, and file managers show a single `�`. Render per CODE UNIT instead: walk the WTF-8, pass valid runs through, and replace each offending unit with one marker. The default (Lossy) now matches C++ exactly, which also lets the G-drive malformed rows reconcile on their own. Add `MalformedRender { Lossy, Normalized }` + `CompactRecord::name_display_with`. Normalized replaces each bad code unit with a greppable, reversible `` sentinel (HHHH = the code unit, e.g. ``). `<` / `>` are invalid in NTFS names so the marker can never collide; the hex keeps two malformed siblings distinct and the true name recoverable. This is the rendering primitive the `--normalize-malformed` switch selects; the hot path is unchanged (valid names borrow, only malformed allocate). Tests: lossy → `evil�.exe` (one marker), normalized → `evil.exe`, valid names byte-identical under both modes. Co-Authored-By: Claude Opus 4.8 --- crates/uffs-core/src/compact.rs | 2 +- crates/uffs-core/src/compact/record.rs | 120 +++++++++++++++++- .../uffs-core/src/compact_tests/malformed.rs | 53 +++++++- 3 files changed, 169 insertions(+), 6 deletions(-) diff --git a/crates/uffs-core/src/compact.rs b/crates/uffs-core/src/compact.rs index 00052dd35..9a2c9ed29 100644 --- a/crates/uffs-core/src/compact.rs +++ b/crates/uffs-core/src/compact.rs @@ -49,7 +49,7 @@ pub use delta::IndexDelta; pub use extension::ExtensionIndex; pub(crate) use path_len::{PathChange, compute_path_lengths, update_path_lengths_incremental}; pub(crate) use record::NTFS_METAFILE_NAMES; -pub use record::{CompactRecord, is_ntfs_metafile_name}; +pub use record::{CompactRecord, MalformedRender, is_ntfs_metafile_name}; /// Touched-record count (adds + tombstones since the last compaction) above /// which [`DriveCompactIndex::apply_index_delta`] folds the delta back into diff --git a/crates/uffs-core/src/compact/record.rs b/crates/uffs-core/src/compact/record.rs index fe48ac80c..74d86b54c 100644 --- a/crates/uffs-core/src/compact/record.rs +++ b/crates/uffs-core/src/compact/record.rs @@ -124,6 +124,92 @@ pub fn is_ntfs_metafile_name(name: &str) -> bool { .any(|reserved| name.eq_ignore_ascii_case(reserved)) } +/// How an **ill-formed** (surrogate-bearing) NTFS name renders for display. +/// +/// Used by [`CompactRecord::name_display_with`]. Well-formed names are returned +/// verbatim under either variant, so this only matters for the rare malformed +/// case and never touches the well-formed hot path. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum MalformedRender { + /// Replace each ill-formed run with U+FFFD (`�`) — the default, matching + /// the reference C++ tool and ordinary file managers. + #[default] + Lossy, + /// Replace each offending UTF-16 code unit with a greppable, reversible + /// sentinel `` (HHHH = the code unit, e.g. an unpaired low + /// surrogate → ``). `<` and `>` are invalid in NTFS names, so the + /// marker can never collide with a real one; the hex is unique per distinct + /// bad unit (so two malformed siblings stay distinct) and lossless (the + /// true name is recoverable). Enabled by `--normalize-malformed`. + Normalized, +} + +/// Render WTF-8 `bytes` one **UTF-16 code unit** at a time, replacing each +/// offending unit (an unpaired surrogate) with a marker; well-formed runs pass +/// through unchanged. Only called on the rare ill-formed path (the caller has +/// already seen `from_utf8` fail). +/// +/// Crucially this is **per code unit**, not per byte: a WTF-8 lone surrogate is +/// the 3-byte sequence `0xED 0xA0..=0xBF 0x80..=0xBF` decoding to a single +/// U+D800..=U+DFFF unit, so it yields ONE marker (matching the reference C++ +/// tool and ordinary file managers) rather than `from_utf8_lossy`'s three. The +/// marker is U+FFFD (`�`) for [`MalformedRender::Lossy`] or `` for +/// [`MalformedRender::Normalized`]. Any other invalid byte (not expected from a +/// faithful NTFS name) is treated as one offending position. +fn render_malformed(bytes: &[u8], render: MalformedRender) -> String { + let mut out = String::with_capacity(bytes.len().saturating_add(8)); + let mut rest = bytes; + while !rest.is_empty() { + match core::str::from_utf8(rest) { + Ok(valid) => { + out.push_str(valid); + break; + } + Err(err) => { + let good = err.valid_up_to(); + let (head, bad) = rest.split_at(good); + // `head` is valid UTF-8 by construction (`valid_up_to`). + out.push_str(core::str::from_utf8(head).unwrap_or("")); + if let [0xED, b1 @ 0xA0..=0xBF, b2 @ 0x80..=0xBF, tail @ ..] = bad { + match render { + MalformedRender::Lossy => out.push('\u{FFFD}'), + MalformedRender::Normalized => { + // Decode the WTF-8 lone surrogate back to its 16-bit + // code unit (lead nibble is the fixed 0xD of 0xED), + // then emit ``. `wrapping_sh*` + bit-ops + + // `char::from_digit` are all panic-free here (values + // masked to ≤ 0xF, base 16). + let unit: u32 = 0xD000 + | (u32::from(*b1) & 0x3F).wrapping_shl(6) + | (u32::from(*b2) & 0x3F); + out.push_str("'); + } + } + rest = tail; + } else { + let adv = err.error_len().unwrap_or(1).min(bad.len()); + let (_, after) = bad.split_at(adv); + match render { + MalformedRender::Lossy => out.push('\u{FFFD}'), + MalformedRender::Normalized => out.push_str(""), + } + rest = after; + } + } + } + } + out +} + impl CompactRecord { /// Directory flag bit in raw NTFS `FILE_ATTRIBUTE_DIRECTORY`. const DIRECTORY_BIT: u32 = 0x0010; @@ -185,7 +271,39 @@ impl CompactRecord { #[inline] #[must_use] pub fn name_display<'a>(&self, names: &'a [u8]) -> alloc::borrow::Cow<'a, str> { - String::from_utf8_lossy(self.name_bytes(names)) + self.name_display_with(names, MalformedRender::Lossy) + } + + /// Whether the stored name is ill-formed UTF-16 (its WTF-8 bytes are not + /// valid UTF-8). Cheap; used to gate the lossy render and the malformed + /// flag. + #[inline] + #[must_use] + pub fn is_name_malformed(&self, names: &[u8]) -> bool { + core::str::from_utf8(self.name_bytes(names)).is_err() + } + + /// Like [`Self::name_display`] but choosing how an ill-formed name renders: + /// [`MalformedRender::Lossy`] (U+FFFD) or [`MalformedRender::Normalized`] + /// (`` markers, for `--normalize-malformed`). A valid name is a + /// zero-cost `Cow::Borrowed` under either mode; only the rare malformed + /// name allocates, so the well-formed hot path is identical to + /// `name_display`. + #[inline] + #[must_use] + pub fn name_display_with<'a>( + &self, + names: &'a [u8], + render: MalformedRender, + ) -> alloc::borrow::Cow<'a, str> { + let bytes = self.name_bytes(names); + // Well-formed (the overwhelming common case) → zero-cost borrow; an + // ill-formed name renders one marker per offending code unit (U+FFFD or + // ``), matching the reference tool's per-unit replacement. + core::str::from_utf8(bytes).map_or_else( + |_| alloc::borrow::Cow::Owned(render_malformed(bytes, render)), + alloc::borrow::Cow::Borrowed, + ) } /// Get the name's **raw bytes** (WTF-8) from a names blob — the lossless diff --git a/crates/uffs-core/src/compact_tests/malformed.rs b/crates/uffs-core/src/compact_tests/malformed.rs index 2c43f9aa0..417af9d5e 100644 --- a/crates/uffs-core/src/compact_tests/malformed.rs +++ b/crates/uffs-core/src/compact_tests/malformed.rs @@ -13,7 +13,7 @@ use uffs_mft::index::{IndexNameRef, MftIndex, ROOT_FRS, SizeInfo}; use super::{fixture_index, push_name}; -use crate::compact::build_compact_index; +use crate::compact::{MalformedRender, build_compact_index}; // ── WI-4.4: a crooked (surrogate-named) file cannot hide from the search // layer. Build the compact/search index from an MftIndex holding an @@ -191,7 +191,7 @@ fn crooked_dir_segment_is_preserved_in_resolved_path() { let (drive, _, _) = build_compact_index(uffs_mft::platform::DriveLetter::C, &idx); // The crooked directory's lossy display: `evil` + U+FFFD + `.exe`. - let crooked_display = String::from_utf8_lossy(CROOKED_NAME_WTF8); + let crooked_display = "evil\u{FFFD}.exe"; // one U+FFFD per offending code unit (matches C++) assert!( crooked_display.contains('\u{FFFD}'), "fixture must render lossily (got {crooked_display:?})" @@ -278,14 +278,59 @@ fn crooked_leaf_file_is_not_dropped_from_results() { &mut filters, ); - let crooked_display = String::from_utf8_lossy(CROOKED_NAME_WTF8); + let crooked_display = "evil\u{FFFD}.exe"; // one U+FFFD per offending code unit (matches C++) let found = rows .iter() .find(|row| row.malformed) .expect("the crooked-leaf file must be enumerated in search results, not silently dropped"); assert!( - found.path.contains(crooked_display.as_ref()), + found.path.contains(crooked_display), "its path must carry the lossy leaf name, got {:?}", found.path ); } + +#[test] +fn malformed_name_renders_lossy_and_normalized() { + // `evil` + lone HIGH surrogate U+D800 + `.exe`. + let mut idx = fixture_index(); + let crooked = push_name_bytes(&mut idx, CROOKED_NAME_WTF8); + let rec = idx.get_or_create(909.into()); + rec.first_name.name = crooked; + rec.first_name.parent_frs = Into::into(ROOT_FRS); + + let (drive, _, _) = build_compact_index(uffs_mft::platform::DriveLetter::C, &idx); + let crooked_rec = drive + .records + .iter() + .find(|cr| cr.name_bytes(&drive.names) == CROOKED_NAME_WTF8) + .expect("crooked record present"); + + // Default (lossy): the surrogate run collapses to a single U+FFFD. + assert_eq!( + crooked_rec.name_display_with(&drive.names, MalformedRender::Lossy), + "evil\u{FFFD}.exe" + ); + // Normalized: the offending code unit (U+D800) becomes a greppable, + // reversible `` marker; the valid prefix/suffix are preserved. + assert_eq!( + crooked_rec.name_display_with(&drive.names, MalformedRender::Normalized), + "evil.exe" + ); + + // A well-formed name is byte-for-byte identical (and borrowed) under both + // modes — the markers never touch valid names. + let docs = drive + .records + .iter() + .find(|cr| cr.name_bytes(&drive.names) == b"Docs") + .expect("Docs present"); + assert_eq!( + docs.name_display_with(&drive.names, MalformedRender::Lossy), + "Docs" + ); + assert_eq!( + docs.name_display_with(&drive.names, MalformedRender::Normalized), + "Docs" + ); +} From b58e1faf30f943db3d4ac19438164f9eb590e186 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Sun, 28 Jun 2026 07:48:39 -0700 Subject: [PATCH 3/3] feat(cli): --normalize-malformed renders corrupt names as MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thread the malformed-render mode end to end so corrupt (ill-formed UTF-16) names can be surfaced as greppable, reversible `` markers instead of the default `�` — for downstream tooling that needs to spot, parse, or round-trip corrupt entries by their path string. (Filtering corrupt entries stays on the existing malformed filter; this is display-only.) * uffs-core: `SearchFilters.normalize_malformed` + `malformed_render()`; a `MalformedRender` param threaded through `resolve_path*` so the resolved path and the name column pick the render mode. Hot path unchanged — valid names still borrow, only malformed allocate. * uffs-client: `SearchParams.normalize_malformed` (serde `#[serde(default)]`, backward-compatible wire) + the `--normalize-malformed` arm in `from_cli_args` (the parser uffs-cli delegates to). * uffs-daemon: maps the request flag onto `SearchFilters` for the search output path; info / aggregate output stays on the default lossy render. Also drops the now-redundant `name` arg from `build_row_cached` (it is exactly `rec.name(...)`, derivable from the `rec` it already receives). Tests: end-to-end normalized path through the resolver (`C:\evil.exe\report.txt`), wire round-trip + omitted-field backward-compat, and CLI flag parsing. Co-Authored-By: Claude Opus 4.8 --- crates/uffs-client/src/protocol/cli_args.rs | 3 ++ crates/uffs-client/src/protocol/mod.rs | 6 +++ crates/uffs-client/src/protocol/tests.rs | 39 +++++++++++++++++++ .../uffs-core/src/compact_tests/malformed.rs | 11 +++++- crates/uffs-core/src/search/filters/mod.rs | 23 +++++++++++ crates/uffs-core/src/search/query/mod.rs | 24 ++++++++++-- .../src/search/query/numeric_top_n.rs | 13 +++++-- .../src/search/query/path_only_top_n.rs | 4 ++ .../src/search/query/path_sorted_top_n.rs | 5 ++- .../src/search/query/prefix_search.rs | 7 +++- .../uffs-core/src/search/query/row_resolve.rs | 11 ++++-- crates/uffs-core/src/search/tree.rs | 26 ++++++++----- crates/uffs-daemon/src/index/aggregation.rs | 8 +++- crates/uffs-daemon/src/index/info.rs | 2 + crates/uffs-daemon/src/index/search.rs | 3 ++ 15 files changed, 162 insertions(+), 23 deletions(-) diff --git a/crates/uffs-client/src/protocol/cli_args.rs b/crates/uffs-client/src/protocol/cli_args.rs index f1dfec88c..d153dd17a 100644 --- a/crates/uffs-client/src/protocol/cli_args.rs +++ b/crates/uffs-client/src/protocol/cli_args.rs @@ -51,6 +51,7 @@ impl SearchParams { "--dirs-only" => raw.dirs_only = true, "--hide-system" => raw.hide_system = true, "--hide-ads" => raw.hide_ads = true, + "--normalize-malformed" => raw.normalize_malformed = true, // WI-4.4 forensic filters: find ill-formed (non-UTF-8) names. "--malformed" => raw.malformed = Some(true), "--well-formed" => raw.malformed = Some(false), @@ -306,6 +307,7 @@ struct RawCliArgs { dirs_only: bool, hide_system: bool, hide_ads: bool, + normalize_malformed: bool, /// WI-4.4: `Some(true)` from `--malformed`, `Some(false)` from /// `--well-formed`, `None` if neither (no filter). malformed: Option, @@ -719,6 +721,7 @@ impl RawCliArgs { // Misc hide_system: self.hide_system, hide_ads: self.hide_ads, + normalize_malformed: self.normalize_malformed, // Profiling profile: self.profile || self.benchmark, aggregations, diff --git a/crates/uffs-client/src/protocol/mod.rs b/crates/uffs-client/src/protocol/mod.rs index 64dbcab61..e8a26cf54 100644 --- a/crates/uffs-client/src/protocol/mod.rs +++ b/crates/uffs-client/src/protocol/mod.rs @@ -428,6 +428,11 @@ pub struct SearchParams { /// Hide NTFS Alternate Data Streams from results. #[serde(default)] pub hide_ads: bool, + /// Render ill-formed (surrogate-bearing) names with greppable `` + /// markers instead of the default U+FFFD (`�`). Display-only — does not + /// filter results (use the malformed filter for that). + #[serde(default)] + pub normalize_malformed: bool, // ── Profiling ────────────────────────────────────────────────── /// Request detailed timing breakdown from the daemon. @@ -598,6 +603,7 @@ impl Default for SearchParams { malformed_path: None, hide_system: false, hide_ads: false, + normalize_malformed: false, profile: false, aggregations: vec![], include_rows: true, diff --git a/crates/uffs-client/src/protocol/tests.rs b/crates/uffs-client/src/protocol/tests.rs index 1e9b6a0a9..d40106240 100644 --- a/crates/uffs-client/src/protocol/tests.rs +++ b/crates/uffs-client/src/protocol/tests.rs @@ -69,6 +69,45 @@ fn search_params_round_trip() { assert_eq!(parsed.response_mode, Some(SearchResponseMode::Json)); } +/// `normalize_malformed` round-trips, and an older payload that omits it +/// deserializes as `false` (backward-compatible via `#[serde(default)]`). +#[test] +fn search_params_normalize_malformed_round_trip_and_default() { + let params = SearchParams { + pattern: "*".to_owned(), + normalize_malformed: true, + ..Default::default() + }; + let json = serde_json::to_value(¶ms).expect("serialize"); + let parsed: SearchParams = serde_json::from_value(json).expect("deserialize"); + assert!( + parsed.normalize_malformed, + "the flag must survive the JSON-RPC round trip" + ); + + // A payload from an older client that never knew the field. + let legacy: SearchParams = + serde_json::from_value(serde_json::json!({ "pattern": "*" })).expect("legacy deserialize"); + assert!( + !legacy.normalize_malformed, + "omitted field defaults off (wire backward-compat)" + ); +} + +/// The CLI surface: `--normalize-malformed` sets the param; absent → off. +#[test] +fn from_cli_args_normalize_malformed_flag() { + let on = SearchParams::from_cli_args(&["*.tmp".to_owned(), "--normalize-malformed".to_owned()]) + .expect("parse with flag"); + assert!( + on.normalize_malformed, + "--normalize-malformed must set the flag" + ); + + let off = SearchParams::from_cli_args(&["*.tmp".to_owned()]).expect("parse without flag"); + assert!(!off.normalize_malformed, "absent flag defaults off"); +} + /// Canonical helpers preserve legacy single-flag sort semantics. /// /// First field: ascending by default (no `--sort-desc`). diff --git a/crates/uffs-core/src/compact_tests/malformed.rs b/crates/uffs-core/src/compact_tests/malformed.rs index 417af9d5e..3095e5d4a 100644 --- a/crates/uffs-core/src/compact_tests/malformed.rs +++ b/crates/uffs-core/src/compact_tests/malformed.rs @@ -222,7 +222,7 @@ fn crooked_dir_segment_is_preserved_in_resolved_path() { // Both resolvers must keep the child UNDER the crooked dir (not at root). let expected = format!("C:\\{crooked_display}\\report.txt"); - let plain = tree::resolve_path(&drive, child_idx, "C:"); + let plain = tree::resolve_path(&drive, child_idx, "C:", MalformedRender::Lossy); assert_eq!( plain, expected, "plain resolver must preserve the crooked segment" @@ -232,6 +232,14 @@ fn crooked_dir_segment_is_preserved_in_resolved_path() { "regression: the crooked segment must not collapse to the parent" ); + // End-to-end: the `--normalize-malformed` render mode threads through the + // resolver, marking the bad segment inline. + let normalized = tree::resolve_path(&drive, child_idx, "C:", MalformedRender::Normalized); + assert_eq!( + normalized, "C:\\evil.exe\\report.txt", + "normalized render must surface the marker in the path" + ); + let mut dir_cache = tree::DirCache::default(); let mut mal_cache = tree::MalformedCache::default(); let (mpath, malformed) = tree::resolve_path_cached_with_malformed( @@ -240,6 +248,7 @@ fn crooked_dir_segment_is_preserved_in_resolved_path() { "C:", &mut dir_cache, &mut mal_cache, + MalformedRender::Lossy, ); assert_eq!( mpath, expected, diff --git a/crates/uffs-core/src/search/filters/mod.rs b/crates/uffs-core/src/search/filters/mod.rs index 5a93ec87f..6ea1c7dd6 100644 --- a/crates/uffs-core/src/search/filters/mod.rs +++ b/crates/uffs-core/src/search/filters/mod.rs @@ -149,6 +149,26 @@ pub struct SearchFilters { /// lossless bytes), never the lossy `&str` view (which is always valid /// UTF-8 and would match nothing). pub malformed: Option, + + /// Render ill-formed names with greppable `` markers instead of + /// the default U+FFFD (`�`). A display-only option (not a filter): it + /// selects [`crate::compact::MalformedRender`] for the resolved path + name + /// column so downstream tooling can spot/round-trip corrupt entries. + /// `false` = default lossy rendering (matches the reference C++ tool). + pub normalize_malformed: bool, +} + +impl SearchFilters { + /// The [`crate::compact::MalformedRender`] mode implied by + /// [`Self::normalize_malformed`]. + #[must_use] + pub const fn malformed_render(&self) -> crate::compact::MalformedRender { + if self.normalize_malformed { + crate::compact::MalformedRender::Normalized + } else { + crate::compact::MalformedRender::Lossy + } + } } /// Raw parameter inputs for constructing [`SearchFilters`]. @@ -380,6 +400,9 @@ impl SearchFilters { // predicate compiler (it is not a legacy positional param), so the // param-based constructor leaves it disabled. malformed: None, + // Display-only; the daemon sets it from the request's + // `normalize_malformed` flag, so it defaults off here. + normalize_malformed: false, } } diff --git a/crates/uffs-core/src/search/query/mod.rs b/crates/uffs-core/src/search/query/mod.rs index b5a9d6dc6..0547c57a9 100644 --- a/crates/uffs-core/src/search/query/mod.rs +++ b/crates/uffs-core/src/search/query/mod.rs @@ -25,7 +25,7 @@ use row_resolve::indices_to_rows; use super::backend::{DisplayRow, FilterMode, PhaseTimings}; use super::field::FieldId; use super::filters::SearchFilters; -use crate::compact::{CompactRecord, DriveCompactIndex}; +use crate::compact::{CompactRecord, DriveCompactIndex, MalformedRender}; use crate::search::tree; /// Whether cache profiling is enabled (`UFFS_CACHE_PROFILE` env var). @@ -210,7 +210,12 @@ pub(crate) fn search_compact_drive_regex( let match_count = match_indices.len(); let t_resolve = std::time::Instant::now(); - let rows = indices_to_rows(drive, &match_indices, volume_prefix); + let rows = indices_to_rows( + drive, + &match_indices, + volume_prefix, + filters.malformed_render(), + ); let resolve_ms = t_resolve.elapsed().as_millis(); if profile { @@ -449,7 +454,12 @@ pub(crate) fn search_compact_drive( } let t_resolve = std::time::Instant::now(); - let rows = indices_to_rows(drive, &match_indices, volume_prefix); + let rows = indices_to_rows( + drive, + &match_indices, + volume_prefix, + filters.malformed_render(), + ); let resolve_ms = t_resolve.elapsed().as_millis(); if profile { @@ -512,6 +522,7 @@ pub(crate) fn search_compact_drive_tree( ) -> Vec { let mut vp_buf = [0_u8; 4]; let volume_prefix = stack_volume_prefix(&mut vp_buf, drive.letter); + let render = filters.malformed_render(); let profile = *CACHE_PROFILE; // Resolve the extension filter for THIS drive. When an `--ext` filter is @@ -555,6 +566,7 @@ pub(crate) fn search_compact_drive_tree( volume_prefix, &mut dir_cache, &mut mal_cache, + render, ); let forensics = row_forensics(rec, &drive.names, path_malformed); Some(make_display_row( @@ -634,10 +646,10 @@ pub(super) fn build_row_cached( drive: &DriveCompactIndex, rec_idx: u32, rec: &CompactRecord, - name: &str, volume_prefix: &str, dir_cache: &mut tree::DirCache, mal_cache: &mut tree::MalformedCache, + render: MalformedRender, ) -> DisplayRow { let (path, path_malformed) = tree::resolve_path_cached_with_malformed( drive, @@ -645,8 +657,12 @@ pub(super) fn build_row_cached( volume_prefix, dir_cache, mal_cache, + render, ); let forensics = row_forensics(rec, &drive.names, path_malformed); + // The leaf name is derived here (it is exactly `rec.name(...)`) so callers + // don't thread it in — keeps the resolve→row arg list lean. + let name = rec.name(&drive.names); make_display_row(rec_idx, drive.letter, rec, name, path, forensics) } diff --git a/crates/uffs-core/src/search/query/numeric_top_n.rs b/crates/uffs-core/src/search/query/numeric_top_n.rs index b18a8a525..76ad3a476 100644 --- a/crates/uffs-core/src/search/query/numeric_top_n.rs +++ b/crates/uffs-core/src/search/query/numeric_top_n.rs @@ -13,7 +13,7 @@ use super::super::filters::SearchFilters; use super::super::tree; use super::numeric_sort_key::extract_sort_key; use super::{HeapEntry, heap_push_capped, make_display_row, row_forensics, stack_volume_prefix}; -use crate::compact::{CompactRecord, DriveCompactIndex}; +use crate::compact::{CompactRecord, DriveCompactIndex, MalformedRender}; /// Target chunk size for parallel path resolution inside /// [`collect_global_top_n_numeric`]. @@ -408,6 +408,7 @@ struct ResolveStats { fn resolve_chunk>( drives: &[D], chunk: &[(u16, u32, i64)], + render: MalformedRender, ) -> ResolveStats { let mut local_caches: std::collections::HashMap = std::collections::HashMap::new(); @@ -450,6 +451,7 @@ fn resolve_chunk>( volume_prefix, cache, mal_cache, + render, ); resolve_fn_ns += t_resolve.elapsed().as_nanos(); let t_build = std::time::Instant::now(); @@ -487,10 +489,11 @@ fn resolve_chunk>( fn resolve_candidates_to_rows + Sync>( drives: &[D], candidates: &[(u16, u32, i64)], + render: MalformedRender, ) -> ResolveStats { candidates .par_chunks(RESOLVE_CHUNK_SIZE) - .map(|chunk| resolve_chunk(drives, chunk)) + .map(|chunk| resolve_chunk(drives, chunk, render)) .reduce( || ResolveStats { rows: Vec::new(), @@ -681,7 +684,11 @@ pub(super) fn collect_global_top_n_numeric + Sync>( // per-chunk cache warm. Measured 4× speedup on 168 K candidates // across 8 workers (63 ms sequential → 17 ms parallel). let t_path_resolve = std::time::Instant::now(); - let stats = resolve_candidates_to_rows(drives, &sorted_candidates); + let stats = resolve_candidates_to_rows( + drives, + &sorted_candidates, + search_filters.malformed_render(), + ); let path_resolve_ms = u64::try_from(t_path_resolve.elapsed().as_millis()).unwrap_or(u64::MAX); let mut rows = stats.rows; diff --git a/crates/uffs-core/src/search/query/path_only_top_n.rs b/crates/uffs-core/src/search/query/path_only_top_n.rs index 7a509e181..8fd2cf27b 100644 --- a/crates/uffs-core/src/search/query/path_only_top_n.rs +++ b/crates/uffs-core/src/search/query/path_only_top_n.rs @@ -430,6 +430,7 @@ fn emit_if_passes( mal_cache: &mut MalformedCache, output: &mut Vec, ) -> bool { + let render = search_filters.malformed_render(); let Some(rec) = drive.records.get(idx as usize) else { return false; }; @@ -446,6 +447,7 @@ fn emit_if_passes( volume_prefix, dir_cache, mal_cache, + render, ); let forensics = row_forensics(rec, &drive.names, path_malformed); let row = make_display_row(idx, drive.letter, rec, name, path, forensics); @@ -509,6 +511,7 @@ fn collect_path_only_via_ext_index + Sync>( // disqualify the fast path; see `SearchFilters::is_ext_only`). let hide_system = search_filters.hide_system; let hide_ads = search_filters.hide_ads; + let render = search_filters.malformed_render(); // Collect (drive_idx, rec_idx) pairs for every candidate that // survives the per-record predicates. We do NOT bound this @@ -615,6 +618,7 @@ fn collect_path_only_via_ext_index + Sync>( volume_prefix, cache, mal_cache, + render, ); let forensics = row_forensics(rec, &drive.names, path_malformed); local_rows.push(make_display_row( diff --git a/crates/uffs-core/src/search/query/path_sorted_top_n.rs b/crates/uffs-core/src/search/query/path_sorted_top_n.rs index 454d21585..ecc38e9a6 100644 --- a/crates/uffs-core/src/search/query/path_sorted_top_n.rs +++ b/crates/uffs-core/src/search/query/path_sorted_top_n.rs @@ -111,6 +111,7 @@ fn walk_tree_path_sorted>( filter_mode: FilterMode, search_filters: &SearchFilters, ) -> Vec { + let render = search_filters.malformed_render(); let mut path_results: Vec = Vec::new(); let mut drive_order: Vec = (0..drives.len()).collect(); drive_order.sort_unstable_by(|&idx_a, &idx_b| { @@ -188,6 +189,7 @@ fn walk_tree_path_sorted>( volume_prefix, &mut dir_cache, &mut mal_cache, + render, ); let forensics = row_forensics(rec, &drive.names, path_malformed); let row = make_display_row(idx, drive.letter, rec, name, path, forensics); @@ -238,6 +240,7 @@ fn collect_path_via_ext_index + Sync>( ) -> Vec { let hide_system = search_filters.hide_system; let hide_ads = search_filters.hide_ads; + let render = search_filters.malformed_render(); // ── Scan phase: collect (drive_idx, rec_idx) candidates ─────── // @@ -338,10 +341,10 @@ fn collect_path_via_ext_index + Sync>( drive, rec_idx, rec, - name, volume_prefix, cache, mal_cache, + render, )); } local_rows diff --git a/crates/uffs-core/src/search/query/prefix_search.rs b/crates/uffs-core/src/search/query/prefix_search.rs index fca5cb543..7de81020c 100644 --- a/crates/uffs-core/src/search/query/prefix_search.rs +++ b/crates/uffs-core/src/search/query/prefix_search.rs @@ -95,7 +95,12 @@ pub(crate) fn search_compact_drive_prefix( let match_count = match_indices.len(); let t_resolve = std::time::Instant::now(); - let rows = indices_to_rows(drive, &match_indices, volume_prefix); + let rows = indices_to_rows( + drive, + &match_indices, + volume_prefix, + filters.malformed_render(), + ); let resolve_ms = t_resolve.elapsed().as_millis(); if profile { diff --git a/crates/uffs-core/src/search/query/row_resolve.rs b/crates/uffs-core/src/search/query/row_resolve.rs index d40e25405..1a43695da 100644 --- a/crates/uffs-core/src/search/query/row_resolve.rs +++ b/crates/uffs-core/src/search/query/row_resolve.rs @@ -11,7 +11,7 @@ use rayon::prelude::*; use super::{DisplayRow, make_display_row, row_forensics}; -use crate::compact::DriveCompactIndex; +use crate::compact::{DriveCompactIndex, MalformedRender}; use crate::search::tree; /// Chunk size for parallel path resolution. At ~370 ns per candidate, @@ -33,12 +33,13 @@ pub(crate) fn indices_to_rows( drive: &DriveCompactIndex, indices: &[u32], volume_prefix: &str, + render: MalformedRender, ) -> Vec { // Parallel overhead is only worth it above a chunk's worth of candidates. if indices.len() < RESOLVE_CHUNK_SIZE { - return indices_to_rows_sequential(drive, indices, volume_prefix); + return indices_to_rows_sequential(drive, indices, volume_prefix, render); } - indices_to_rows_parallel(drive, indices, volume_prefix) + indices_to_rows_parallel(drive, indices, volume_prefix, render) } /// Sequential path resolution for small candidate sets (`< @@ -47,6 +48,7 @@ fn indices_to_rows_sequential( drive: &DriveCompactIndex, indices: &[u32], volume_prefix: &str, + render: MalformedRender, ) -> Vec { let mut dir_cache = tree::dir_cache_with_capacity(256); let mut mal_cache = tree::malformed_cache_with_capacity(256); @@ -64,6 +66,7 @@ fn indices_to_rows_sequential( volume_prefix, &mut dir_cache, &mut mal_cache, + render, ); let forensics = row_forensics(rec, &drive.names, path_malformed); Some(make_display_row( @@ -89,6 +92,7 @@ fn indices_to_rows_parallel( drive: &DriveCompactIndex, indices: &[u32], volume_prefix: &str, + render: MalformedRender, ) -> Vec { indices .par_chunks(RESOLVE_CHUNK_SIZE) @@ -111,6 +115,7 @@ fn indices_to_rows_parallel( volume_prefix, &mut dir_cache, &mut mal_cache, + render, ); let forensics = row_forensics(rec, &drive.names, path_malformed); local_rows.push(make_display_row( diff --git a/crates/uffs-core/src/search/tree.rs b/crates/uffs-core/src/search/tree.rs index 0a4a10f1f..334733f79 100644 --- a/crates/uffs-core/src/search/tree.rs +++ b/crates/uffs-core/src/search/tree.rs @@ -10,7 +10,7 @@ use rustc_hash::{FxBuildHasher, FxHashMap}; -use crate::compact::DriveCompactIndex; +use crate::compact::{DriveCompactIndex, MalformedRender}; /// Directory path cache for `resolve_path_cached`. /// @@ -40,8 +40,13 @@ pub(crate) fn dir_cache_with_capacity(capacity: usize) -> DirCache { /// /// Returns path like `C:\Users\Photos\beach.jpg`. #[must_use] -pub fn resolve_path(drive: &DriveCompactIndex, record_idx: usize, volume_prefix: &str) -> String { - resolve_path_inner(drive, record_idx, volume_prefix, None) +pub fn resolve_path( + drive: &DriveCompactIndex, + record_idx: usize, + volume_prefix: &str, + render: MalformedRender, +) -> String { + resolve_path_inner(drive, record_idx, volume_prefix, None, render) } /// Resolve a record's full path with directory caching. @@ -59,8 +64,9 @@ pub fn resolve_path_cached( record_idx: usize, volume_prefix: &str, dir_cache: &mut DirCache, + render: MalformedRender, ) -> String { - resolve_path_inner(drive, record_idx, volume_prefix, Some(dir_cache)) + resolve_path_inner(drive, record_idx, volume_prefix, Some(dir_cache), render) } /// Cache of "is this directory's resolved path ill-formed?", keyed by record @@ -96,6 +102,7 @@ pub fn resolve_path_cached_with_malformed( volume_prefix: &str, dir_cache: &mut DirCache, mal_cache: &mut MalformedCache, + render: MalformedRender, ) -> (String, bool) { let mut chain: Vec = Vec::with_capacity(8); let mut current_idx = record_idx; @@ -156,7 +163,7 @@ pub fn resolve_path_cached_with_malformed( // `name_display()` (lossy, U+FFFD for ill-formed) is what is // pushed into the displayed path; reserve for its length, which // may differ from `bytes.len()`. - Some(1 + rec.name_display(&drive.names).len()) + Some(1 + rec.name_display_with(&drive.names, render).len()) } }) .sum(); @@ -183,7 +190,7 @@ pub fn resolve_path_cached_with_malformed( // U+FFFD (`�`) rather than an empty segment, so the malformed directory // keeps its place in the path instead of collapsing everything beneath // it onto its parent. - let name = rec.name_display(&drive.names); + let name = rec.name_display_with(&drive.names, render); if !path.ends_with('\\') && !path.is_empty() { path.push('\\'); @@ -212,6 +219,7 @@ fn resolve_path_inner( record_idx: usize, volume_prefix: &str, dir_cache: Option<&mut DirCache>, + render: MalformedRender, ) -> String { let mut chain: Vec = Vec::with_capacity(8); let mut current_idx = record_idx; @@ -266,7 +274,7 @@ fn resolve_path_inner( if bytes.is_empty() || bytes == b"." { None } else { - Some(1 + rec.name_display(&drive.names).len()) + Some(1 + rec.name_display_with(&drive.names, render).len()) } }) .sum(); @@ -277,7 +285,7 @@ fn resolve_path_inner( if let Some(rec) = drive.records.get(idx) { let bytes = rec.name_bytes(&drive.names); if !bytes.is_empty() && bytes != b"." { - let name = rec.name_display(&drive.names); + let name = rec.name_display_with(&drive.names, render); if !path.ends_with('\\') && !path.is_empty() { path.push('\\'); } @@ -297,7 +305,7 @@ fn resolve_path_inner( if bytes.is_empty() || bytes == b"." { continue; } - let name = rec.name_display(&drive.names); + let name = rec.name_display_with(&drive.names, render); if !dir_path.ends_with('\\') && !dir_path.is_empty() { dir_path.push('\\'); } diff --git a/crates/uffs-daemon/src/index/aggregation.rs b/crates/uffs-daemon/src/index/aggregation.rs index a0435495c..6cbff194b 100644 --- a/crates/uffs-daemon/src/index/aggregation.rs +++ b/crates/uffs-daemon/src/index/aggregation.rs @@ -51,6 +51,7 @@ impl DaemonFileReader<'_> { drive, record_idx, &volume_prefix, + uffs_core::compact::MalformedRender::Lossy, )) } } @@ -147,7 +148,12 @@ fn materialize_dup_members( let record = drive.records.get(rec_idx)?; let name = record.name(&drive.names).to_owned(); let volume_prefix = format!("{}:\\", drive.letter); - let path = uffs_core::search::tree::resolve_path(drive, rec_idx, &volume_prefix); + let path = uffs_core::search::tree::resolve_path( + drive, + rec_idx, + &volume_prefix, + uffs_core::compact::MalformedRender::Lossy, + ); let mut fields = std::collections::HashMap::new(); fields.insert("name".to_owned(), name); diff --git a/crates/uffs-daemon/src/index/info.rs b/crates/uffs-daemon/src/index/info.rs index 861e8433d..7557aec24 100644 --- a/crates/uffs-daemon/src/index/info.rs +++ b/crates/uffs-daemon/src/index/info.rs @@ -105,6 +105,7 @@ impl IndexManager { drive, uffs_mft::u32_as_usize(root_idx), &volume_prefix, + uffs_core::compact::MalformedRender::Lossy, ); return Some(Self::build_info_json(drive, rec, &resolved)); } @@ -125,6 +126,7 @@ impl IndexManager { drive, uffs_mft::u32_as_usize(child_idx), &volume_prefix, + uffs_core::compact::MalformedRender::Lossy, ); return Some(Self::build_info_json(drive, rec, &resolved)); } diff --git a/crates/uffs-daemon/src/index/search.rs b/crates/uffs-daemon/src/index/search.rs index dd31a646e..d7b795ff1 100644 --- a/crates/uffs-daemon/src/index/search.rs +++ b/crates/uffs-daemon/src/index/search.rs @@ -141,6 +141,9 @@ impl IndexManager { max_tree_allocated: ep.max_tree_allocated, allowed_months: &ep.allowed_months, }); + // Display-only: select the malformed-name render mode for resolved + // paths + the name column (`--normalize-malformed`). + filters.normalize_malformed = ep.normalize_malformed; // Overlay canonical predicates that can be compiled into the hot // path (size / descendant bounds).