From 80b8d36fe244eba8453b001217a86f8d4678628c Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 20 Jul 2026 07:18:11 -0700 Subject: [PATCH 1/7] fix(mft): populate usn/security_id/owner_id on the two production parsers StandardInfo already declares usn/security_id/owner_id and StandardInfo:: from_extended already copies them through, but the two record-parsing pipelines that actually run in production never called it: - io/parser/unified.rs process_record (the default MftReadMode::Auto -> SlidingIocpInline bulk-load path) unconditionally read only the 36-byte NTFS 1.2 StandardInformation and built StandardInfo via from_raw_ntfs_flags, whose own doc says the caller must set these three fields separately -- it never did. - parse/direct_index.rs parse_record_to_index (the live USN-journal incremental-update path, called from usn/windows.rs) had the identical bug: read only the 36-byte struct and dropped usn/security_id/owner_id before reaching StandardInfo::from_extended. Only the legacy ParsedRecord pipeline (parse/attribute_helpers.rs parse_standard_info_full, reachable via non-default read modes / the DataFrame export path) was already correct -- it branches on value_length to read the 72-byte NTFS 3.0+ StandardInformationExtended form when present. Fix: bump parse_standard_info_full to pub(crate) and call it from both production parsers instead of duplicating (buggy) inline logic. Net deletion of duplicated code in favor of the one already-tested implementation. These fields have no bearing on ordinary search/filter/display -- only forensic/security-auditing consumers (timestomping detection via STD_INFO vs FILE_NAME divergence, ACL/USN correlation) read them -- so this was a silent, zero-functional-impact gap until now, not a live bug. Fixing it is free: the fields already exist in StandardInfo's on-disk layout, the bytes are already read off disk in the same pass, and the extra cost is one well-predicted branch. Out of scope for this commit (flagged, not touched): - io/parser/index.rs's parse_record_to_index has zero production call sites (only its own test module calls it) -- looks like dead code, left alone to avoid fixing unreachable paths. - $FILE_NAME's own per-name timestamps: the modern MftIndex per-name storage (IndexNameRef/first_name/hard-link chain) has no timestamp fields at all, unlike the legacy FileRecord/NameInfo struct. Adding them means real, permanent per-hard-link storage growth -- a different cost/benefit decision than this free fix. Added a regression test exercising both production parsers end-to-end against a synthetic 72-byte StandardInformationExtended record. --- crates/uffs-mft/src/io/parser/mod.rs | 122 ++++++++++++++++++ crates/uffs-mft/src/io/parser/unified.rs | 32 ++--- crates/uffs-mft/src/parse.rs | 9 +- .../uffs-mft/src/parse/attribute_helpers.rs | 6 +- crates/uffs-mft/src/parse/direct_index.rs | 34 ++--- 5 files changed, 156 insertions(+), 47 deletions(-) diff --git a/crates/uffs-mft/src/io/parser/mod.rs b/crates/uffs-mft/src/io/parser/mod.rs index a4d61af13..ddb952cbd 100644 --- a/crates/uffs-mft/src/io/parser/mod.rs +++ b/crates/uffs-mft/src/io/parser/mod.rs @@ -73,6 +73,128 @@ mod tests { ); } + /// Regression pin: both production `$STANDARD_INFORMATION` parsers — + /// `process_record` (the default bulk-load pipeline) and + /// `crate::parse::parse_record_to_index` (the live USN-journal + /// incremental-update pipeline, wired from `usn::windows`) — must + /// recognize the NTFS 3.0+ 72-byte `StandardInformationExtended` form + /// and populate `usn`/`security_id`/`owner_id`, not just the 4 + /// timestamps. Before this fix both silently treated every record as + /// NTFS 1.2 (36 bytes) and left those three fields at zero. + #[test] + fn standard_information_extended_fields_reach_both_production_parsers() { + let creation_time = 1_i64; + let modification_time = 2_i64; + let mft_change_time = 3_i64; + let access_time = 4_i64; + let file_attributes = 0x20_u32; // FILE_ATTRIBUTE_ARCHIVE + let owner_id = 44_u32; + let security_id = 55_u32; + let usn = 66_u64; + + // 72-byte StandardInformationExtended payload, field order per + // `ntfs::metadata::StandardInformationExtended`. + let mut payload = Vec::new(); + payload.extend_from_slice(&creation_time.to_le_bytes()); + payload.extend_from_slice(&modification_time.to_le_bytes()); + payload.extend_from_slice(&mft_change_time.to_le_bytes()); + payload.extend_from_slice(&access_time.to_le_bytes()); + payload.extend_from_slice(&file_attributes.to_le_bytes()); + payload.extend_from_slice(&0_u32.to_le_bytes()); // max_versions + payload.extend_from_slice(&0_u32.to_le_bytes()); // version_number + payload.extend_from_slice(&0_u32.to_le_bytes()); // class_id + payload.extend_from_slice(&owner_id.to_le_bytes()); + payload.extend_from_slice(&security_id.to_le_bytes()); + payload.extend_from_slice(&0_u64.to_le_bytes()); // quota_charged + payload.extend_from_slice(&usn.to_le_bytes()); + assert_eq!( + payload.len(), + 72, + "test fixture must match the real on-disk layout" + ); + + // Resident attribute: 16-byte AttributeRecordHeader + 4-byte + // value_length + 2-byte value_offset + 2-byte resident-flags/padding + // = 24-byte prefix, then the 72-byte payload at value_offset = 24. + let std_info_total_len = u32::try_from(24 + payload.len()).expect("fits in u32"); + + // A minimal $FILE_NAME attribute: `direct_index::parse_record_to_index` + // only returns `true` once a record has a name (real MFT records + // always do). 66-byte fixed `FileNameAttribute` (per + // `ntfs::metadata::FileNameAttribute`'s field order) + a 1-char name. + let mut fn_payload = Vec::new(); + fn_payload.extend_from_slice(&0_u64.to_le_bytes()); // parent_directory + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // creation_time + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // modification_time + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // mft_change_time + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // access_time + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // allocated_size + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // data_size + fn_payload.extend_from_slice(&0_u32.to_le_bytes()); // file_attributes + fn_payload.extend_from_slice(&0_u16.to_le_bytes()); // packed_ea_size + fn_payload.extend_from_slice(&0_u16.to_le_bytes()); // reserved + fn_payload.push(1); // file_name_length = 1 char + fn_payload.push(1); // namespace = Win32 (2 = DOS-only would be skipped) + fn_payload.extend_from_slice(&0x0061_u16.to_le_bytes()); // "a" + assert_eq!( + fn_payload.len(), + 68, + "66-byte FileNameAttribute + 1 UTF-16 char" + ); + let file_name_total_len = u32::try_from(24 + fn_payload.len()).expect("fits in u32"); + + let mut record = RecordBuilder::new(56) + .attr(0x10, std_info_total_len, 0, 0, 0) + .raw(&72_u32.to_le_bytes()) // value_length (signals the extended form) + .raw(&24_u16.to_le_bytes()) // value_offset + .raw(&[0_u8; 2]) // resident flags + reserved + .raw(&payload) + .attr(0x30, file_name_total_len, 0, 0, 0) + .raw(&u32::try_from(fn_payload.len()).expect("fits in u32").to_le_bytes()) // value_length + .raw(&24_u16.to_le_bytes()) // value_offset + .raw(&[0_u8; 2]) // resident flags + reserved + .raw(&fn_payload) + .build(); + + // `RecordBuilder` zeroes `bytes_in_use` (header offset 24..28); patch + // it to the real length so the attribute loop actually runs. + let total_len = u32::try_from(record.len()).expect("fits in u32"); + record + .get_mut(24..28) + .expect("record is well over 28 bytes long") + .copy_from_slice(&total_len.to_le_bytes()); + + // Path 1: process_record — the default bulk-load pipeline. + let mut unified_index = MftIndex::new(crate::platform::DriveLetter::C); + let mut name_buf = String::new(); + process_record(&record, 42, &mut unified_index, &mut name_buf); + let unified_rec = unified_index + .find(crate::frs::Frs::new(42)) + .expect("process_record must create the base record"); + assert_eq!(unified_rec.stdinfo.created, creation_time); + assert_eq!(unified_rec.stdinfo.usn, usn); + assert_eq!(unified_rec.stdinfo.security_id, security_id); + assert_eq!(unified_rec.stdinfo.owner_id, owner_id); + + // Path 2: crate::parse::parse_record_to_index — the live + // USN-journal incremental-update pipeline (direct_index.rs). + // Fully qualified: this module's own `parse_record_to_index` import + // (above) is the unrelated, production-dead `io::parser::index` copy. + let mut direct_index = MftIndex::new(crate::platform::DriveLetter::C); + assert!(crate::parse::parse_record_to_index( + &record, + 42, + &mut direct_index + )); + let direct_rec = direct_index + .find(crate::frs::Frs::new(42)) + .expect("parse_record_to_index must create the base record"); + assert_eq!(direct_rec.stdinfo.created, creation_time); + assert_eq!(direct_rec.stdinfo.usn, usn); + assert_eq!(direct_rec.stdinfo.security_id, security_id); + assert_eq!(direct_rec.stdinfo.owner_id, owner_id); + } + // ── WI-5.2 panic-resistance corpus ────────────────────────────── // // The daemon builds with `panic = "abort"`: a single parser panic on a diff --git a/crates/uffs-mft/src/io/parser/unified.rs b/crates/uffs-mft/src/io/parser/unified.rs index 8718ecca0..464c3865f 100644 --- a/crates/uffs-mft/src/io/parser/unified.rs +++ b/crates/uffs-mft/src/io/parser/unified.rs @@ -29,7 +29,7 @@ use crate::index::{ }; use crate::ntfs::{ AttributeRecordHeader, AttributeType, FileNameAttribute, FileRecordSegmentHeader, - StandardInformation, file_reference_to_frs, + file_reference_to_frs, }; /// Decode a UTF-16LE byte slice into `out`, replacing unpaired surrogates @@ -389,25 +389,19 @@ pub fn process_record(data: &[u8], frs: u64, index: &mut MftIndex, name_buf: &mu // ── $STANDARD_INFORMATION (0x10) ───────────────────────── Some(AttributeType::StandardInformation) => { if attr_header.is_non_resident == 0 { - let vo = usize::from(rd_u16(data, offset.saturating_add(20))); - if let Some(si_off) = offset.checked_add(vo) - && let Some(si_slice) = data.get(si_off..) - && let Ok((si, _)) = StandardInformation::read_from_prefix(si_slice) - { - // Fast path: map raw NTFS flags directly to our - // compact bitmask — skips the intermediate - // ExtendedStandardInfo struct entirely. - let mut info = - crate::index::StandardInfo::from_raw_ntfs_flags(si.file_attributes); - info.created = si.creation_time; - info.modified = si.modification_time; - info.accessed = si.access_time; - info.mft_changed = si.mft_change_time; - if is_directory { - info.set_directory(true); - } - index.records[base_ri].stdinfo = info; + // Shared with the legacy and direct-index pipelines: reads + // the 72-byte NTFS 3.0+ `StandardInformationExtended` form + // (usn/security_id/owner_id) when `value_length` says it's + // present, falling back to the 36-byte NTFS 1.2 form + // otherwise — see `parse::attribute_helpers` for the + // single-source-of-truth rationale. + let mut ext = crate::ntfs::ExtendedStandardInfo::default(); + crate::parse::parse_standard_info_full(data, offset, &mut ext); + let mut info = crate::index::StandardInfo::from_extended(&ext); + if is_directory { + info.set_directory(true); } + index.records[base_ri].stdinfo = info; } } diff --git a/crates/uffs-mft/src/parse.rs b/crates/uffs-mft/src/parse.rs index 200eb2c50..3a1368eb2 100644 --- a/crates/uffs-mft/src/parse.rs +++ b/crates/uffs-mft/src/parse.rs @@ -55,9 +55,12 @@ mod tests; mod types; mod zero_alloc; -use attribute_helpers::{ - parse_data_attribute_full, parse_file_name_full, parse_standard_info_full, -}; +// Re-exported crate-wide: `crate::io::parser::unified` (outside this module +// tree) calls this directly to share the one extended-aware +// $STANDARD_INFORMATION parser with the legacy and direct-index pipelines, +// instead of duplicating it. +pub(crate) use attribute_helpers::parse_standard_info_full; +use attribute_helpers::{parse_data_attribute_full, parse_file_name_full}; pub use columns::ParsedColumns; pub use direct_index::parse_record_to_index; pub use fixup::apply_fixup; diff --git a/crates/uffs-mft/src/parse/attribute_helpers.rs b/crates/uffs-mft/src/parse/attribute_helpers.rs index 5943ef4ff..5530b75cf 100644 --- a/crates/uffs-mft/src/parse/attribute_helpers.rs +++ b/crates/uffs-mft/src/parse/attribute_helpers.rs @@ -12,7 +12,11 @@ use crate::ntfs::{ExtendedStandardInfo, NameInfo, StreamInfo}; /// /// Handles both NTFS 1.2 (36 bytes) and NTFS 3.0+ (72 bytes) formats. /// For NTFS 3.0+, also extracts `usn`, `security_id`, and `owner_id`. -pub(super) fn parse_standard_info_full( +/// +/// `pub(crate)`: this is the single source of truth for `$STANDARD_INFORMATION` +/// parsing and is also called directly from `crate::io::parser::unified`, which +/// sits outside the `parse` module tree. +pub(crate) fn parse_standard_info_full( data: &[u8], attr_offset: usize, result: &mut ExtendedStandardInfo, diff --git a/crates/uffs-mft/src/parse/direct_index.rs b/crates/uffs-mft/src/parse/direct_index.rs index 52b6f25ed..9804528a7 100644 --- a/crates/uffs-mft/src/parse/direct_index.rs +++ b/crates/uffs-mft/src/parse/direct_index.rs @@ -74,7 +74,7 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf use crate::index::{IndexNameRef, LinkInfo, NO_ENTRY, SizeInfo, StandardInfo, len_to_u16}; use crate::ntfs::{ AttributeRecordHeader, AttributeType, FileNameAttribute, FileRecordSegmentHeader, - StandardInformation, file_reference_to_frs, + file_reference_to_frs, }; if data.len() < size_of::() { @@ -143,29 +143,15 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf match attr_type { Some(AttributeType::StandardInformation) => { if attr_header.is_non_resident == 0 { - // Parse $STANDARD_INFORMATION - let value_offset_bytes = &data[offset + 20..offset + 22]; - let value_offset = usize::from(u16::from_le_bytes( - value_offset_bytes.try_into().unwrap_or([0, 0]), - )); - let si_offset = offset + value_offset; - if si_offset + size_of::() <= data.len() { - let si = match StandardInformation::read_from_prefix(&data[si_offset..]) { - Ok((si, _)) => si, - Err(_) => break, - }; - // Two-step canonical approach: - // 1. Parse raw attrs to ExtendedStandardInfo (complete parsing) - // 2. Convert to compact StandardInfo (single source of truth) - let ext = - crate::ntfs::ExtendedStandardInfo::from_attributes(si.file_attributes); - let mut info = StandardInfo::from_extended(&ext); - info.created = si.creation_time; - info.modified = si.modification_time; - info.accessed = si.access_time; - info.mft_changed = si.mft_change_time; - std_info = info; - } + // Shared with the legacy and unified pipelines: reads the + // 72-byte NTFS 3.0+ `StandardInformationExtended` form + // (usn/security_id/owner_id) when `value_length` says it's + // present, falling back to the 36-byte NTFS 1.2 form + // otherwise — see `attribute_helpers::parse_standard_info_full` + // for the single-source-of-truth rationale. + let mut ext = crate::ntfs::ExtendedStandardInfo::default(); + super::parse_standard_info_full(data, offset, &mut ext); + std_info = StandardInfo::from_extended(&ext); } } Some(AttributeType::FileName) => { From 9f49d7466f30414f1d130ff969fa62e10114db7b Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 20 Jul 2026 07:43:08 -0700 Subject: [PATCH 2/7] fix(mft): harden direct_index.rs against panic-on-malformed-input; delete superseded dead parser Investigating whether io/parser/index.rs's parse_record_to_index (zero production call sites, only its own test module) was legacy/dead code surfaced a more serious finding: it isn't just dead -- it's the WI-5.2 panic-hardened (checked arithmetic, .get()-only access) sibling of the parser that's actually live, and the live one never got that hardening. parse/direct_index.rs's parse_record_to_index -- the parser actually wired to the daemon's live USN-journal incremental-update path via usn/windows.rs:466 -- reads a resident attribute's 4-byte value_length (offset+16..20) and 2-byte value_offset (offset+20..22) via raw &data[a..b] slicing in 7 places (StandardInformation, FileName, ReparsePoint x2, IndexRoot x2, ObjectId/EA/etc, and the unknown-type catch-all), with no bounds check beyond the outer attribute-length gate. A resident attribute whose *declared* length is short enough to pass that gate but too short to cover those fixed fields -- e.g. sitting at the tail of a truncated or corrupted record -- panics: range start index 76 out of range for slice of length 74 reproduced live against a crafted record. The daemon builds with panic = "abort", so this is a real whole-process DoS on a malformed MFT record during journal replay, not a theoretical one. The existing malformed_records_do_not_panic corpus never caught it because (a) it never exercised this specific parser at all, and (b) RecordBuilder leaves bytes_in_use at 0, which makes every parser's attribute loop short-circuit before touching a single attribute byte -- so the corpus wasn't reaching the code it claimed to stress-test for any parser. Fix: - Add rd_u16/rd_u32 (checked, .get()-based, mirroring unified.rs's existing helpers) and route all 7 unguarded reads through them. - assert_all_parsers_survive now patches bytes_in_use to the record's real length before running each parser, so the existing corpus actually reaches attribute-body code instead of trivially short- circuiting; added crate::parse::parse_record_to_index (the real, live parser) to the parsers it exercises. - Added a dedicated regression case reproducing the exact bug shape (short declared length at the tail of the buffer) across all 6 affected attribute types. Verified red before the fix (panics with the exact message above), green after. With direct_index.rs now at parity, io/parser/index.rs (+ its index_extension.rs helper, 1671 lines total) is pure duplicated dead code, not a fallback worth keeping -- deleted both, along with the index_helpers.rs functions (InternalStreamChain, ExtensionSnapshot, merge_extension_streams, merge_extension_names) that existed only to serve them. Updated io.rs / io/parser/mod.rs re-exports accordingly (parse_record_to_index is no longer reachable via uffs_mft::io::*; confirmed zero consumers anywhere in this repo or uffs-products). Fixed docs/architecture/engine/03-ntfs-parsing.md's two source-path references, which had pointed at the dead file all along. --- crates/uffs-mft/src/io.rs | 2 +- crates/uffs-mft/src/io/parser/index.rs | 844 ------------------ .../uffs-mft/src/io/parser/index_extension.rs | 827 ----------------- crates/uffs-mft/src/io/parser/mod.rs | 54 +- crates/uffs-mft/src/parse/direct_index.rs | 66 +- crates/uffs-mft/src/parse/index_helpers.rs | 152 +--- docs/architecture/engine/03-ntfs-parsing.md | 4 +- scripts/ci/file_size_exceptions.txt | 4 +- 8 files changed, 85 insertions(+), 1868 deletions(-) delete mode 100644 crates/uffs-mft/src/io/parser/index.rs delete mode 100644 crates/uffs-mft/src/io/parser/index_extension.rs diff --git a/crates/uffs-mft/src/io.rs b/crates/uffs-mft/src/io.rs index ca4550c18..152715f12 100644 --- a/crates/uffs-mft/src/io.rs +++ b/crates/uffs-mft/src/io.rs @@ -46,7 +46,7 @@ pub use parser::parse_record_to_fragment; pub use parser::{ ExtensionAttributes, ParseResult, ParsedColumns, ParsedRecord, add_missing_parent_placeholders_to_vec, create_placeholder_record, parse_record, - parse_record_full, parse_record_to_index, parse_record_zero_alloc, process_record, + parse_record_full, parse_record_zero_alloc, process_record, }; #[cfg(windows)] pub(crate) use readers::{IoCompletionPort, MftRecordReader, OverlappedRead}; diff --git a/crates/uffs-mft/src/io/parser/index.rs b/crates/uffs-mft/src/io/parser/index.rs deleted file mode 100644 index 78f45ef79..000000000 --- a/crates/uffs-mft/src/io/parser/index.rs +++ /dev/null @@ -1,844 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Single-pass direct-to-index parser. -//! -//! Exception: Core MFT record parser with unified parse_record_to_index and -//! forensic mode. This is the performance-critical hot path. -//! -//! This module implements the high-performance single-pass parser that matches -//! an `MftIndex` directly from raw MFT bytes. It parses records into `MftIndex` -//! without creating intermediate `ParsedRecord` allocations, which is critical -//! for IOCP performance. -//! -//! # Hardening (WI-5.2) -//! This module parses **untrusted on-disk bytes**. Every offset/length -//! derived from those bytes is combined with `checked_add`/`checked_mul` -//! (or `saturating_*` where overflow is provably unreachable) and every -//! slice into `data` goes through `.get()` / the `rd_u*` helpers — never -//! `data[a..b]` indexing. The daemon builds with `panic = "abort"`, so a -//! single parser panic on a malformed record would be a whole-process -//! denial of service. -//! `arithmetic_side_effects` is enabled module-wide as a regression guard: -//! any new raw `+`/`*` on a byte-derived value is a compile error here. -#![warn(clippy::arithmetic_side_effects)] -// Performance-critical hot-path parser — minimal, scoped lint suppressions. -// Each suppression is justified with a reason. -#![expect( - clippy::doc_markdown, - reason = "NTFS terminology like WoF, MftIndex does not need backticks" -)] -#![expect( - clippy::manual_let_else, - reason = "explicit match is clearer in NTFS attribute dispatch" -)] -#![expect( - clippy::single_match_else, - reason = "explicit match arms are clearer for attribute type dispatch" -)] -use core::mem::size_of; - -use smallvec::SmallVec; -use zerocopy::FromBytes as _; - -use super::index_extension::parse_extension_to_index; -use crate::index::{len_to_u16, nonneg_to_u64, u32_as_usize}; -use crate::parse::index_helpers::{ - ExtensionSnapshot, InternalStreamChain, add_child_entry, add_link_to_index, - add_stream_to_index, build_internal_stream_chain, chain_links, chain_streams, - merge_extension_names, merge_extension_streams, -}; - -/// Parses a record directly into `MftIndex` (single-pass inline parsing). -/// -/// This function parses the record and adds it directly to the index, -/// creating parent placeholders on-demand. This single-pass approach -/// eliminates the intermediate `ParsedRecord` allocation. -/// -/// Handles ALL attribute types that `parse_record_full()` handles, including: -/// - `$STANDARD_INFORMATION`, `$FILE_NAME`, `$DATA` (default + ADS) -/// - `$REPARSE_POINT` (for WoF detection and junctions/symlinks) -/// - `$INDEX_ROOT`, `$INDEX_ALLOCATION`, `$BITMAP` (directory indexes) -/// - `$OBJECT_ID`, `$VOLUME_NAME`, `$VOLUME_INFORMATION`, `$PROPERTY_SET` -/// - `$EA`, `$EA_INFORMATION`, `$LOGGED_UTILITY_STREAM` -/// - `$SECURITY_DESCRIPTOR`, `$ATTRIBUTE_LIST` -/// - Unknown attribute types (counted as streams per NTFS convention) -/// -/// # Returns -/// -/// `true` if a record was added to the index, `false` if skipped. -#[expect( - clippy::too_many_lines, - reason = "monolithic parser kept for performance-critical hot path" -)] -#[expect( - clippy::cognitive_complexity, - reason = "NTFS attribute dispatch is inherently complex" -)] -#[expect( - clippy::indexing_slicing, - reason = "remaining [] are internal arena indices (index.records[..]/stream_indices[..]/\ - link_indices[..]) keyed by indices minted by this fn; not attacker-controlled. \ - All untrusted-`data` reads go through .get()/rd_u* (WI-5.2)." -)] -pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::MftIndex) -> bool { - use crate::index::{IndexNameRef, LinkInfo, NO_ENTRY, SizeInfo, StandardInfo}; - use crate::ntfs::{ - AttributeRecordHeader, AttributeType, FileNameAttribute, FileRecordSegmentHeader, - StandardInformation, file_reference_to_frs, - }; - - if data.len() < size_of::() { - return false; - } - - let header = match FileRecordSegmentHeader::read_from_prefix(data) { - Ok((header, _)) => header, - Err(_) => return false, - }; - - // Check if record is in use - if !header.is_in_use() { - return false; - } - - // Check magic - let multi_sector_header = header.multi_sector_header; - if !multi_sector_header.is_file_record() { - return false; - } - - // Handle extension records: add their names/streams to the base record. - // Extension records reference a base FRS; their attributes are merged inline. - if !header.is_base_record() { - let base_frs = file_reference_to_frs(header.base_file_record_segment); - return parse_extension_to_index(data, base_frs, index); - } - - let is_directory = header.is_directory(); - - // Parse attributes - let mut offset = usize::from(header.first_attribute_offset); - let max_offset = core::cmp::min(u32_as_usize(header.bytes_in_use), data.len()); - - // Temporary storage for parsed data - let mut std_info = StandardInfo::default(); - let mut primary_name: Option<(String, u64, u8, u16)> = None; // (name, parent_frs, namespace, parse_index) - let mut additional_names: SmallVec<[(String, u64, u16); 4]> = SmallVec::new(); - let mut name_parse_counter: u16 = 0; - let mut default_size = 0_u64; - let mut default_allocated = 0_u64; - // User-visible ADS: (stream_name, size, allocated) - let mut additional_streams: SmallVec<[(String, u64, u64); 4]> = SmallVec::new(); - // Internal NTFS streams (e.g. $REPARSE, $EA, $OBJECT_ID) — not emitted as - // output rows but still tracked for tree-metrics accounting. - // (size, allocated) - let mut internal_streams: SmallVec<[(u64, u64); 4]> = SmallVec::new(); - let mut reparse_tag: u32 = 0; - let mut dir_index_size: u64 = 0; - let mut dir_index_allocated: u64 = 0; - - // WI-5.2: every offset advance and slice below is derived from - // attacker-controllable record bytes, so all arithmetic uses - // `checked_*` and all slicing uses `data.get(..)` (fallible) — a - // malformed record `break`s the loop / skips the field instead of - // panicking. The daemon runs `panic = "abort"`, so a parser panic is a - // whole-process DoS. - while offset - .checked_add(size_of::()) - .is_some_and(|end| end <= max_offset) - { - let Some(attr_slice) = data.get(offset..) else { - break; - }; - let attr_header = match AttributeRecordHeader::read_from_prefix(attr_slice) { - Ok((attr_header, _)) => attr_header, - Err(_) => break, - }; - - if attr_header.type_code == AttributeType::END_MARKER { - break; - } - - let attr_len = u32_as_usize(attr_header.length); - let attr_end = offset.checked_add(attr_len); - if attr_header.length == 0 || attr_end.is_none_or(|end| end > max_offset) { - break; - } - - // Validate that the attribute's declared length fits within the record data - // This prevents reading past record boundaries when attributes are truncated - if attr_end.is_none_or(|end| end > data.len()) { - break; // Attribute extends past record — stop processing - } - - let attr_type = AttributeType::from_u32(attr_header.type_code); - match attr_type { - Some(AttributeType::StandardInformation) => { - if attr_header.is_non_resident == 0 { - // Parse $STANDARD_INFORMATION - let value_offset = usize::from(rd_u16(data, offset.saturating_add(20))); - if let Some(si_slice) = offset - .checked_add(value_offset) - .filter(|si_off| { - si_off.saturating_add(size_of::()) <= data.len() - }) - .and_then(|si_off| data.get(si_off..)) - { - let si = match StandardInformation::read_from_prefix(si_slice) { - Ok((si, _)) => si, - Err(_) => break, - }; - // Two-step canonical approach: - // 1. Parse raw attrs to ExtendedStandardInfo (complete parsing) - // 2. Convert to compact StandardInfo (single source of truth) - let ext = - crate::ntfs::ExtendedStandardInfo::from_attributes(si.file_attributes); - let mut info = StandardInfo::from_extended(&ext); - // Override timestamps from actual NTFS values - info.created = si.creation_time; - info.modified = si.modification_time; - info.accessed = si.access_time; - info.mft_changed = si.mft_change_time; - std_info = info; - } - } - } - Some(AttributeType::FileName) => { - if attr_header.is_non_resident == 0 { - // Parse $FILE_NAME - let value_offset = usize::from(rd_u16(data, offset.saturating_add(20))); - let fn_offset = offset.checked_add(value_offset); - if let Some(fn_slice) = fn_offset - .filter(|fn_off| { - fn_off.saturating_add(size_of::()) <= data.len() - }) - .and_then(|fn_off| data.get(fn_off..)) - && let Some(fn_off) = fn_offset - { - let fn_attr = match FileNameAttribute::read_from_prefix(fn_slice) { - Ok((fn_attr, _)) => fn_attr, - Err(_) => break, - }; - let name_len = usize::from(fn_attr.file_name_length); - let name_bytes_offset = - fn_off.saturating_add(size_of::()); - // `name_len` is a u16 (<= 65535); `*2` and `+ offset` use - // checked form so the parser is provably total, and let - // `data.get(..)` do the bounds check (None on a - // declared-length that overruns the record → skip name). - if let Some(name_bytes) = name_len - .checked_mul(2) - .and_then(|byte_len| name_bytes_offset.checked_add(byte_len)) - .and_then(|name_end| data.get(name_bytes_offset..name_end)) - { - // SmallVec avoids heap allocation for typical filenames (<= 64 chars) - let name_u16: SmallVec<[u16; 64]> = name_bytes - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes(*pair)) - .collect(); - let name = crate::io::parser::unified::decode_name_u16(&name_u16).0; - let parent_frs = file_reference_to_frs(fn_attr.parent_directory); - let namespace = fn_attr.file_name_namespace; - - // Skip DOS-only names (namespace 2) - if namespace != 2 { - let parse_idx = name_parse_counter; - // Monotonic name counter; one $FILE_NAME per record - // iteration, bounded by record size — cannot overflow u16. - name_parse_counter = name_parse_counter.saturating_add(1); - let is_better = match namespace { - 1 | 3 => true, // Win32 or Win32+DOS - 0 => primary_name.is_none(), // POSIX only if no name yet - _ => false, - }; - if is_better || primary_name.is_none() { - // Move old primary to additional if exists - if let Some((old_name, old_parent, _, old_parse_idx)) = - primary_name.take() - { - additional_names.push(( - old_name, - old_parent, - old_parse_idx, - )); - } - primary_name = Some((name, parent_frs, namespace, parse_idx)); - } else { - additional_names.push((name, parent_frs, parse_idx)); - } - } - } - } - } - } - Some(AttributeType::Data) => { - // legacy-output parity: Only primary attributes (LowestVCN == 0) count as - // streams. Continuation extents (LowestVCN > 0) are skipped. - // See ntfs_index_load.hpp:358 - let is_primary = if attr_header.is_non_resident == 0 { - true // Resident attributes are always primary - } else { - // Assume primary if can't read LowestVCN (None → true). - offset - .checked_add(16) - .and_then(|nr| nr.checked_add(8).and_then(|end| data.get(nr..end))) - .and_then(|sl| <[u8; 8]>::try_from(sl).ok()) - .is_none_or(|bytes| i64::from_le_bytes(bytes) == 0) - }; - - if !is_primary { - // Skip continuation extents - they don't count as new streams - offset = offset.saturating_add(u32_as_usize(attr_header.length)); - continue; - } - - // Parse $DATA - track both default stream and ADS - let name_len = usize::from(attr_header.name_length); - let (size, allocated) = if attr_header.is_non_resident != 0 { - // Non-resident: size at offset 48, allocated at offset 40 - // For compressed/sparse files, use CompressedSize at offset 64 - let nr_offset = offset.saturating_add(16); - let alloc_offset = offset.saturating_add(40); - let size_offset = offset.saturating_add(48); - if size_offset.saturating_add(8) <= data.len() { - // Check if compressed or sparse - let is_compressed_or_sparse = (attr_header.flags & 0x8001) != 0; - let compression_unit = rd_u16(data, nr_offset.saturating_add(18)); - let has_compression_unit = compression_unit > 0; - - let use_compressed_size = is_compressed_or_sparse || has_compression_unit; - let compressed_size_offset = nr_offset.saturating_add(48); // offset + 64 - - let allocated = if use_compressed_size - && compressed_size_offset.saturating_add(8) <= data.len() - { - // Read CompressedSize for compressed/sparse files - rd_u64(data, compressed_size_offset) - } else { - // Read AllocatedLength for normal files - rd_u64(data, alloc_offset) - }; - - let size = rd_u64(data, size_offset); - (size, allocated) - } else if alloc_offset.saturating_add(8) <= data.len() { - // Can read AllocatedSize but not DataSize — use AllocatedSize for both - let allocated = rd_u64(data, alloc_offset); - (allocated, allocated) - } else { - (0, 0) - } - } else { - // Resident: value_length at offset 16 - let len_offset = offset.saturating_add(16); - if len_offset.saturating_add(4) <= data.len() { - let len = rd_u32(data, len_offset); - (u64::from(len), 0) // allocated_size = 0 for resident files - } else { - (0, 0) - } - }; - - if name_len == 0 { - // Default stream — mark that unnamed $DATA exists - // (distinguishes "empty $DATA" from "no $DATA"). - // Boundary: lift parser-local raw `u64` to typed `Frs`. - let rec = index.get_or_create(crate::frs::Frs::new(frs)); - rec.set_has_default_data(); - default_size = size; - default_allocated = allocated; - } else { - // Alternate Data Stream (ADS) - let name_offset = offset.saturating_add(usize::from(attr_header.name_offset)); - if let Some(name_bytes) = name_len - .checked_mul(2) - .and_then(|byte_len| name_offset.checked_add(byte_len)) - .and_then(|name_end| data.get(name_offset..name_end)) - { - let name_u16: SmallVec<[u16; 64]> = name_bytes - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes(*pair)) - .collect(); - let stream_name = crate::io::parser::unified::decode_name_u16(&name_u16).0; - - // $BadClus:$Bad (FRS 8) uses InitializedSize - // instead of DataSize/AllocatedSize to avoid counting the - // entire volume size (ntfs_index_load.hpp lines 431-452). - let (stream_size, stream_alloc) = if frs == 8 - && attr_header.name_length == 4 - && stream_name == "$Bad" - && attr_header.is_non_resident != 0 - { - let init_size_offset = offset.saturating_add(56); - if init_size_offset.saturating_add(8) <= data.len() { - let init_size = rd_u64(data, init_size_offset); - (init_size, init_size) - } else { - (0, 0) - } - } else { - (size, allocated) - }; - - // ALL named $DATA streams create regular stream entries - // (counted in stream_count). Internal ones (names - // starting with $) are filtered from *output* by - // is_internal_windows_stream checks in the output layer, - // but must be counted here for correct descendants. - additional_streams.push((stream_name, stream_size, stream_alloc)); - } - } - } - Some(AttributeType::ReparsePoint) => { - // Parse $REPARSE_POINT to get the reparse tag. - // Both resident and non-resident forms are handled. - // $REPARSE_POINT is counted as a stream (affects descendants). - let (rp_size, rp_allocated) = if attr_header.is_non_resident == 0 { - // Resident reparse point (common case) - let value_length = u64::from(rd_u32(data, offset.saturating_add(16))); - - let value_offset = usize::from(rd_u16(data, offset.saturating_add(20))); - if let Some(rp_offset) = offset.checked_add(value_offset) { - // Read reparse tag (first 4 bytes of reparse point data) - reparse_tag = rd_u32(data, rp_offset); - } - (value_length, 0_u64) // Resident, allocated=0 - } else { - // Non-resident reparse point (rare - large reparse data) - let nr_offset = offset.saturating_add(16); - if nr_offset.saturating_add(48) <= data.len() { - let allocated = nonneg_to_u64(rd_i64(data, nr_offset.saturating_add(24))); - let data_size = nonneg_to_u64(rd_i64(data, nr_offset.saturating_add(32))); - (data_size, allocated) - } else { - (0_u64, 0_u64) - } - }; - - // $REPARSE_POINT is an internal stream — tracked for tree metrics - // but not emitted as a user-visible output row - internal_streams.push((rp_size, rp_allocated)); - } - Some( - AttributeType::IndexRoot | AttributeType::IndexAllocation | AttributeType::Bitmap, - ) => { - // $INDEX_ROOT and $INDEX_ALLOCATION with name $I30 contribute to - // directory size. Non-$I30 indexes are counted as individual streams. - - // Extract attribute name - let name_len = usize::from(attr_header.name_length); - let name_offset = offset.saturating_add(usize::from(attr_header.name_offset)); - // None when name_len == 0 or the declared length overruns the - // record → treated as non-$I30 with an empty name (matches the - // original guarded-out behavior). - let name_bytes_opt = if name_len > 0 { - name_len - .checked_mul(2) - .and_then(|byte_len| name_offset.checked_add(byte_len)) - .and_then(|name_end| data.get(name_offset..name_end)) - } else { - None - }; - let (is_i30, _attr_name) = name_bytes_opt.map_or_else( - || (false, String::new()), - |name_bytes| { - // Check for "$I30" in UTF-16LE - let is_i30 = - attr_header.name_length == 4 && name_bytes == b"$\x00I\x003\x000\x00"; - // Decode name for non-$I30 indexes - let name = if is_i30 { - String::new() - } else { - let name_u16: SmallVec<[u16; 64]> = name_bytes - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes(*pair)) - .collect(); - crate::io::parser::unified::decode_name_u16(&name_u16).0 - }; - (is_i30, name) - }, - ); - - if is_i30 { - // Accumulate $I30 sizes for directories - if attr_header.is_non_resident == 0 { - let value_length = u64::from(rd_u32(data, offset.saturating_add(16))); - // Directory index sizes are bounded by the volume; accumulating - // them cannot overflow u64 in practice — saturate to stay total. - dir_index_size = dir_index_size.saturating_add(value_length); - } else { - let (size, allocated) = read_nonresident_size_alloc(data, offset); - dir_index_size = dir_index_size.saturating_add(size); - dir_index_allocated = dir_index_allocated.saturating_add(allocated); - } - } else { - // Non-$I30 index - count as stream - // Check if primary attribute (LowestVCN == 0) - if is_nonresident_primary(data, offset, &attr_header) { - let (size, allocated) = read_size_alloc(data, offset, &attr_header); - // Non-$I30 index attributes are internal streams - internal_streams.push((size, allocated)); - } - } - } - Some( - AttributeType::ObjectId - | AttributeType::VolumeName - | AttributeType::VolumeInformation - | AttributeType::PropertySet - | AttributeType::Ea - | AttributeType::EaInformation - | AttributeType::LoggedUtilityStream - | AttributeType::SecurityDescriptor - | AttributeType::AttributeList, - ) => { - // All these are internal streams — tracked for tree metrics but - // not emitted as user-visible output rows. - // Check if primary attribute (LowestVCN == 0) - if is_nonresident_primary(data, offset, &attr_header) { - let (size, allocated) = read_size_alloc(data, offset, &attr_header); - internal_streams.push((size, allocated)); - } - } - _ => { - // Unknown attribute types are internal streams — tracked for - // tree metrics but not emitted as user-visible output rows. - // Check if primary attribute (LowestVCN == 0) - if is_nonresident_primary(data, offset, &attr_header) { - let (size, allocated) = read_size_alloc(data, offset, &attr_header); - internal_streams.push((size, allocated)); - } - } - } - - // `attr_header.length` was validated above (`offset + length <= data.len()`), - // so this advance cannot overflow; `saturating_add` keeps it total. - offset = offset.saturating_add(u32_as_usize(attr_header.length)); - } - - // Set directory flag in std_info BEFORE checking for filename - // This ensures is_directory is set even when $FILE_NAME is in extension record - if is_directory { - std_info.set_directory(true); - // For directories, set default size to directory index size - if dir_index_size > 0 { - default_size = dir_index_size; - default_allocated = dir_index_allocated; - } - } - - // Handle records without a filename in the base record - // The $FILE_NAME may be in an extension record - we still need to store stdinfo - let (name, parent_frs, _namespace, primary_parse_index) = match primary_name { - Some(n) => n, - None => { - // No $FILE_NAME in base record - store stdinfo anyway - // The extension record will add the name later - // - // IMPORTANT: We must still add ADS streams from the base record! - // The $FILE_NAME may be in an extension record, but the ADS are here. - // Without this, ADS on files/directories with extension records are lost. - - // Pre-process user-visible ADS streams BEFORE creating the record - let additional_stream_count = additional_streams.len(); - let stream_indices: Vec = additional_streams - .into_iter() - .map(|(name, size, alloc)| add_stream_to_index(index, &name, size, alloc)) - .collect(); - - // Build internal stream chain for tree-metrics accounting - let internal_stream_count = internal_streams.len(); - let InternalStreamChain { - first: first_internal, - size_total: internal_size_total, - alloc_total: internal_alloc_total, - } = build_internal_stream_chain(index, internal_streams); - - // Snapshot and setup record using helper. Lift parser-local raw - // `u64` to typed `Frs` once for all the typed-API call sites. - let frs_typed = crate::frs::Frs::new(frs); - let record = index.get_or_create(frs_typed); - let ext = ExtensionSnapshot { - stream_head: record.first_stream.next_entry, - stream_count: record.stream_count.saturating_sub(1), - total_extra: record.total_stream_count.saturating_sub(1), - name_next: NO_ENTRY, - name_count: 0, - internal_head: record.first_internal_stream, - internal_size: record.internal_streams_size, - internal_alloc: record.internal_streams_allocated, - first_stream_len: record.first_stream.size.length, - first_stream_alloc: record.first_stream.size.allocated, - }; - - record.stdinfo = std_info; - record.first_stream.size = SizeInfo { - length: default_size.saturating_add(ext.first_stream_len), - allocated: default_allocated.saturating_add(ext.first_stream_alloc), - }; - record.first_stream.flags = if record.stdinfo.is_directory() { - 0 - } else { - 8_u8 << 2_u8 - }; - record.internal_streams_size = internal_size_total; - record.internal_streams_allocated = internal_alloc_total; - record.first_internal_stream = first_internal; - - // Chain ADS streams and set counts - if !stream_indices.is_empty() { - chain_streams(index, &stream_indices); - let rec_chain = index.get_or_create(frs_typed); - rec_chain.first_stream.next_entry = stream_indices[0]; - } - let rec_counts = index.get_or_create(frs_typed); - // Stream counts are bounded by attributes-per-record; saturate to stay total. - rec_counts.stream_count = len_to_u16(additional_stream_count).saturating_add(1); - rec_counts.total_stream_count = len_to_u16(additional_stream_count) - .saturating_add(1) - .saturating_add(len_to_u16(internal_stream_count)); - - // Merge extension data - merge_extension_streams( - index, - frs, - stream_indices.last().copied(), - first_internal, - &ext, - ); - return true; - } - }; - - // Add primary name to names buffer and get reference - let name_offset = index.add_name(&name); - let name_len = name.len(); - let is_ascii = name.is_ascii(); - let extension_id = index.intern_extension(&name); - let name_ref = IndexNameRef::new(name_offset, len_to_u16(name_len), is_ascii, extension_id); - - // Pre-process additional names: add to names buffer and links list BEFORE - // getting record reference This avoids borrow checker issues with holding - // &mut record while modifying index - let additional_count = additional_names.len(); - // Collect parent FRS values for building children array later - let mut additional_parent_frs: SmallVec<[(u64, u16); 4]> = - SmallVec::with_capacity(additional_count); - let link_indices: Vec = additional_names - .into_iter() - .map(|(link_name, link_parent, link_parse_idx)| { - additional_parent_frs.push((link_parent, link_parse_idx)); - add_link_to_index(index, &link_name, link_parent) - }) - .collect(); - - // Pre-process user-visible ADS streams: add to names buffer and streams list - let additional_stream_count = additional_streams.len(); - let stream_indices: Vec = additional_streams - .into_iter() - .map(|(stream_name, stream_size, stream_alloc)| { - add_stream_to_index(index, &stream_name, stream_size, stream_alloc) - }) - .collect(); - - // Build internal stream chain for tree-metrics accounting - let internal_stream_count = internal_streams.len(); - let InternalStreamChain { - first: first_internal, - size_total: internal_size_total, - alloc_total: internal_alloc_total, - } = build_internal_stream_chain(index, internal_streams); - - // Ensure parent exists (create placeholder if needed) - do this before - // getting our record. Lift parser-local raw `u64` to typed `Frs`. - if parent_frs != frs && parent_frs != 0 { - index.get_or_create(crate::frs::Frs::new(parent_frs)); - // ^ side effect: ensures parent placeholder exists - } - - // Snapshot and setup record - let record = index.get_or_create(crate::frs::Frs::new(frs)); - let ext = ExtensionSnapshot { - stream_head: record.first_stream.next_entry, - stream_count: record.stream_count.saturating_sub(1), - total_extra: record.total_stream_count.saturating_sub(1), - name_next: record.first_name.next_entry, - name_count: if record.first_name.name.is_valid() { - record.name_count - } else { - 0 - }, - internal_head: record.first_internal_stream, - internal_size: record.internal_streams_size, - internal_alloc: record.internal_streams_allocated, - first_stream_len: record.first_stream.size.length, - first_stream_alloc: record.first_stream.size.allocated, - }; - - record.stdinfo = std_info; - record.first_stream.size = SizeInfo { - length: default_size.saturating_add(ext.first_stream_len), - allocated: default_allocated.saturating_add(ext.first_stream_alloc), - }; - record.first_stream.flags = if record.stdinfo.is_directory() { - 0 - } else { - 8_u8 << 2_u8 - }; - record.first_name = LinkInfo { - next_entry: NO_ENTRY, - name: name_ref, - _pad0: [0; 4], - // Typed `ParentFrs` slot — lift raw `u64` parser local. - parent_frs: crate::frs::ParentFrs::new(parent_frs), - }; - // Name/stream counts are bounded by attributes-per-record; saturate to stay - // total. - record.name_count = len_to_u16(additional_count).saturating_add(1); - record.stream_count = len_to_u16(additional_stream_count).saturating_add(1); - record.total_stream_count = len_to_u16(additional_stream_count) - .saturating_add(1) - .saturating_add(len_to_u16(internal_stream_count)); - record.internal_streams_size = internal_size_total; - record.internal_streams_allocated = internal_alloc_total; - record.first_internal_stream = first_internal; - record.reparse_tag = reparse_tag; - - // Chain links and streams, attach to record - if !link_indices.is_empty() { - record.first_name.next_entry = link_indices[0]; - } - if !stream_indices.is_empty() { - record.first_stream.next_entry = stream_indices[0]; - } - chain_links(index, &link_indices); - chain_streams(index, &stream_indices); - - // Merge extension data - merge_extension_streams( - index, - frs, - stream_indices.last().copied(), - first_internal, - &ext, - ); - merge_extension_names(index, frs, link_indices.last().copied(), &ext); - - // Build parent-child relationship for tree metrics computation - // This is critical for compute_tree_metrics() to work correctly. - // Each name (primary + additional) creates a child entry in its parent. - add_child_entry(index, parent_frs, frs, primary_parse_index); - - // Add child entries for additional names (hardlinks) - for &(link_parent_frs, link_parse_idx) in &additional_parent_frs { - add_child_entry(index, link_parent_frs, frs, link_parse_idx); - } - - true -} - -// ── Helpers ───────────────────────────────────────────────────────────── - -/// Read a little-endian u16 from the given offset, returning 0 if out of -/// bounds. -#[inline] -fn rd_u16(buf: &[u8], off: usize) -> u16 { - off.checked_add(2) - .and_then(|end| buf.get(off..end)) - .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) - .map_or(0, u16::from_le_bytes) -} - -/// Read a little-endian u32 from the given offset, returning 0 if out of -/// bounds. -#[inline] -fn rd_u32(buf: &[u8], off: usize) -> u32 { - off.checked_add(4) - .and_then(|end| buf.get(off..end)) - .and_then(|sl| <[u8; 4]>::try_from(sl).ok()) - .map_or(0, u32::from_le_bytes) -} - -/// Read a little-endian u64 from the given offset, returning 0 if out of -/// bounds. -#[inline] -fn rd_u64(buf: &[u8], off: usize) -> u64 { - off.checked_add(8) - .and_then(|end| buf.get(off..end)) - .and_then(|sl| <[u8; 8]>::try_from(sl).ok()) - .map_or(0, u64::from_le_bytes) -} - -/// Read a little-endian i64 from the given offset, returning 0 if out of -/// bounds. -#[inline] -fn rd_i64(buf: &[u8], off: usize) -> i64 { - off.checked_add(8) - .and_then(|end| buf.get(off..end)) - .and_then(|sl| <[u8; 8]>::try_from(sl).ok()) - .map_or(0, i64::from_le_bytes) -} - -/// Determine whether a non-`$DATA` attribute is the primary extent -/// (`LowestVCN == 0`). -/// -/// Resident attributes are always primary. For non-resident attributes the -/// `LowestVCN` lives at `offset + 16` (8 bytes); a truncated record that -/// cannot supply it is treated as **not** primary (preserves the original -/// `else { false }` semantics of the internal-stream branches). -#[inline] -fn is_nonresident_primary( - data: &[u8], - offset: usize, - attr_header: &crate::ntfs::AttributeRecordHeader, -) -> bool { - if attr_header.is_non_resident == 0 { - return true; - } - offset - .checked_add(16) - .filter(|nr| nr.saturating_add(8) <= data.len()) - .is_some_and(|nr| rd_i64(data, nr) == 0) -} - -/// Read the `(DataSize, AllocatedSize)` pair from a non-resident attribute's -/// header at `offset`, clamping negative values to 0. -/// -/// Returns `(0, 0)` if the header is truncated (the `nr + 48 <= len` guard -/// preserves the original "all fields present" semantics). -#[inline] -fn read_nonresident_size_alloc(data: &[u8], offset: usize) -> (u64, u64) { - let nr_offset = offset.saturating_add(16); - if nr_offset.saturating_add(48) <= data.len() { - let allocated = nonneg_to_u64(rd_i64(data, nr_offset.saturating_add(24))); - let data_size = nonneg_to_u64(rd_i64(data, nr_offset.saturating_add(32))); - (data_size, allocated) - } else { - (0, 0) - } -} - -/// Read the `(size, allocated)` pair for an internal-stream attribute, -/// dispatching on residency. -/// -/// Resident attributes report `(value_length@offset+16, 0)`; non-resident -/// attributes delegate to [`read_nonresident_size_alloc`]. -#[inline] -fn read_size_alloc( - data: &[u8], - offset: usize, - attr_header: &crate::ntfs::AttributeRecordHeader, -) -> (u64, u64) { - if attr_header.is_non_resident == 0 { - (u64::from(rd_u32(data, offset.saturating_add(16))), 0) - } else { - read_nonresident_size_alloc(data, offset) - } -} diff --git a/crates/uffs-mft/src/io/parser/index_extension.rs b/crates/uffs-mft/src/io/parser/index_extension.rs deleted file mode 100644 index 1167f0539..000000000 --- a/crates/uffs-mft/src/io/parser/index_extension.rs +++ /dev/null @@ -1,827 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Extension record parser for direct-to-index path. -//! -//! Exception: This file is intentionally large (720+ LOC) to match the -//! completeness of `index.rs` - it handles all the same attribute types that -//! can appear in extension records. See `scripts/ci/file_size_exceptions.txt`. -//! -//! This module handles extension records for the single-pass parser, extracting -//! names, streams, and all attribute types from extension records and merging -//! them into base records in the index. -//! -//! # Hardening (WI-5.2) -//! This module parses **untrusted on-disk bytes**. Every offset/length derived -//! from those bytes is combined with `checked_add`/`checked_mul` (or -//! `saturating_*` where overflow is provably unreachable) and every slice into -//! `data` goes through `.get()` / the `rd_u*` helpers — never `data[a..b]` -//! indexing. The daemon builds with `panic = "abort"`, so a single parser panic -//! on a malformed record would be a whole-process denial of service. -//! `arithmetic_side_effects` is enabled module-wide as a regression guard: any -//! new raw `+`/`*` on a byte-derived value is a compile error here. -#![warn(clippy::arithmetic_side_effects)] - -use core::mem::size_of; - -use smallvec::SmallVec; -use zerocopy::FromBytes as _; - -use crate::index::{frs_to_usize, len_to_u16, len_to_u32, u32_as_usize}; - -/// Parses an extension record and adds its names/streams to the base record. -/// -/// Extension records contain additional `$FILE_NAME` attributes (hard links) -/// and additional attributes (ADS, system attributes, etc.) that don't fit -/// in the base record. This function extracts those attributes and adds them -/// to the base record in the index. -/// -/// Handles ALL attribute types that `parse_record_full()` handles, including: -/// - `$FILE_NAME` (hard links) -/// - `$DATA` (ADS) -/// - `$REPARSE_POINT`, `$INDEX_ROOT`, `$INDEX_ALLOCATION`, `$BITMAP` -/// - `$OBJECT_ID`, `$EA`, `$LOGGED_UTILITY_STREAM`, etc. -/// - Unknown attribute types -/// -/// # Arguments -/// -/// * `data` - The raw extension record data (after fixup) -/// * `base_frs` - The FRS of the base record this extension belongs to -/// * `index` - The MFT index to update -/// -/// # Returns -/// -/// `true` if any names/streams were added, `false` otherwise. -#[expect( - clippy::cognitive_complexity, - reason = "NTFS attribute dispatch is inherently complex" -)] -#[expect( - clippy::too_many_lines, - reason = "monolithic extension parser for performance" -)] -#[expect( - clippy::indexing_slicing, - reason = "the only `[]` indexing that remains is into internal arena vectors \ - (index.records / index.links / index.streams / index.internal_streams / \ - index.frs_to_idx) whose indices are produced by this code, not by untrusted \ - on-disk bytes. All reads of the untrusted `data` slice go through `.get()` / \ - the `rd_u*` helpers (WI-5.2)" -)] -pub(super) fn parse_extension_to_index( - data: &[u8], - base_frs: u64, - index: &mut crate::index::MftIndex, -) -> bool { - use crate::index::{ChildInfo, IndexNameRef, IndexStreamInfo, LinkInfo, NO_ENTRY, SizeInfo}; - use crate::ntfs::{ - AttributeRecordHeader, AttributeType, FileNameAttribute, FileRecordSegmentHeader, - }; - - if data.len() < size_of::() { - return false; - } - - let Ok((header, _)) = FileRecordSegmentHeader::read_from_prefix(data) else { - return false; - }; - - // Parse attributes to find $FILE_NAME and $DATA - let mut offset = usize::from(header.first_attribute_offset); - let max_offset = core::cmp::min(u32_as_usize(header.bytes_in_use), data.len()); - - // Collect names and streams from extension record - let mut names: SmallVec<[(String, u64); 4]> = SmallVec::new(); - // User-visible ADS only - let mut streams: SmallVec<[(String, u64, u64); 4]> = SmallVec::new(); - // Internal NTFS streams (for tree-metrics accounting) - let mut ext_internal_streams: SmallVec<[(u64, u64); 4]> = SmallVec::new(); - let mut dir_index_size: u64 = 0; - let mut dir_index_allocated: u64 = 0; - // Default $DATA stream (unnamed, name_len == 0) found in extension record - let mut default_data_size: u64 = 0; - let mut default_data_allocated: u64 = 0; - let mut found_default_data = false; - - while offset - .checked_add(size_of::()) - .is_some_and(|end| end <= max_offset) - { - let Some(attr_slice) = data.get(offset..) else { - break; - }; - let Ok((attr_header, _)) = AttributeRecordHeader::read_from_prefix(attr_slice) else { - break; - }; - - if attr_header.type_code == AttributeType::END_MARKER { - break; - } - - // `offset + length` can overflow on a crafted `length`; checked_add → break. - let Some(attr_end) = offset.checked_add(u32_as_usize(attr_header.length)) else { - break; - }; - if attr_header.length == 0 || attr_end > max_offset { - break; - } - - let attr_type = AttributeType::from_u32(attr_header.type_code); - match attr_type { - Some(AttributeType::FileName) => { - // Parse $FILE_NAME attribute - if attr_header.is_non_resident == 0 { - let value_offset = usize::from(rd_u16(data, offset.saturating_add(20))); - // `offset + value_offset` is byte-derived; checked, then re-validated - // by the `.get()` below. - if let Some(fn_offset) = offset.checked_add(value_offset) - && let Some(fn_slice) = fn_offset - .checked_add(size_of::()) - .filter(|end| *end <= data.len()) - .and_then(|_| data.get(fn_offset..)) - { - let Ok((fn_attr, _)) = FileNameAttribute::read_from_prefix(fn_slice) else { - break; - }; - - // Skip DOS-only names (namespace 2) - if fn_attr.file_name_namespace != 2 { - let name_len = usize::from(fn_attr.file_name_length); - let name_start = - fn_offset.saturating_add(size_of::()); - // `name_len * 2` (UTF-16) overflows on a crafted length → - // checked_mul/checked_add, then `.get()` bounds the slice. - if let Some(name_bytes) = name_len - .checked_mul(2) - .and_then(|byte_len| name_start.checked_add(byte_len)) - .and_then(|end| data.get(name_start..end)) - { - let name_u16: SmallVec<[u16; 64]> = name_bytes - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes(*pair)) - .collect(); - let name = crate::io::parser::unified::decode_name_u16(&name_u16).0; - let parent_frs = fn_attr.parent_directory & 0x0000_FFFF_FFFF_FFFF; - names.push((name, parent_frs)); - } - } - } - } - } - Some(AttributeType::Data) => { - // legacy-output parity: Only primary attributes (LowestVCN == 0) count as - // streams. Continuation extents (LowestVCN > 0) are skipped. - // See ntfs_index_load.hpp:358 - let is_primary = nr_is_primary(data, attr_header.is_non_resident, offset); - - if !is_primary { - // Skip continuation extents - they don't count as new streams - offset = offset.saturating_add(u32_as_usize(attr_header.length)); - continue; - } - - // Parse $DATA attribute — default stream (unnamed) or ADS (named) - let name_len = usize::from(attr_header.name_length); - let (size, allocated) = if attr_header.is_non_resident != 0 { - // `rd_u*` are individually bounds-safe (return 0 OOB); the - // `nr + 48 <= len` guard preserves the original "all fields - // present" semantics. `nr` via checked_add. - offset - .checked_add(16) - .filter(|nr| nr.saturating_add(48) <= data.len()) - .map_or((0, 0), |nr_offset| { - // Check if compressed or sparse - let is_compressed_or_sparse = (attr_header.flags & 0x8001) != 0; - let compression_unit = rd_u16(data, nr_offset.saturating_add(18)); - let has_compression_unit = compression_unit > 0; - - let use_compressed_size = - is_compressed_or_sparse || has_compression_unit; - // offset + 64 - let compressed_size_offset = nr_offset.saturating_add(48); - - // Preserve original fallback: only read CompressedSize - // when its 8-byte field is fully in bounds; otherwise - // read AllocatedLength. - let allocated = if use_compressed_size - && compressed_size_offset.saturating_add(8) <= data.len() - { - // Read CompressedSize for compressed/sparse files - rd_u64(data, compressed_size_offset).cast_signed() - } else { - // Read AllocatedLength for normal files - rd_u64(data, nr_offset.saturating_add(24)).cast_signed() - }; - - let size = rd_u64(data, nr_offset.saturating_add(32)).cast_signed(); - ( - size.max(0).cast_unsigned(), - allocated.max(0).cast_unsigned(), - ) - }) - } else { - (u64::from(rd_u32(data, offset.saturating_add(16))), 0) - }; - - if name_len == 0 { - // Default $DATA stream — update base record size - // Mark that unnamed $DATA exists on the base record - // (distinguishes "empty $DATA" from "no $DATA") - { - let bf = frs_to_usize(base_frs); - if bf < index.frs_to_idx.len() { - let base_idx = index.frs_to_idx[bf]; - if base_idx != NO_ENTRY { - index.records[u32_as_usize(base_idx)].set_has_default_data(); - } - } - } - default_data_size = size; - default_data_allocated = allocated; - found_default_data = true; - } else { - // ADS (named stream) - let name_offset = offset.saturating_add(usize::from(attr_header.name_offset)); - if let Some(name_bytes) = read_name_bytes(data, name_offset, name_len) { - let name_u16: SmallVec<[u16; 64]> = name_bytes - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes(*pair)) - .collect(); - let stream_name = crate::io::parser::unified::decode_name_u16(&name_u16).0; - // ALL named $DATA streams create regular - // stream entries. Internal ones are filtered from - // output by is_internal_windows_stream in the output layer. - streams.push((stream_name, size, allocated)); - } - } - } - Some(AttributeType::ReparsePoint) => { - // Parse $REPARSE_POINT - add as stream - let (rp_size, rp_allocated) = - read_attr_size(data, attr_header.is_non_resident, offset); - ext_internal_streams.push((rp_size, rp_allocated)); - } - Some( - AttributeType::IndexRoot | AttributeType::IndexAllocation | AttributeType::Bitmap, - ) => { - // Extract attribute name - let name_len = usize::from(attr_header.name_length); - let (is_i30, _attr_name) = if name_len > 0 { - let name_offset = offset.saturating_add(usize::from(attr_header.name_offset)); - read_name_bytes(data, name_offset, name_len).map_or_else( - || (false, String::new()), - |name_bytes| { - let is_i30 = attr_header.name_length == 4 - && name_bytes == b"$\x00I\x003\x000\x00"; - let name = if is_i30 { - String::new() - } else { - let name_u16: SmallVec<[u16; 64]> = name_bytes - .as_chunks::<2>() - .0 - .iter() - .map(|pair| u16::from_le_bytes(*pair)) - .collect(); - crate::io::parser::unified::decode_name_u16(&name_u16).0 - }; - (is_i30, name) - }, - ) - } else { - (false, String::new()) - }; - - if is_i30 { - // Accumulate $I30 sizes - if attr_header.is_non_resident == 0 { - let value_length = u64::from(rd_u32(data, offset.saturating_add(16))); - // Disk-derived accumulator; saturate to avoid overflow panic. - dir_index_size = dir_index_size.saturating_add(value_length); - } else if let Some(nr_offset) = offset - .checked_add(16) - .filter(|nr| nr.saturating_add(48) <= data.len()) - { - let allocated = rd_u64(data, nr_offset.saturating_add(24)).cast_signed(); - let data_size = rd_u64(data, nr_offset.saturating_add(32)).cast_signed(); - // Disk-derived accumulators; saturate to avoid overflow panic. - dir_index_size = - dir_index_size.saturating_add(data_size.max(0).cast_unsigned()); - dir_index_allocated = - dir_index_allocated.saturating_add(allocated.max(0).cast_unsigned()); - } - } else { - // Non-$I30 index — internal stream for tree metrics - if nr_is_primary(data, attr_header.is_non_resident, offset) { - let (size, allocated) = - read_attr_size(data, attr_header.is_non_resident, offset); - ext_internal_streams.push((size, allocated)); - } - } - } - Some( - AttributeType::ObjectId - | AttributeType::VolumeName - | AttributeType::VolumeInformation - | AttributeType::PropertySet - | AttributeType::Ea - | AttributeType::EaInformation - | AttributeType::LoggedUtilityStream - | AttributeType::SecurityDescriptor - | AttributeType::AttributeList, - ) => { - // All counted as streams - if nr_is_primary(data, attr_header.is_non_resident, offset) { - let (size, allocated) = - read_attr_size(data, attr_header.is_non_resident, offset); - ext_internal_streams.push((size, allocated)); - } - } - Some(AttributeType::StandardInformation) => { - // Skip - not expected in extension records - } - _ => { - // Unknown attribute types — counted as streams (catch-all). - if nr_is_primary(data, attr_header.is_non_resident, offset) { - let (size, allocated) = - read_attr_size(data, attr_header.is_non_resident, offset); - ext_internal_streams.push((size, allocated)); - } - } - } - - // Disk-derived advance; saturate so a crafted length can't overflow. - offset = offset.saturating_add(u32_as_usize(attr_header.length)); - } - - // If no names, user-visible streams, internal streams, default data, or - // directory index sizes found, nothing to do - if names.is_empty() - && streams.is_empty() - && ext_internal_streams.is_empty() - && !found_default_data - && dir_index_size == 0 - && dir_index_allocated == 0 - { - return false; - } - - // Add names to the base record - // First, add all names to the names buffer and create LinkInfo entries - let mut link_indices: Vec = Vec::with_capacity(names.len()); - for (name, parent_frs) in &names { - let name_offset = index.add_name(name); - let name_len = name.len(); - let is_ascii = name.is_ascii(); - let extension_id = index.intern_extension(name); - let name_ref = IndexNameRef::new(name_offset, len_to_u16(name_len), is_ascii, extension_id); - - let link_idx = len_to_u32(index.links.len()); - index.links.push(LinkInfo { - next_entry: NO_ENTRY, - name: name_ref, - _pad0: [0; 4], - // Typed `ParentFrs` slot — lift parser-local raw `u64`. - parent_frs: crate::frs::ParentFrs::new(*parent_frs), - }); - link_indices.push(link_idx); - } - - // Add streams to the streams buffer - let mut stream_indices: Vec = Vec::with_capacity(streams.len()); - for (stream_name, size, allocated) in &streams { - let name_offset = index.add_name(stream_name); - let name_len = stream_name.len(); - let is_ascii = stream_name.is_ascii(); - let extension_id = index.intern_extension(stream_name); - let name_ref = IndexNameRef::new(name_offset, len_to_u16(name_len), is_ascii, extension_id); - - let stream_idx = len_to_u32(index.streams.len()); - index.streams.push(IndexStreamInfo { - size: SizeInfo { - length: *size, - allocated: *allocated, - }, - next_entry: NO_ENTRY, - name: name_ref, - flags: 8_u8 << 2_u8, - _pad0: [0; 3], - }); - stream_indices.push(stream_idx); - } - - // Ensure parent directories exist for the new names. Boundary: lift - // parser-local raw `u64` to typed `Frs` at the typed-API call site. - for (_, parent_frs) in &names { - if *parent_frs != base_frs && *parent_frs != 0 { - index.get_or_create(crate::frs::Frs::new(*parent_frs)); - // ^ side effect: ensures parent placeholder exists - } - } - - // Get the base record and add the names/streams to it - let base_frs_usize = frs_to_usize(base_frs); - if base_frs_usize >= index.frs_to_idx.len() { - // Base record doesn't exist yet — create a placeholder - index.get_or_create(crate::frs::Frs::new(base_frs)); - } - - let record_idx = index.frs_to_idx[base_frs_usize]; - if record_idx == NO_ENTRY { - // Base record doesn't exist — create it - index.get_or_create(crate::frs::Frs::new(base_frs)); - } - - // Now get the record and chain the new links/streams - let base_idx = index.frs_to_idx[base_frs_usize]; - if base_idx != NO_ENTRY { - // Snapshot fields from the record before any re-borrowing - let (pre_chain_name_count, has_valid_name, first_name_next, first_stream_next) = { - let rec = &index.records[u32_as_usize(base_idx)]; - ( - rec.name_count, - rec.first_name.name.is_valid(), - rec.first_name.next_entry, - rec.first_stream.next_entry, - ) - }; - - // Add new links to the record - if !link_indices.is_empty() { - // Check if base record has no name (first_name is empty) - // This happens when the $FILE_NAME attribute is ONLY in extension records - if has_valid_name { - // Base record already has a name — chain extension names as additional hard - // links. Find the end of the current link chain. - let last_link_idx = if first_name_next == NO_ENTRY { - None - } else { - let mut idx = first_name_next; - while index.links[u32_as_usize(idx)].next_entry != NO_ENTRY { - idx = index.links[u32_as_usize(idx)].next_entry; - } - Some(idx) - }; - - // Chain the new links together - for pair in link_indices.windows(2) { - if let [current, next] = *pair { - index.links[u32_as_usize(current)].next_entry = next; - } - } - - // Attach to the chain - if let Some(last_idx) = last_link_idx { - index.links[u32_as_usize(last_idx)].next_entry = link_indices[0]; - } else { - // first_name has no next_entry, attach directly - let rec_link = &mut index.records[u32_as_usize(base_idx)]; - rec_link.first_name.next_entry = link_indices[0]; - } - - // Update name count (bounded internal counter; saturate). - let rec_name_count = &mut index.records[u32_as_usize(base_idx)]; - rec_name_count.name_count = rec_name_count - .name_count - .saturating_add(len_to_u16(link_indices.len())); - } else { - // Copy the first extension name directly into first_name - // This matches established behavior (ntfs_index.hpp lines 559-567) - let first_link_name = index.links[u32_as_usize(link_indices[0])].name; - let first_link_parent = index.links[u32_as_usize(link_indices[0])].parent_frs; - let rec_first = &mut index.records[u32_as_usize(base_idx)]; - rec_first.first_name.name = first_link_name; - rec_first.first_name.parent_frs = first_link_parent; - // Don't increment name_count for the first name (it's already counted as 1) - - // Chain remaining links (if any) to first_name.next_entry - if link_indices.len() > 1 { - // Chain the remaining links together (links[1..]); link 0 was - // copied into first_name above and is not chained from. - if let Some(rest) = link_indices.get(1..) { - for pair in rest.windows(2) { - if let [current, next] = *pair { - index.links[u32_as_usize(current)].next_entry = next; - } - } - } - // Attach remaining links to first_name - let rec_extra = &mut index.records[u32_as_usize(base_idx)]; - rec_extra.first_name.next_entry = link_indices[1]; - // Update name count for additional links only (saturate). - rec_extra.name_count = rec_extra - .name_count - .saturating_add(len_to_u16(link_indices.len().saturating_sub(1))); - } - } - } - - // Chain new streams to the end of the existing stream chain - if !stream_indices.is_empty() { - // Find the end of the current stream chain (using snapshot) - let last_stream_idx = if first_stream_next == NO_ENTRY { - None - } else { - let mut idx = first_stream_next; - while index.streams[u32_as_usize(idx)].next_entry != NO_ENTRY { - idx = index.streams[u32_as_usize(idx)].next_entry; - } - Some(idx) - }; - - // Chain the new streams together - for pair in stream_indices.windows(2) { - if let [current, next] = *pair { - index.streams[u32_as_usize(current)].next_entry = next; - } - } - - // Attach to the chain - if let Some(last_idx) = last_stream_idx { - index.streams[u32_as_usize(last_idx)].next_entry = stream_indices[0]; - } else { - // first_stream has no next_entry, attach directly - let rec_stream_attach = &mut index.records[u32_as_usize(base_idx)]; - rec_stream_attach.first_stream.next_entry = stream_indices[0]; - } - - // Update stream count (user-visible only; bounded counters, saturate). - let rec_stream_count = &mut index.records[u32_as_usize(base_idx)]; - let stream_added = len_to_u16(stream_indices.len()); - rec_stream_count.stream_count = - rec_stream_count.stream_count.saturating_add(stream_added); - rec_stream_count.total_stream_count = rec_stream_count - .total_stream_count - .saturating_add(stream_added); - } - - // Build internal stream chain for extension record attributes - if !ext_internal_streams.is_empty() { - let rec_internal = &mut index.records[u32_as_usize(base_idx)]; - - // Find end of existing internal stream chain - let last_internal_idx = if rec_internal.first_internal_stream == NO_ENTRY { - None - } else { - let mut idx = rec_internal.first_internal_stream; - while index.internal_streams[u32_as_usize(idx)].next_entry != NO_ENTRY { - idx = index.internal_streams[u32_as_usize(idx)].next_entry; - } - Some(idx) - }; - - let mut first_new_internal = NO_ENTRY; - let mut prev_internal = NO_ENTRY; - for (ist_size, ist_allocated) in &ext_internal_streams { - rec_internal.internal_streams_size = - rec_internal.internal_streams_size.saturating_add(*ist_size); - rec_internal.internal_streams_allocated = rec_internal - .internal_streams_allocated - .saturating_add(*ist_allocated); - - let new_idx = len_to_u32(index.internal_streams.len()); - index - .internal_streams - .push(crate::index::InternalStreamInfo { - size: SizeInfo { - length: *ist_size, - allocated: *ist_allocated, - }, - next_entry: NO_ENTRY, - flags: 0, - }); - - if first_new_internal == NO_ENTRY { - first_new_internal = new_idx; - } - if prev_internal != NO_ENTRY { - index.internal_streams[u32_as_usize(prev_internal)].next_entry = new_idx; - } - prev_internal = new_idx; - } - - // Attach to existing chain or set as head - if let Some(last_idx) = last_internal_idx { - index.internal_streams[u32_as_usize(last_idx)].next_entry = first_new_internal; - } else { - let rec_head = &mut index.records[u32_as_usize(base_idx)]; - rec_head.first_internal_stream = first_new_internal; - } - - // Update total_stream_count to include new internal streams (saturate). - let rec_total = &mut index.records[u32_as_usize(base_idx)]; - rec_total.total_stream_count = rec_total - .total_stream_count - .saturating_add(len_to_u16(ext_internal_streams.len())); - } - - // Merge default $DATA stream from extension record into base record. - // This handles files whose $DATA attribute doesn't fit in the base MFT - // record (e.g., large files with extensive run lists). - if found_default_data { - let rec_data = &mut index.records[u32_as_usize(base_idx)]; - // Ensure has_default_data bit is set (may not have been set - // earlier if the base record didn't exist at attribute-parse time) - rec_data.set_has_default_data(); - - // If base record has no $DATA (both fields are 0), use extension's $DATA. - // Otherwise, accumulate extension $DATA to base $DATA. - if rec_data.first_stream.size.length == 0 && rec_data.first_stream.size.allocated == 0 { - // Base has no $DATA — use extension's values - rec_data.first_stream.size.length = default_data_size; - rec_data.first_stream.size.allocated = default_data_allocated; - } else { - // Base has partial $DATA — accumulate extension values - rec_data.first_stream.size.length = rec_data - .first_stream - .size - .length - .saturating_add(default_data_size); - rec_data.first_stream.size.allocated = rec_data - .first_stream - .size - .allocated - .saturating_add(default_data_allocated); - } - } - - // Merge directory index sizes from extension records - if dir_index_size > 0 || dir_index_allocated > 0 { - let rec_dir = &mut index.records[u32_as_usize(base_idx)]; - // Add to the first_stream size (which represents the default stream for - // directories). Disk-derived sizes; saturate to avoid overflow panic. - rec_dir.first_stream.size.length = rec_dir - .first_stream - .size - .length - .saturating_add(dir_index_size); - rec_dir.first_stream.size.allocated = rec_dir - .first_stream - .size - .allocated - .saturating_add(dir_index_allocated); - } - - // Build parent-child relationship for names added from extension records - // This is critical for compute_tree_metrics() to work correctly. - // Use the name_count from BEFORE link-chaining to avoid overflow - let existing_name_count = pre_chain_name_count; - - for (name_idx, (_, parent_frs)) in names.iter().enumerate() { - let p_frs = *parent_frs; - if p_frs == base_frs || p_frs == u64::from(NO_ENTRY) { - continue; - } - - // Ensure parent exists - let parent_idx = { - let p_frs_usize = frs_to_usize(p_frs); - if p_frs_usize >= index.frs_to_idx.len() { - // `p_frs` is masked to 48 bits, so `+ 1` cannot overflow usize on - // 64-bit; saturate defensively to keep arithmetic panic-free. - index - .frs_to_idx - .resize(p_frs_usize.saturating_add(1), NO_ENTRY); - } - if index.frs_to_idx[p_frs_usize] == NO_ENTRY { - // Create placeholder parent - let new_idx = len_to_u32(index.records.len()); - index.frs_to_idx[p_frs_usize] = new_idx; - index - .records - .push(crate::index::FileRecord::new(crate::frs::Frs::new(p_frs))); - } - index.frs_to_idx[p_frs_usize] - }; - - // Add child entry - // name_index is the position in the combined name list (existing + new) - // For extension records, the first name might replace first_name (if empty), - // so we need to account for that - // - // FIX: The off-by-one bug was here. Extension names are appended AFTER - // existing names, so the index should be existing_name_count + name_idx, - // not existing_name_count - 1 + name_idx. - // - // Example: base has 1 name (index 0), extension adds 1 name - // - existing_name_count = 1 - // - name_idx = 0 (first extension name) - // - effective_name_idx should be 1 (the second name overall) - let effective_name_idx = if existing_name_count == 0 { - // First extension name became first_name, so name_index starts at 0 - len_to_u16(name_idx) - } else { - // Extension names are appended after existing names (bounded u16 - // name-index counter; saturate). - existing_name_count.saturating_add(len_to_u16(name_idx)) - }; - - let child_idx = len_to_u32(index.children.len()); - let parent = &mut index.records[u32_as_usize(parent_idx)]; - let old_first_child = parent.first_child; - parent.first_child = child_idx; - - index.children.push(ChildInfo { - next_entry: old_first_child, - _pad0: [0; 4], - // Typed `Frs` slot — lift parser-local raw `u64`. - child_frs: crate::frs::Frs::new(base_frs), - name_index: effective_name_idx, - _pad1: [0; 6], - }); - } - } - - !names.is_empty() - || !streams.is_empty() - || !ext_internal_streams.is_empty() - || found_default_data - || dir_index_size > 0 - || dir_index_allocated > 0 -} - -// ── Helpers (untrusted-byte readers, WI-5.2) ──────────────────────────────── - -/// Read a little-endian `u16` from `buf` at `off`, returning 0 if the 2-byte -/// field is out of bounds. -#[inline] -fn rd_u16(buf: &[u8], off: usize) -> u16 { - off.checked_add(2) - .and_then(|end| buf.get(off..end)) - .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) - .map_or(0, u16::from_le_bytes) -} - -/// Read a little-endian `u32` from `buf` at `off`, returning 0 if the 4-byte -/// field is out of bounds. -#[inline] -fn rd_u32(buf: &[u8], off: usize) -> u32 { - off.checked_add(4) - .and_then(|end| buf.get(off..end)) - .and_then(|sl| <[u8; 4]>::try_from(sl).ok()) - .map_or(0, u32::from_le_bytes) -} - -/// Read a little-endian `u64` from `buf` at `off`, returning 0 if the 8-byte -/// field is out of bounds. -#[inline] -fn rd_u64(buf: &[u8], off: usize) -> u64 { - off.checked_add(8) - .and_then(|end| buf.get(off..end)) - .and_then(|sl| <[u8; 8]>::try_from(sl).ok()) - .map_or(0, u64::from_le_bytes) -} - -/// Determine whether a non-resident attribute is a *primary* extent -/// (`LowestVCN == 0`); resident attributes are always primary. -/// -/// Mirrors the original guarded read: when the 8-byte `LowestVCN` field is out -/// of bounds the attribute is treated as non-primary ("can't verify, skip to be -/// safe"). -#[inline] -fn nr_is_primary(data: &[u8], is_non_resident: u8, offset: usize) -> bool { - if is_non_resident == 0 { - return true; - } - offset - .checked_add(16) - .filter(|nr| nr.saturating_add(8) <= data.len()) - .is_some_and(|nr_offset| rd_u64(data, nr_offset).cast_signed() == 0) -} - -/// Read the `(size, allocated)` pair for an attribute, mirroring the original -/// per-branch logic exactly: -/// - resident: `(ValueLength @ offset+16 as u32, 0)` -/// - non-resident: only when the 48-byte non-resident header is fully present, -/// `(DataSize @ +32, AllocatedSize @ +24)` clamped to `>= 0`; otherwise `(0, -/// 0)`. -#[inline] -fn read_attr_size(data: &[u8], is_non_resident: u8, offset: usize) -> (u64, u64) { - if is_non_resident == 0 { - return (u64::from(rd_u32(data, offset.saturating_add(16))), 0); - } - offset - .checked_add(16) - .filter(|nr| nr.saturating_add(48) <= data.len()) - .map_or((0, 0), |nr_offset| { - let allocated = rd_u64(data, nr_offset.saturating_add(24)).cast_signed(); - let data_size = rd_u64(data, nr_offset.saturating_add(32)).cast_signed(); - ( - data_size.max(0).cast_unsigned(), - allocated.max(0).cast_unsigned(), - ) - }) -} - -/// Return the `name_len * 2` UTF-16 name bytes starting at `name_offset`, or -/// `None` if the (byte-derived) range overflows or lies outside `data`. -#[inline] -fn read_name_bytes(data: &[u8], name_offset: usize, name_len: usize) -> Option<&[u8]> { - name_len - .checked_mul(2) - .and_then(|byte_len| name_offset.checked_add(byte_len)) - .and_then(|end| data.get(name_offset..end)) -} diff --git a/crates/uffs-mft/src/io/parser/mod.rs b/crates/uffs-mft/src/io/parser/mod.rs index ddb952cbd..18350438c 100644 --- a/crates/uffs-mft/src/io/parser/mod.rs +++ b/crates/uffs-mft/src/io/parser/mod.rs @@ -7,8 +7,6 @@ mod fragment; mod fragment_extension; -mod index; -mod index_extension; pub(crate) mod unified; #[expect( @@ -16,7 +14,6 @@ pub(crate) mod unified; reason = "re-exporting deprecated API for backward compatibility" )] pub use fragment::parse_record_to_fragment; -pub use index::parse_record_to_index; pub use unified::process_record; pub use crate::parse::{ @@ -29,13 +26,15 @@ pub use crate::parse::{ mod tests { #[expect(deprecated, reason = "testing deprecated parse_record_to_fragment API")] use super::parse_record_to_fragment; - use super::{parse_record_to_index, process_record}; + use super::process_record; use crate::index::{MftIndex, MftIndexFragment}; #[test] fn parse_record_to_index_rejects_short_buffers() { let mut index = MftIndex::new(crate::platform::DriveLetter::C); - assert!(!parse_record_to_index(&[0_u8; 3], 42, &mut index)); + assert!(!crate::parse::parse_record_to_index( + &[0_u8; 3], 42, &mut index + )); } #[test] @@ -262,13 +261,33 @@ mod tests { /// Run every malformed record through all three entry points; the test /// passes iff none of them panics (the return value is irrelevant). - fn assert_all_parsers_survive(record: &[u8]) { + fn assert_all_parsers_survive(input: &[u8]) { + // `RecordBuilder` leaves `bytes_in_use` (header offset 24..28) at 0, + // which makes every parser's attribute loop compute `max_offset = 0` + // and exit before touching a single attribute byte -- so unless we + // patch it to the record's real length, this whole corpus never + // actually reaches the code it claims to stress-test. Patch a local + // copy rather than requiring every call site to do it. + let mut patched = input.to_vec(); + if let Some(bytes_in_use_field) = patched.get_mut(24..28) { + let len = u32::try_from(input.len()).unwrap_or(u32::MAX); + bytes_in_use_field.copy_from_slice(&len.to_le_bytes()); + } + let record = patched.as_slice(); + // The return value is irrelevant — reaching the end of this function // at all means none of the three parsers panicked, which is the // property under test. `black_box` consumes each result so it is // neither an unused binding nor an under-typed `let _` discard. - let mut index = MftIndex::new(crate::platform::DriveLetter::C); - core::hint::black_box(parse_record_to_index(record, 42, &mut index)); + // `crate::parse::parse_record_to_index` (direct_index.rs) is the + // parser actually wired to the live USN-journal incremental update + // path (usn::windows). + let mut direct_index = MftIndex::new(crate::platform::DriveLetter::C); + core::hint::black_box(crate::parse::parse_record_to_index( + record, + 42, + &mut direct_index, + )); let mut unified_index = MftIndex::new(crate::platform::DriveLetter::C); let mut name_buf = String::new(); @@ -328,5 +347,24 @@ mod tests { .map(|n| n.wrapping_mul(31).wrapping_add(7)) .collect(); assert_all_parsers_survive(&RecordBuilder::new(56).raw(&garbage).build()); + + // 9. Regression pin: a resident attribute whose *declared* length is short + // enough to pass the `offset + length <= max_offset` gate, but too short to + // actually cover the fixed `value_length` (offset+16..20) / `value_offset` + // (offset+20..22) fields the parser reads unconditionally, right at the tail + // of the buffer. `crate::parse::parse_record_to_index` used to read these + // via raw `&data[a..b]` slicing (no bounds check at all beyond the + // attribute-length gate above) and panicked with "range start index ... out + // of range" on exactly this shape. Covers StandardInformation, FileName, + // ReparsePoint, IndexRoot, ObjectId, and the unknown-type catch-all -- every + // arm in direct_index.rs that reads those two fixed fields. + for type_code in [0x10, 0x30, 0xC0, 0x90, 0x40, 0x77] { + assert_all_parsers_survive( + &RecordBuilder::new(56) + .attr(type_code, 17, 0, 0, 0) + .raw(&[0_u8; 2]) + .build(), + ); + } } } diff --git a/crates/uffs-mft/src/parse/direct_index.rs b/crates/uffs-mft/src/parse/direct_index.rs index 9804528a7..5d79372bf 100644 --- a/crates/uffs-mft/src/parse/direct_index.rs +++ b/crates/uffs-mft/src/parse/direct_index.rs @@ -44,6 +44,32 @@ use super::index_helpers::{ }; use crate::index::{nonneg_to_u64, u32_as_usize}; +/// Read a little-endian u16 from the given offset, returning 0 if out of +/// bounds. WI-5.2: this file's attribute-length gate (`offset + attr_header. +/// length <= max_offset`) does not by itself guarantee any *specific* fixed +/// field inside the attribute is in bounds — a short declared `length` can +/// pass that gate while still being too small to cover `value_length`/ +/// `value_offset`. Reads of those fields go through this helper instead of +/// raw slicing so a malformed/truncated record degrades gracefully instead +/// of panicking (the daemon builds with `panic = "abort"`). +#[inline] +fn rd_u16(buf: &[u8], off: usize) -> u16 { + off.checked_add(2) + .and_then(|end| buf.get(off..end)) + .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) + .map_or(0, u16::from_le_bytes) +} + +/// Read a little-endian u32 from the given offset, returning 0 if out of +/// bounds. See [`rd_u16`] for the rationale. +#[inline] +fn rd_u32(buf: &[u8], off: usize) -> u32 { + off.checked_add(4) + .and_then(|end| buf.get(off..end)) + .and_then(|sl| <[u8; 4]>::try_from(sl).ok()) + .map_or(0, u32::from_le_bytes) +} + /// Parses a record directly into `MftIndex` (single-pass inline parsing). /// /// This function parses the record and adds it directly to the index, @@ -157,10 +183,7 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf Some(AttributeType::FileName) => { if attr_header.is_non_resident == 0 { // Parse $FILE_NAME - let value_offset_bytes = &data[offset + 20..offset + 22]; - let value_offset = usize::from(u16::from_le_bytes( - value_offset_bytes.try_into().unwrap_or([0, 0]), - )); + let value_offset = usize::from(rd_u16(data, offset + 20)); let fn_offset = offset + value_offset; if fn_offset + size_of::() <= data.len() { let fn_attr = match FileNameAttribute::read_from_prefix(&data[fn_offset..]) @@ -304,15 +327,8 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf // $REPARSE_POINT is counted as a stream (affects descendants). let (rp_size, rp_allocated) = if attr_header.is_non_resident == 0 { // Resident reparse point (common case) - let value_length_bytes = &data[offset + 16..offset + 20]; - let value_length = u64::from(u32::from_le_bytes( - value_length_bytes.try_into().unwrap_or([0, 0, 0, 0]), - )); - - let value_offset_bytes = &data[offset + 20..offset + 22]; - let value_offset = usize::from(u16::from_le_bytes( - value_offset_bytes.try_into().unwrap_or([0, 0]), - )); + let value_length = u64::from(rd_u32(data, offset + 16)); + let value_offset = usize::from(rd_u16(data, offset + 20)); let rp_offset = offset + value_offset; if rp_offset + 4 <= data.len() { // Read reparse tag (first 4 bytes of reparse point data) @@ -377,11 +393,7 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf if is_i30 { // Accumulate $I30 sizes for directories if attr_header.is_non_resident == 0 { - let value_length_bytes = &data[offset + 16..offset + 20]; - let value_length = u64::from(u32::from_le_bytes( - value_length_bytes.try_into().unwrap_or([0; 4]), - )); - dir_index_size += value_length; + dir_index_size += u64::from(rd_u32(data, offset + 16)); } else { let nr_offset = offset + 16; if nr_offset + 48 <= data.len() { @@ -414,11 +426,7 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf if is_primary { let (size, allocated) = if attr_header.is_non_resident == 0 { - let value_length_bytes = &data[offset + 16..offset + 20]; - let value_length = u64::from(u32::from_le_bytes( - value_length_bytes.try_into().unwrap_or([0; 4]), - )); - (value_length, 0_u64) + (u64::from(rd_u32(data, offset + 16)), 0_u64) } else { let nr_offset = offset + 16; if nr_offset + 48 <= data.len() { @@ -499,11 +507,7 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf }; let (size, allocated) = if attr_header.is_non_resident == 0 { - let value_length_bytes = &data[offset + 16..offset + 20]; - let value_length = u64::from(u32::from_le_bytes( - value_length_bytes.try_into().unwrap_or([0; 4]), - )); - (value_length, 0_u64) + (u64::from(rd_u32(data, offset + 16)), 0_u64) } else { let nr_offset = offset + 16; if nr_offset + 48 <= data.len() { @@ -586,11 +590,7 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf }; let (size, allocated) = if attr_header.is_non_resident == 0 { - let value_length_bytes = &data[offset + 16..offset + 20]; - let value_length = u64::from(u32::from_le_bytes( - value_length_bytes.try_into().unwrap_or([0; 4]), - )); - (value_length, 0_u64) + (u64::from(rd_u32(data, offset + 16)), 0_u64) } else { let nr_offset = offset + 16; if nr_offset + 48 <= data.len() { diff --git a/crates/uffs-mft/src/parse/index_helpers.rs b/crates/uffs-mft/src/parse/index_helpers.rs index 049aef4a9..79d33b715 100644 --- a/crates/uffs-mft/src/parse/index_helpers.rs +++ b/crates/uffs-mft/src/parse/index_helpers.rs @@ -6,14 +6,9 @@ //! These helpers reduce code duplication in the main parser while maintaining //! performance through inlining. -#![expect( - clippy::if_not_else, - reason = "!= NO_ENTRY is clearer for sentinel value checks" -)] - use crate::index::{ - ChildInfo, IndexNameRef, IndexStreamInfo, InternalStreamInfo, LinkInfo, MftIndex, NO_ENTRY, - SizeInfo, frs_to_usize, len_to_u16, len_to_u32, u32_as_usize, + ChildInfo, IndexNameRef, IndexStreamInfo, LinkInfo, MftIndex, NO_ENTRY, SizeInfo, frs_to_usize, + len_to_u16, len_to_u32, u32_as_usize, }; /// Adds a stream to the index and returns its index. @@ -50,57 +45,6 @@ pub(crate) fn add_stream_to_index( stream_idx } -/// Result of building an internal stream chain. -pub(crate) struct InternalStreamChain { - /// First index in the chain, or `NO_ENTRY` if empty. - pub first: u32, - /// Total size of all internal streams. - pub size_total: u64, - /// Total allocated size of all internal streams. - pub alloc_total: u64, -} - -/// Builds an internal stream chain from size/allocated pairs. -#[inline] -pub(crate) fn build_internal_stream_chain( - index: &mut MftIndex, - streams: I, -) -> InternalStreamChain -where - I: IntoIterator, -{ - let mut size_total = 0_u64; - let mut alloc_total = 0_u64; - let mut first = NO_ENTRY; - let mut last = NO_ENTRY; - - for (ist_size, ist_allocated) in streams { - size_total = size_total.saturating_add(ist_size); - alloc_total = alloc_total.saturating_add(ist_allocated); - let new_idx = len_to_u32(index.internal_streams.len()); - index.internal_streams.push(InternalStreamInfo { - size: SizeInfo { - length: ist_size, - allocated: ist_allocated, - }, - next_entry: NO_ENTRY, - flags: 0, - }); - if last == NO_ENTRY { - first = new_idx; - } else { - index.internal_streams[u32_as_usize(last)].next_entry = new_idx; - } - last = new_idx; - } - - InternalStreamChain { - first, - size_total, - alloc_total, - } -} - /// Chains stream indices together and returns the first index. #[inline] pub(crate) fn chain_streams(index: &mut MftIndex, stream_indices: &[u32]) { @@ -194,95 +138,3 @@ pub(crate) fn add_child_entry( _pad1: [0; 6], }); } - -/// Data snapshot from an extension record that needs to be merged into the -/// base. -pub(crate) struct ExtensionSnapshot { - /// Head of the extension's stream chain. - pub stream_head: u32, - /// Number of additional streams from extension (excluding default). - pub stream_count: u16, - /// Total extra count from extension (excluding default). - pub total_extra: u16, - /// Head of the extension's name chain. - pub name_next: u32, - /// Number of names from extension. - pub name_count: u16, - /// Head of the extension's internal stream chain. - pub internal_head: u32, - /// Size of internal streams from extension. - pub internal_size: u64, - /// Allocated size of internal streams from extension. - pub internal_alloc: u64, - /// Default stream length from extension. - pub first_stream_len: u64, - /// Default stream allocated from extension. - pub first_stream_alloc: u64, -} - -/// Merges extension streams into the base record's stream chain. -#[inline] -pub(crate) fn merge_extension_streams( - index: &mut MftIndex, - frs: u64, - base_stream_tail: Option, - first_internal: u32, - ext: &ExtensionSnapshot, -) { - // Lift parser-local raw `u64` to typed `Frs` once for all the typed - // `get_or_create` calls below. - let frs_typed = crate::frs::Frs::new(frs); - // Merge user-visible streams - if ext.stream_count > 0 { - let tail = base_stream_tail.unwrap_or(NO_ENTRY); - if tail != NO_ENTRY { - index.streams[u32_as_usize(tail)].next_entry = ext.stream_head; - } else { - let record = index.get_or_create(frs_typed); - record.first_stream.next_entry = ext.stream_head; - } - let record = index.get_or_create(frs_typed); - record.stream_count += ext.stream_count; - record.total_stream_count += ext.stream_count; - } - - // Merge internal streams - if ext.internal_head != NO_ENTRY { - if first_internal != NO_ENTRY { - let mut tail = first_internal; - while index.internal_streams[u32_as_usize(tail)].next_entry != NO_ENTRY { - tail = index.internal_streams[u32_as_usize(tail)].next_entry; - } - index.internal_streams[u32_as_usize(tail)].next_entry = ext.internal_head; - } else { - let record = index.get_or_create(frs_typed); - record.first_internal_stream = ext.internal_head; - } - let record = index.get_or_create(frs_typed); - record.internal_streams_size += ext.internal_size; - record.internal_streams_allocated += ext.internal_alloc; - record.total_stream_count += ext.total_extra.saturating_sub(ext.stream_count); - } -} - -/// Merges extension names into the base record's name chain. -#[inline] -pub(crate) fn merge_extension_names( - index: &mut MftIndex, - frs: u64, - base_name_tail: Option, - ext: &ExtensionSnapshot, -) { - if ext.name_count > 0 { - let frs_typed = crate::frs::Frs::new(frs); - let tail = base_name_tail.unwrap_or(NO_ENTRY); - if tail != NO_ENTRY { - index.links[u32_as_usize(tail)].next_entry = ext.name_next; - } else { - let record = index.get_or_create(frs_typed); - record.first_name.next_entry = ext.name_next; - } - let record = index.get_or_create(frs_typed); - record.name_count += ext.name_count; - } -} diff --git a/docs/architecture/engine/03-ntfs-parsing.md b/docs/architecture/engine/03-ntfs-parsing.md index ce1c15ff8..9e696b00e 100644 --- a/docs/architecture/engine/03-ntfs-parsing.md +++ b/docs/architecture/engine/03-ntfs-parsing.md @@ -430,7 +430,7 @@ impl Iterator for AttributeIterator<'_> { ### Base Record Parser -**Source:** `io/parser/index.rs` — `parse_record_to_index()` +**Source:** `parse/direct_index.rs` — `parse_record_to_index()` This is the **hot path** — called for every 1KB record during IOCP reading. @@ -483,7 +483,7 @@ parse_record_to_index(buffer: &[u8], frs: u64, index: &mut MftIndex) -> bool ### Extension Record Parser -**Source:** `io/parser/index_extension.rs` +**Source:** `parse/direct_index_extension.rs` When `base_file_record_segment != 0`, the record is an extension: diff --git a/scripts/ci/file_size_exceptions.txt b/scripts/ci/file_size_exceptions.txt index 91304a425..7c0b95561 100644 --- a/scripts/ci/file_size_exceptions.txt +++ b/scripts/ci/file_size_exceptions.txt @@ -7,9 +7,7 @@ crates/uffs-core/src/search/field/field_metadata.rs|PERMANENT: Single const fn m crates/uffs-core/src/search/filters/tests.rs|PERMANENT: Integration test suite for filter pipeline; splitting further would scatter related test fixtures crates/uffs-core/src/search/filters/mod.rs|PERMANENT: Cohesive SearchFilters/SearchFilterParams definitions + from_params construction; kept together so the full per-field filter contract is auditable in one place crates/uffs-client/src/schema/field_metadata.rs|PERMANENT: Single const fn match table — one FieldMeta per FieldId variant; mirrors uffs-core version -crates/uffs-mft/src/io/parser/index.rs|PERMANENT: Performance-critical single-pass MFT record parser; monolithic loop for cache locality -crates/uffs-mft/src/io/parser/index_extension.rs|PERMANENT: Extension record parser mirroring index.rs structure; must stay parallel for maintenance parity -crates/uffs-mft/src/parse/direct_index_extension.rs|PERMANENT: Offline variant of extension parser; same structural reasoning as io/parser counterparts +crates/uffs-mft/src/parse/direct_index_extension.rs|PERMANENT: Extension record parser for the direct-to-index pipeline; same structural reasoning as its base-record counterpart crates/uffs-mft/src/reader/index_read.rs|PERMANENT: Single impl MftReader block with tightly coupled cfg-gated pipeline stages crates/uffs-diag/src/bin/compare_scan_parity.rs|PERMANENT: Standalone diagnostic binary; single-file readability outweighs LOC policy for tooling crates/uffs-mcp/src/cookbook.rs|PERMANENT: Declarative JSON data (curated agent cookbook examples); splitting by line count would fragment the cohesive narrative From fca0d67f1045da85d790b57580e5236a4ae5b3aa Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 20 Jul 2026 08:11:11 -0700 Subject: [PATCH 3/7] fix(mft): populate ADS/stream is_sparse and is_resident on both production parsers IndexStreamInfo and InternalStreamInfo both already declare bit0=is_sparse, bit1=is_resident in their packed flags byte -- but every write site in both production parsers (unified.rs's process_record and direct_index.rs's parse_record_to_index) hardcoded those two bits to 0/false regardless of the real attribute, for every stream type: the default $DATA stream, every named $DATA (ADS), $REPARSE_POINT, non-$I30 $INDEX_ROOT/$INDEX_ALLOCATION/$BITMAP, $OBJECT_ID and friends, and the unknown-type catch-all. Every ADS reported as non-sparse/non-resident no matter what it actually was. Both bits are free to populate: is_resident is already known everywhere (attr_header.is_non_resident == 0, already read to pick the size-calc branch), and is_sparse lives in the attribute record header's own ATTRIBUTE_FLAG_SPARSE bit (0x8000), already-parsed data with no new I/O. No change to any on-disk struct layout or size. While auditing this, found and fixed the same panic-on-malformed-input gap in parse/direct_index_extension.rs (the extension-record sibling of direct_index.rs) that was fixed for direct_index.rs itself in the previous commit: unguarded 16-byte value_length / 2-byte value_offset reads via raw &data[a..b] slicing, now routed through checked rd_u16/rd_u32 helpers. This file processes extension records on the same live USN-journal path and had never received the WI-5.2 hardening pass either. - index_helpers.rs: add_stream_to_index now takes is_sparse/is_resident and bakes them into the flags byte alongside the existing type_name_id bits. Added a StreamEntry type alias ((name, size, allocated, is_sparse, is_resident)) to keep the SmallVec tuple under clippy::type_complexity. - direct_index.rs / direct_index_extension.rs: compute is_sparse/is_resident at every one of the ~10 stream-producing sites (Data/ADS, ReparsePoint, IndexRoot family, ObjectId family, catch-all) and thread them through; set the default stream's first_stream.flags, which neither file did before (it was left at its zero default, losing type_name_id too). - unified.rs: computed once per attribute in the shared catch-all dispatch arm and applied to all 4 write sites ($I30, default $DATA, ADS, internal-stream). Added a regression test building a synthetic non-resident, sparse-flagged ADS and asserting IndexStreamInfo::is_sparse()/is_resident() come out correct (not hardcoded false) on both process_record and parse_record_to_index. --- crates/uffs-mft/src/io/parser/mod.rs | 144 ++++++++++++++++++ crates/uffs-mft/src/io/parser/unified.rs | 22 ++- crates/uffs-mft/src/parse/direct_index.rs | 71 +++++++-- .../src/parse/direct_index_extension.rs | 101 +++++++----- crates/uffs-mft/src/parse/index_helpers.rs | 12 +- scripts/ci/file_size_exceptions.txt | 1 + 6 files changed, 298 insertions(+), 53 deletions(-) diff --git a/crates/uffs-mft/src/io/parser/mod.rs b/crates/uffs-mft/src/io/parser/mod.rs index 18350438c..10ad7cf50 100644 --- a/crates/uffs-mft/src/io/parser/mod.rs +++ b/crates/uffs-mft/src/io/parser/mod.rs @@ -194,6 +194,128 @@ mod tests { assert_eq!(direct_rec.stdinfo.owner_id, owner_id); } + /// Regression pin: a named `$DATA` (ADS) attribute's real `is_sparse`/ + /// `is_resident` status must reach the index's `IndexStreamInfo`, on + /// both production parsers. Both fields already existed in the struct + /// (`bit0`/`bit1` of `flags`) but every write site hardcoded them to + /// `false` regardless of the real attribute — every ADS reported as + /// non-sparse/non-resident no matter what it actually was. + #[test] + fn ads_sparse_and_resident_bits_reach_both_production_parsers() { + // Minimal $FILE_NAME so both parsers accept the record and give it a + // name (see the extended-standard-info test above for field order). + let mut fn_payload = Vec::new(); + fn_payload.extend_from_slice(&0_u64.to_le_bytes()); // parent_directory + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // creation_time + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // modification_time + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // mft_change_time + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // access_time + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // allocated_size + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // data_size + fn_payload.extend_from_slice(&0_u32.to_le_bytes()); // file_attributes + fn_payload.extend_from_slice(&0_u16.to_le_bytes()); // packed_ea_size + fn_payload.extend_from_slice(&0_u16.to_le_bytes()); // reserved + fn_payload.push(1); // file_name_length = 1 char + fn_payload.push(1); // namespace = Win32 + fn_payload.extend_from_slice(&0x0061_u16.to_le_bytes()); // "a" + let file_name_total_len = u32::try_from(24 + fn_payload.len()).expect("fits in u32"); + + // Named, non-resident $DATA (an ADS) flagged ATTRIBUTE_FLAG_SPARSE + // (0x8000) in the attribute-record header. Layout after the 16-byte + // common header: LowestVCN(8)=0, HighestVCN(8), MappingPairsOffset(2) + // + CompressionUnit(1) + Reserved(5), AllocatedSize(8), DataSize(8), + // InitializedSize(8) — 48 bytes total — then the 6-byte UTF-16LE + // name "ads" at name_offset=48+16=64. + let allocated_size = 8192_i64; + let data_size = 4096_i64; + let mut ads_nr = Vec::new(); + ads_nr.extend_from_slice(&0_i64.to_le_bytes()); // LowestVCN = 0 (primary) + ads_nr.extend_from_slice(&0_i64.to_le_bytes()); // HighestVCN + ads_nr.extend_from_slice(&[0_u8; 8]); // MappingPairsOffset+CompressionUnit+Reserved + ads_nr.extend_from_slice(&allocated_size.to_le_bytes()); + ads_nr.extend_from_slice(&data_size.to_le_bytes()); + ads_nr.extend_from_slice(&0_i64.to_le_bytes()); // InitializedSize + assert_eq!(ads_nr.len(), 48, "NonResidentAttributeData is 48 bytes"); + let ads_name: Vec = "ads".encode_utf16().flat_map(u16::to_le_bytes).collect(); + let ads_total_len = u32::try_from(16 + ads_nr.len() + ads_name.len()).expect("fits in u32"); + + let mut record = RecordBuilder::new(56) + .attr(0x30, file_name_total_len, 0, 0, 0) + .raw(&u32::try_from(fn_payload.len()).expect("fits in u32").to_le_bytes()) + .raw(&24_u16.to_le_bytes()) // value_offset + .raw(&[0_u8; 2]) + .raw(&fn_payload) + .attr_flags(0x80, ads_total_len, 1, 3, 64, 0x8000) + .raw(&ads_nr) + .raw(&ads_name) + .build(); + + let total_len = u32::try_from(record.len()).expect("fits in u32"); + record + .get_mut(24..28) + .expect("record is well over 28 bytes long") + .copy_from_slice(&total_len.to_le_bytes()); + + // Path 1: process_record — the default bulk-load pipeline. + let mut unified_index = MftIndex::new(crate::platform::DriveLetter::C); + let mut name_buf = String::new(); + process_record(&record, 42, &mut unified_index, &mut name_buf); + let unified_rec = unified_index + .find(crate::frs::Frs::new(42)) + .expect("process_record must create the base record"); + assert_ne!( + unified_rec.first_stream.next_entry, + crate::index::NO_ENTRY, + "the ADS must be chained onto the record" + ); + let unified_stream = unified_index + .streams + .get(crate::index::u32_as_usize( + unified_rec.first_stream.next_entry, + )) + .expect("chained stream index must be valid"); + assert!( + unified_stream.is_sparse(), + "process_record dropped is_sparse" + ); + assert!( + !unified_stream.is_resident(), + "a non-resident ADS must not report is_resident" + ); + assert_eq!( + unified_stream.size.length, + u64::try_from(data_size).unwrap() + ); + assert_eq!( + unified_stream.size.allocated, + u64::try_from(allocated_size).unwrap() + ); + + // Path 2: crate::parse::parse_record_to_index — the live + // USN-journal incremental-update pipeline (direct_index.rs). + let mut direct_index = MftIndex::new(crate::platform::DriveLetter::C); + assert!(crate::parse::parse_record_to_index( + &record, + 42, + &mut direct_index + )); + let direct_rec = direct_index + .find(crate::frs::Frs::new(42)) + .expect("parse_record_to_index must create the base record"); + assert_ne!(direct_rec.first_stream.next_entry, crate::index::NO_ENTRY); + let direct_stream = direct_index + .streams + .get(crate::index::u32_as_usize( + direct_rec.first_stream.next_entry, + )) + .expect("chained stream index must be valid"); + assert!( + direct_stream.is_sparse(), + "parse_record_to_index dropped is_sparse" + ); + assert!(!direct_stream.is_resident()); + } + // ── WI-5.2 panic-resistance corpus ────────────────────────────── // // The daemon builds with `panic = "abort"`: a single parser panic on a @@ -247,6 +369,28 @@ mod tests { self } + /// Same as [`Self::attr`], but with an explicit NTFS attribute-flags + /// value (e.g. `0x8000` = `ATTRIBUTE_FLAG_SPARSE`) instead of always + /// zeroing that field. + fn attr_flags( + mut self, + type_code: u32, + length: u32, + non_resident: u8, + name_length: u8, + name_offset: u16, + attr_flags: u16, + ) -> Self { + self.bytes.extend_from_slice(&type_code.to_le_bytes()); + self.bytes.extend_from_slice(&length.to_le_bytes()); + self.bytes.push(non_resident); + self.bytes.push(name_length); + self.bytes.extend_from_slice(&name_offset.to_le_bytes()); + self.bytes.extend_from_slice(&attr_flags.to_le_bytes()); + self.bytes.extend_from_slice(&[0_u8; 2]); // instance + self + } + /// Append raw filler bytes (used to reach a target value offset or to /// pad with garbage). fn raw(mut self, bytes: &[u8]) -> Self { diff --git a/crates/uffs-mft/src/io/parser/unified.rs b/crates/uffs-mft/src/io/parser/unified.rs index 464c3865f..320caefd0 100644 --- a/crates/uffs-mft/src/io/parser/unified.rs +++ b/crates/uffs-mft/src/io/parser/unified.rs @@ -590,12 +590,22 @@ pub fn process_record(data: &[u8], frs: u64, index: &mut MftIndex, name_buf: &mu (u64::from(rd_u32(data, offset.saturating_add(16))), 0) }; + // Already-parsed attribute-header data, free to read — + // `IndexStreamInfo`/`InternalStreamInfo` both reserve + // bit0=is_sparse, bit1=is_resident, but every write site + // below used to hardcode them to 0/false regardless of + // the real attribute. + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + // ── Classify and store ─────────────────────────── if is_i30 { // $I30: accumulate into first_stream (directory index) let rec = &mut index.records[base_ri]; rec.stdinfo.set_directory(true); - rec.first_stream.flags = 0; // type_name_id=0 for $I30 + // type_name_id=0 for $I30 + rec.first_stream.flags = + u8::from(is_sparse) | (u8::from(is_resident) << 1_u8); rec.first_stream.size.length = rec.first_stream.size.length.saturating_add(size); @@ -624,7 +634,9 @@ pub fn process_record(data: &[u8], frs: u64, index: &mut MftIndex, name_buf: &mu rec.first_stream.size.length.saturating_add(size); rec.first_stream.size.allocated = rec.first_stream.size.allocated.saturating_add(alloc); - rec.first_stream.flags = 8_u8 << 2_u8; // type_name_id=8 for $DATA + // type_name_id=8 for $DATA + rec.first_stream.flags = + u8::from(is_sparse) | (u8::from(is_resident) << 1_u8) | (8_u8 << 2_u8); } else if attr_type == AttributeType::DATA_TYPE && aname_len > 0 { // Named $DATA: ADS (user-visible stream). // Output layer filters internal streams. @@ -651,7 +663,9 @@ pub fn process_record(data: &[u8], frs: u64, index: &mut MftIndex, name_buf: &mu }, next_entry: NO_ENTRY, name: nr, - flags: 8_u8 << 2_u8, + flags: u8::from(is_sparse) + | (u8::from(is_resident) << 1_u8) + | (8_u8 << 2_u8), _pad0: [0; 3], }); @@ -681,7 +695,7 @@ pub fn process_record(data: &[u8], frs: u64, index: &mut MftIndex, name_buf: &mu allocated: alloc, }, next_entry: NO_ENTRY, - flags: 0, + flags: u8::from(is_sparse) | (u8::from(is_resident) << 1_u8), }); // Chain to record's internal stream list diff --git a/crates/uffs-mft/src/parse/direct_index.rs b/crates/uffs-mft/src/parse/direct_index.rs index 5d79372bf..76e13a2c1 100644 --- a/crates/uffs-mft/src/parse/direct_index.rs +++ b/crates/uffs-mft/src/parse/direct_index.rs @@ -3,6 +3,10 @@ //! Single-pass direct-to-index parser. //! +//! Exception: Performance-critical single-pass MFT record parser; monolithic +//! attribute-dispatch loop kept together for cache locality and to mirror the +//! NTFS on-disk attribute layout one arm at a time. +//! //! This module implements the high-performance single-pass parser that builds //! an `MftIndex` directly from raw MFT records without creating intermediate //! `ParsedRecord` allocations. @@ -40,7 +44,8 @@ use zerocopy::FromBytes as _; use super::direct_index_extension::parse_extension_to_index; use super::index_helpers::{ - add_child_entry, add_link_to_index, add_stream_to_index, chain_links, chain_streams, + StreamEntry, add_child_entry, add_link_to_index, add_stream_to_index, chain_links, + chain_streams, }; use crate::index::{nonneg_to_u64, u32_as_usize}; @@ -143,8 +148,9 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf let mut name_parse_counter: u16 = 0; let mut default_size = 0_u64; let mut default_allocated = 0_u64; - // ADS: (stream_name, size, allocated) - let mut additional_streams: SmallVec<[(String, u64, u64); 4]> = SmallVec::new(); + let mut default_is_sparse = false; + let mut default_is_resident = false; + let mut additional_streams: SmallVec<[StreamEntry; 4]> = SmallVec::new(); // Internal streams for tree-metrics (size, allocated) let internal_streams: SmallVec<[(u64, u64); 4]> = SmallVec::new(); let mut reparse_tag: u32 = 0; @@ -298,10 +304,20 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf } }; + // WI-5.2-adjacent correctness fix: `is_resident` was already + // computed above (`attr_header.is_non_resident == 0`) but + // never carried into the index; `is_sparse` lives in the + // attribute header's own `flags` (ATTR_IS_SPARSE = 0x8000), + // already-parsed data — both are free to read, no new I/O. + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + if name_len == 0 { // Default stream default_size = size; default_allocated = allocated; + default_is_sparse = is_sparse; + default_is_resident = is_resident; } else { // Alternate Data Stream (ADS) let name_offset = offset + usize::from(attr_header.name_offset); @@ -317,7 +333,13 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf // ALL named $DATA streams create regular stream entries. // Internal ones are filtered from // output by is_internal_windows_stream in the output layer. - additional_streams.push((stream_name, size, allocated)); + additional_streams.push(( + stream_name, + size, + allocated, + is_sparse, + is_resident, + )); } } } @@ -353,7 +375,15 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf }; // Add $REPARSE_POINT as a stream (contributes to stream counting) - additional_streams.push((String::from("$REPARSE"), rp_size, rp_allocated)); + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + additional_streams.push(( + String::from("$REPARSE"), + rp_size, + rp_allocated, + is_sparse, + is_resident, + )); } Some( AttributeType::IndexRoot | AttributeType::IndexAllocation | AttributeType::Bitmap, @@ -454,7 +484,15 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf } else { attr_name }; - additional_streams.push((stream_name, size, allocated)); + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + additional_streams.push(( + stream_name, + size, + allocated, + is_sparse, + is_resident, + )); } } } @@ -545,7 +583,9 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf } else { attr_name }; - additional_streams.push((stream_name, size, allocated)); + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + additional_streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } _ => { @@ -611,7 +651,9 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf } else { attr_name }; - additional_streams.push((stream_name, size, allocated)); + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + additional_streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } } @@ -646,7 +688,9 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf let additional_stream_count = additional_streams.len(); let stream_indices: Vec = additional_streams .into_iter() - .map(|(name, size, alloc)| add_stream_to_index(index, &name, size, alloc)) + .map(|(name, size, alloc, is_sparse, is_resident)| { + add_stream_to_index(index, &name, size, alloc, is_sparse, is_resident) + }) .collect(); // Setup record and chain streams. @@ -658,6 +702,9 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf length: default_size, allocated: default_allocated, }; + record.first_stream.flags = u8::from(default_is_sparse) + | (u8::from(default_is_resident) << 1_u8) + | (8_u8 << 2_u8); if !stream_indices.is_empty() { chain_streams(index, &stream_indices); @@ -693,7 +740,9 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf let additional_stream_count = additional_streams.len(); let stream_indices: Vec = additional_streams .into_iter() - .map(|(name, size, alloc)| add_stream_to_index(index, &name, size, alloc)) + .map(|(name, size, alloc, is_sparse, is_resident)| { + add_stream_to_index(index, &name, size, alloc, is_sparse, is_resident) + }) .collect(); // Ensure parent exists (create placeholder if needed) - do this before @@ -716,6 +765,8 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf length: default_size, allocated: default_allocated, }; + record.first_stream.flags = + u8::from(default_is_sparse) | (u8::from(default_is_resident) << 1_u8) | (8_u8 << 2_u8); record.first_name = LinkInfo { next_entry: NO_ENTRY, name: name_ref, diff --git a/crates/uffs-mft/src/parse/direct_index_extension.rs b/crates/uffs-mft/src/parse/direct_index_extension.rs index e0d513808..118e9b4a0 100644 --- a/crates/uffs-mft/src/parse/direct_index_extension.rs +++ b/crates/uffs-mft/src/parse/direct_index_extension.rs @@ -41,9 +41,31 @@ use core::mem::size_of; use smallvec::SmallVec; use zerocopy::FromBytes as _; -use super::index_helpers::{add_link_to_index, add_stream_to_index}; +use super::index_helpers::{StreamEntry, add_link_to_index, add_stream_to_index}; use crate::index::{frs_to_usize, len_to_u16, len_to_u32, nonneg_to_u64, u32_as_usize}; +/// Read a little-endian u16 from the given offset, returning 0 if out of +/// bounds. WI-5.2: mirrors `direct_index.rs`'s helper of the same name — see +/// its doc comment for why the outer attribute-length gate alone doesn't +/// guarantee these fixed fields are in bounds. +#[inline] +fn rd_u16(buf: &[u8], off: usize) -> u16 { + off.checked_add(2) + .and_then(|end| buf.get(off..end)) + .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) + .map_or(0, u16::from_le_bytes) +} + +/// Read a little-endian u32 from the given offset, returning 0 if out of +/// bounds. See [`rd_u16`] for the rationale. +#[inline] +fn rd_u32(buf: &[u8], off: usize) -> u32 { + off.checked_add(4) + .and_then(|end| buf.get(off..end)) + .and_then(|sl| <[u8; 4]>::try_from(sl).ok()) + .map_or(0, u32::from_le_bytes) +} + /// Parses an extension record and adds its names/streams to the base record. /// /// Extension records contain additional `$FILE_NAME` attributes (hard links) @@ -100,13 +122,15 @@ pub(super) fn parse_extension_to_index( // Collect names and streams from extension record let mut names: SmallVec<[(String, u64); 4]> = SmallVec::new(); - let mut streams: SmallVec<[(String, u64, u64); 4]> = SmallVec::new(); + let mut streams: SmallVec<[StreamEntry; 4]> = SmallVec::new(); let ext_internal_streams: SmallVec<[(u64, u64); 4]> = SmallVec::new(); let mut dir_index_size: u64 = 0; let mut dir_index_allocated: u64 = 0; // Default $DATA stream (unnamed, name_len == 0) found in extension record let mut default_data_size: u64 = 0; let mut default_data_allocated: u64 = 0; + let mut default_data_is_sparse = false; + let mut default_data_is_resident = false; let mut found_default_data = false; while offset + size_of::() <= max_offset { @@ -128,10 +152,7 @@ pub(super) fn parse_extension_to_index( Some(AttributeType::FileName) => { // Parse $FILE_NAME attribute if attr_header.is_non_resident == 0 { - let value_offset_bytes = &data[offset + 20..offset + 22]; - let value_offset = usize::from(u16::from_le_bytes( - value_offset_bytes.try_into().unwrap_or([0, 0]), - )); + let value_offset = usize::from(rd_u16(data, offset + 20)); let fn_offset = offset + value_offset; if fn_offset + size_of::() <= data.len() { let fn_attr = match FileNameAttribute::read_from_prefix(&data[fn_offset..]) @@ -217,10 +238,17 @@ pub(super) fn parse_extension_to_index( } }; + // Already-parsed attribute-header data, free to read: see + // `direct_index.rs`'s identical fix for is_sparse/is_resident. + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + if name_len == 0 { // Default $DATA stream — update base record size default_data_size = size; default_data_allocated = allocated; + default_data_is_sparse = is_sparse; + default_data_is_resident = is_resident; found_default_data = true; } else { // ADS (named stream) @@ -237,18 +265,14 @@ pub(super) fn parse_extension_to_index( // ALL named $DATA streams create regular // stream entries. Internal ones are filtered from // output by is_internal_windows_stream in the output layer. - streams.push((stream_name, size, allocated)); + streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } } Some(AttributeType::ReparsePoint) => { // Parse $REPARSE_POINT - add as stream let (rp_size, rp_allocated) = if attr_header.is_non_resident == 0 { - let value_length_bytes = &data[offset + 16..offset + 20]; - let value_length = u64::from(u32::from_le_bytes( - value_length_bytes.try_into().unwrap_or([0; 4]), - )); - (value_length, 0_u64) + (u64::from(rd_u32(data, offset + 16)), 0_u64) } else { let nr_offset = offset + 16; if nr_offset + 48 <= data.len() { @@ -262,7 +286,15 @@ pub(super) fn parse_extension_to_index( (0_u64, 0_u64) } }; - streams.push((String::from("$REPARSE"), rp_size, rp_allocated)); + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + streams.push(( + String::from("$REPARSE"), + rp_size, + rp_allocated, + is_sparse, + is_resident, + )); } Some( AttributeType::IndexRoot | AttributeType::IndexAllocation | AttributeType::Bitmap, @@ -297,11 +329,7 @@ pub(super) fn parse_extension_to_index( if is_i30 { // Accumulate $I30 sizes if attr_header.is_non_resident == 0 { - let value_length_bytes = &data[offset + 16..offset + 20]; - let value_length = u64::from(u32::from_le_bytes( - value_length_bytes.try_into().unwrap_or([0; 4]), - )); - dir_index_size += value_length; + dir_index_size += u64::from(rd_u32(data, offset + 16)); } else { let nr_offset = offset + 16; if nr_offset + 48 <= data.len() { @@ -333,11 +361,7 @@ pub(super) fn parse_extension_to_index( if is_primary { let (size, allocated) = if attr_header.is_non_resident == 0 { - let value_length_bytes = &data[offset + 16..offset + 20]; - let value_length = u64::from(u32::from_le_bytes( - value_length_bytes.try_into().unwrap_or([0; 4]), - )); - (value_length, 0_u64) + (u64::from(rd_u32(data, offset + 16)), 0_u64) } else { let nr_offset = offset + 16; if nr_offset + 48 <= data.len() { @@ -365,7 +389,9 @@ pub(super) fn parse_extension_to_index( } else { attr_name }; - streams.push((stream_name, size, allocated)); + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } } @@ -416,11 +442,7 @@ pub(super) fn parse_extension_to_index( }; let (size, allocated) = if attr_header.is_non_resident == 0 { - let value_length_bytes = &data[offset + 16..offset + 20]; - let value_length = u64::from(u32::from_le_bytes( - value_length_bytes.try_into().unwrap_or([0; 4]), - )); - (value_length, 0_u64) + (u64::from(rd_u32(data, offset + 16)), 0_u64) } else { let nr_offset = offset + 16; if nr_offset + 48 <= data.len() { @@ -458,7 +480,9 @@ pub(super) fn parse_extension_to_index( } else { attr_name }; - streams.push((stream_name, size, allocated)); + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } Some(AttributeType::StandardInformation) => { @@ -503,11 +527,7 @@ pub(super) fn parse_extension_to_index( }; let (size, allocated) = if attr_header.is_non_resident == 0 { - let value_length_bytes = &data[offset + 16..offset + 20]; - let value_length = u64::from(u32::from_le_bytes( - value_length_bytes.try_into().unwrap_or([0; 4]), - )); - (value_length, 0_u64) + (u64::from(rd_u32(data, offset + 16)), 0_u64) } else { let nr_offset = offset + 16; if nr_offset + 48 <= data.len() { @@ -528,7 +548,9 @@ pub(super) fn parse_extension_to_index( } else { attr_name }; - streams.push((stream_name, size, allocated)); + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } } @@ -557,7 +579,9 @@ pub(super) fn parse_extension_to_index( .collect(); let stream_indices: Vec = streams .iter() - .map(|(name, size, alloc)| add_stream_to_index(index, name, *size, *alloc)) + .map(|(name, size, alloc, is_sparse, is_resident)| { + add_stream_to_index(index, name, *size, *alloc, *is_sparse, *is_resident) + }) .collect(); // Ensure parent directories exist for the new names. Parser-local @@ -698,6 +722,9 @@ pub(super) fn parse_extension_to_index( // Base has no $DATA — use extension's values record.first_stream.size.length = default_data_size; record.first_stream.size.allocated = default_data_allocated; + record.first_stream.flags = u8::from(default_data_is_sparse) + | (u8::from(default_data_is_resident) << 1_u8) + | (8_u8 << 2_u8); } else { // Base has partial $DATA — accumulate extension values record.first_stream.size.length = record diff --git a/crates/uffs-mft/src/parse/index_helpers.rs b/crates/uffs-mft/src/parse/index_helpers.rs index 79d33b715..5a7daedc9 100644 --- a/crates/uffs-mft/src/parse/index_helpers.rs +++ b/crates/uffs-mft/src/parse/index_helpers.rs @@ -11,6 +11,11 @@ use crate::index::{ len_to_u16, len_to_u32, u32_as_usize, }; +/// A pending stream, collected while walking a record's attributes and +/// applied to the index in one batch via [`add_stream_to_index`]: +/// `(name, size, allocated, is_sparse, is_resident)`. +pub(crate) type StreamEntry = (String, u64, u64, bool, bool); + /// Adds a stream to the index and returns its index. #[inline] pub(crate) fn add_stream_to_index( @@ -18,6 +23,8 @@ pub(crate) fn add_stream_to_index( stream_name: &str, stream_size: u64, stream_allocated: u64, + is_sparse: bool, + is_resident: bool, ) -> u32 { let stream_name_offset = index.add_name(stream_name); let stream_name_len = stream_name.len(); @@ -38,8 +45,9 @@ pub(crate) fn add_stream_to_index( }, next_entry: NO_ENTRY, name: stream_name_ref, - // type_name_id=8 for $DATA (0x80 >> 4), stored in bits 2-7 - flags: 8 << 2, + // bit0=is_sparse, bit1=is_resident, type_name_id=8 for $DATA + // (0x80 >> 4) in bits 2-7. + flags: u8::from(is_sparse) | (u8::from(is_resident) << 1) | (8 << 2), _pad0: [0; 3], }); stream_idx diff --git a/scripts/ci/file_size_exceptions.txt b/scripts/ci/file_size_exceptions.txt index 7c0b95561..40be1cc88 100644 --- a/scripts/ci/file_size_exceptions.txt +++ b/scripts/ci/file_size_exceptions.txt @@ -7,6 +7,7 @@ crates/uffs-core/src/search/field/field_metadata.rs|PERMANENT: Single const fn m crates/uffs-core/src/search/filters/tests.rs|PERMANENT: Integration test suite for filter pipeline; splitting further would scatter related test fixtures crates/uffs-core/src/search/filters/mod.rs|PERMANENT: Cohesive SearchFilters/SearchFilterParams definitions + from_params construction; kept together so the full per-field filter contract is auditable in one place crates/uffs-client/src/schema/field_metadata.rs|PERMANENT: Single const fn match table — one FieldMeta per FieldId variant; mirrors uffs-core version +crates/uffs-mft/src/parse/direct_index.rs|PERMANENT: Performance-critical single-pass MFT record parser; monolithic loop for cache locality crates/uffs-mft/src/parse/direct_index_extension.rs|PERMANENT: Extension record parser for the direct-to-index pipeline; same structural reasoning as its base-record counterpart crates/uffs-mft/src/reader/index_read.rs|PERMANENT: Single impl MftReader block with tightly coupled cfg-gated pipeline stages crates/uffs-diag/src/bin/compare_scan_parity.rs|PERMANENT: Standalone diagnostic binary; single-file readability outweighs LOC policy for tooling From 33364117fd50a9e65586a9314e1ada347f585d8a Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:35:25 -0700 Subject: [PATCH 4/7] fix(mft): populate FileRecord.lsn/namespace/fn_* timestamps on both production parsers FileRecord already declares lsn (Log File Sequence Number, from the header's own log_file_sequence_number), namespace (the primary name's $FILE_NAME namespace), and fn_created/fn_modified/fn_accessed/fn_mft_changed ($FILE_NAME's own timestamps, which often differ from $STANDARD_INFORMATION -- e.g. timestomping alters STD_INFO but leaves FILE_NAME original). All five fields are exactly the same bug class as usn/security_id/owner_id and is_sparse/ is_resident fixed in the previous two commits: already declared on the struct, already fully decoded in memory (the header and $FILE_NAME attribute are both read in full regardless), but never copied onto the record by either unified.rs's process_record or direct_index.rs's parse_record_to_index -- so every record silently read back lsn=0, namespace=0, and all four fn_* timestamps=0, no matter what the disk actually held. - unified.rs: lsn set alongside the existing base-record-only sequence_number (extension records carry their own, differently-scoped LSN, matching the sequence_number precedent already established there). namespace/fn_* set at the same point $FILE_NAME's name/parent_frs are already written, matching the file's existing "push-to-front, most recent $FILE_NAME wins" model. - direct_index.rs: same fields threaded through via new locals (primary_fn_created/modified/accessed/mft_changed), captured whenever a name is chosen as primary (the existing Win32 > POSIX > DOS priority logic), written to the record alongside sequence_number. While auditing this, found and fixed a related pre-existing gap in direct_index.rs: the "no $FILE_NAME in base record" early-return path (the name arrives later via an extension record) never set sequence_number or lsn at all -- and nothing else in the pipeline would either, since an extension record's header carries its own, different-meaning sequence/LSN. A record whose name lands in an extension record was silently missing its own identity fields. Added regression tests: one asserting lsn/namespace/fn_* reach both production parsers from a real $FILE_NAME attribute, one asserting sequence_number/lsn are still set via the no-name early-return path. --- crates/uffs-mft/src/io/parser/mod.rs | 131 ++++++++++++++++++++++ crates/uffs-mft/src/io/parser/unified.rs | 24 ++++ crates/uffs-mft/src/parse/direct_index.rs | 37 +++++- scripts/ci/file_size_exceptions.txt | 1 + 4 files changed, 192 insertions(+), 1 deletion(-) diff --git a/crates/uffs-mft/src/io/parser/mod.rs b/crates/uffs-mft/src/io/parser/mod.rs index 10ad7cf50..acec683d1 100644 --- a/crates/uffs-mft/src/io/parser/mod.rs +++ b/crates/uffs-mft/src/io/parser/mod.rs @@ -316,6 +316,137 @@ mod tests { assert!(!direct_stream.is_resident()); } + /// Regression pin: `FileRecord.lsn` (from the header's own + /// `log_file_sequence_number`) and `$FILE_NAME`'s own + /// `namespace`/timestamps (which often differ from + /// `$STANDARD_INFORMATION` — e.g. timestomping alters `STD_INFO` but + /// leaves `FILE_NAME` original) must reach both production parsers. All + /// five fields already existed on `FileRecord`; the header and + /// `$FILE_NAME` attribute are both already fully decoded in memory by + /// the time these values are read, so populating them is free. + #[test] + fn lsn_and_file_name_own_fields_reach_both_production_parsers() { + let lsn = 0x1122_3344_5566_7788_u64; + let sequence_number = 0x2222_u16; + let namespace = 1_u8; // Win32 + let fn_created = 10_i64; + let fn_modified = 20_i64; + let fn_accessed = 30_i64; + let fn_mft_changed = 40_i64; + + let mut fn_payload = Vec::new(); + fn_payload.extend_from_slice(&0_u64.to_le_bytes()); // parent_directory + fn_payload.extend_from_slice(&fn_created.to_le_bytes()); + fn_payload.extend_from_slice(&fn_modified.to_le_bytes()); + fn_payload.extend_from_slice(&fn_mft_changed.to_le_bytes()); + fn_payload.extend_from_slice(&fn_accessed.to_le_bytes()); + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // allocated_size + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // data_size + fn_payload.extend_from_slice(&0_u32.to_le_bytes()); // file_attributes + fn_payload.extend_from_slice(&0_u16.to_le_bytes()); // packed_ea_size + fn_payload.extend_from_slice(&0_u16.to_le_bytes()); // reserved + fn_payload.push(1); // file_name_length = 1 char + fn_payload.push(namespace); + fn_payload.extend_from_slice(&0x0061_u16.to_le_bytes()); // "a" + let file_name_total_len = u32::try_from(24 + fn_payload.len()).expect("fits in u32"); + + let mut record = RecordBuilder::new(56) + .attr(0x30, file_name_total_len, 0, 0, 0) + .raw( + &u32::try_from(fn_payload.len()) + .expect("fits in u32") + .to_le_bytes(), + ) + .raw(&24_u16.to_le_bytes()) + .raw(&[0_u8; 2]) + .raw(&fn_payload) + .build(); + + let total_len = u32::try_from(record.len()).expect("fits in u32"); + record + .get_mut(24..28) + .expect("record well over 28 bytes") + .copy_from_slice(&total_len.to_le_bytes()); + // Header: log_file_sequence_number @ offset 8 (u64), sequence_number + // @ offset 16 (u16). + record + .get_mut(8..16) + .expect("record well over 16 bytes") + .copy_from_slice(&lsn.to_le_bytes()); + record + .get_mut(16..18) + .expect("record well over 18 bytes") + .copy_from_slice(&sequence_number.to_le_bytes()); + + let mut unified_index = MftIndex::new(crate::platform::DriveLetter::C); + let mut name_buf = String::new(); + process_record(&record, 42, &mut unified_index, &mut name_buf); + let unified_rec = unified_index + .find(crate::frs::Frs::new(42)) + .expect("process_record must create the base record"); + assert_eq!(unified_rec.lsn, lsn); + assert_eq!(unified_rec.sequence_number, sequence_number); + assert_eq!(unified_rec.namespace, namespace); + assert_eq!(unified_rec.fn_created, fn_created); + assert_eq!(unified_rec.fn_modified, fn_modified); + assert_eq!(unified_rec.fn_accessed, fn_accessed); + assert_eq!(unified_rec.fn_mft_changed, fn_mft_changed); + + let mut direct_index = MftIndex::new(crate::platform::DriveLetter::C); + assert!(crate::parse::parse_record_to_index( + &record, + 42, + &mut direct_index + )); + let direct_rec = direct_index + .find(crate::frs::Frs::new(42)) + .expect("parse_record_to_index must create the base record"); + assert_eq!(direct_rec.lsn, lsn); + assert_eq!(direct_rec.sequence_number, sequence_number); + assert_eq!(direct_rec.namespace, namespace); + assert_eq!(direct_rec.fn_created, fn_created); + assert_eq!(direct_rec.fn_modified, fn_modified); + assert_eq!(direct_rec.fn_accessed, fn_accessed); + assert_eq!(direct_rec.fn_mft_changed, fn_mft_changed); + } + + /// Regression pin: a record whose base segment has **no** `$FILE_NAME` + /// at all (name arrives later via an extension record — the normal case + /// for files with enough attributes to overflow the base MFT record) + /// must still get `sequence_number`/`lsn` from its own header. Before + /// this fix, `parse_record_to_index`'s no-name early-return path set + /// neither, and nothing else in the pipeline ever would (an extension + /// record's header carries a different, per-segment sequence/LSN). + #[test] + fn sequence_number_and_lsn_set_even_without_a_base_file_name() { + let lsn = 0xAABB_CCDD_EEFF_0011_u64; + let sequence_number = 0x3333_u16; + + let mut record = RecordBuilder::new(56).build(); + record + .get_mut(8..16) + .expect("record well over 16 bytes") + .copy_from_slice(&lsn.to_le_bytes()); + record + .get_mut(16..18) + .expect("record well over 18 bytes") + .copy_from_slice(&sequence_number.to_le_bytes()); + + let mut direct_index = MftIndex::new(crate::platform::DriveLetter::C); + // Returns false (no name found yet), but must still create the + // record with its header-derived identity fields set. + assert!(!crate::parse::parse_record_to_index( + &record, + 42, + &mut direct_index + )); + let direct_rec = direct_index + .find(crate::frs::Frs::new(42)) + .expect("the no-name path must still create the base record"); + assert_eq!(direct_rec.sequence_number, sequence_number); + assert_eq!(direct_rec.lsn, lsn); + } + // ── WI-5.2 panic-resistance corpus ────────────────────────────── // // The daemon builds with `panic = "abort"`: a single parser panic on a diff --git a/crates/uffs-mft/src/io/parser/unified.rs b/crates/uffs-mft/src/io/parser/unified.rs index 320caefd0..0e8edd26f 100644 --- a/crates/uffs-mft/src/io/parser/unified.rs +++ b/crates/uffs-mft/src/io/parser/unified.rs @@ -3,6 +3,10 @@ //! Unified MFT record processor. //! +//! Exception: single-pass MFT record processor; the monolithic attribute +//! loop is kept together for cache locality and to mirror the NTFS on-disk +//! attribute layout one arm at a time. +//! //! ONE function processes ALL records (base AND extension) through the SAME //! attribute loop. This eliminates the dual-parser architecture that caused //! name-ordering and stream-counting discrepancies. @@ -353,6 +357,12 @@ pub fn process_record(data: &[u8], frs: u64, index: &mut MftIndex, name_buf: &mu // sequence, so only a base record's header sets the file's sequence. if header.is_base_record() { index.records[base_ri].sequence_number = header.sequence_number; + // Log File Sequence Number, correlates with the $LogFile journal + // (forensic value) — already-parsed header data, free to store. + // Same base-record-only scope as sequence_number above: an + // extension record segment has its own LSN, a different concept + // from the file's own. + index.records[base_ri].lsn = header.log_file_sequence_number; } // ── Attribute loop ───────────────────────────────────────────────── @@ -467,6 +477,20 @@ pub fn process_record(data: &[u8], frs: u64, index: &mut MftIndex, name_buf: &mu index.records[base_ri].first_name.parent_frs = crate::frs::ParentFrs::new(parent_frs); + // $FILE_NAME's own namespace/timestamps (often + // differ from $STANDARD_INFORMATION — e.g. + // timestomping leaves STD_INFO altered but + // FILE_NAME original). `fn_attr` is already fully + // decoded above; these are free reads of already- + // resident memory. Push-to-front: whichever name + // is currently "first" also owns these fields. + let rec = &mut index.records[base_ri]; + rec.namespace = fn_attr.file_name_namespace; + rec.fn_created = fn_attr.creation_time; + rec.fn_modified = fn_attr.modification_time; + rec.fn_accessed = fn_attr.access_time; + rec.fn_mft_changed = fn_attr.mft_change_time; + // Build parent-child relationship. // name_index = name_count BEFORE increment let name_index = index.records[base_ri].name_count; diff --git a/crates/uffs-mft/src/parse/direct_index.rs b/crates/uffs-mft/src/parse/direct_index.rs index 76e13a2c1..dd0e7a8e7 100644 --- a/crates/uffs-mft/src/parse/direct_index.rs +++ b/crates/uffs-mft/src/parse/direct_index.rs @@ -146,6 +146,14 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf let mut primary_name: Option<(String, u64, u8, u16)> = None; // (name, parent_frs, namespace, parse_index) let mut additional_names: SmallVec<[(String, u64, u16); 4]> = SmallVec::new(); let mut name_parse_counter: u16 = 0; + // $FILE_NAME's own timestamps for whichever name is currently primary + // (often differ from $STANDARD_INFORMATION). Only the primary name's + // values are stored here since FileRecord carries just one set — see + // `FileRecord::fn_created`'s doc. + let mut primary_fn_created = 0_i64; + let mut primary_fn_modified = 0_i64; + let mut primary_fn_accessed = 0_i64; + let mut primary_fn_mft_changed = 0_i64; let mut default_size = 0_u64; let mut default_allocated = 0_u64; let mut default_is_sparse = false; @@ -234,6 +242,13 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf )); } primary_name = Some((name, parent_frs, namespace, parse_idx)); + // $FILE_NAME's own timestamps for the new + // primary name — free reads of the + // already-decoded `fn_attr`. + primary_fn_created = fn_attr.creation_time; + primary_fn_modified = fn_attr.modification_time; + primary_fn_accessed = fn_attr.access_time; + primary_fn_mft_changed = fn_attr.mft_change_time; } else { additional_names.push((name, parent_frs, parse_idx)); } @@ -674,7 +689,7 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf // Handle records without a filename in the base record // The $FILE_NAME may be in an extension record - we still need to store stdinfo - let (name, parent_frs, _namespace, primary_parse_index) = match primary_name { + let (name, parent_frs, primary_namespace, primary_parse_index) = match primary_name { Some(n) => n, None => { // No $FILE_NAME in base record - store stdinfo anyway @@ -697,6 +712,15 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf // Boundary: lift the raw `u64` FRS argument (kernel/USN buffer) // into a typed `Frs` once for the typed index API. let record = index.get_or_create(crate::frs::Frs::new(frs)); + // Pre-existing gap, fixed alongside this one: this early-return + // path never set sequence_number/lsn at all (only the main path + // below did), so a record whose $FILE_NAME arrives via a later + // extension record got neither — nothing else in the pipeline + // sets them for it (extension records carry their own, + // different-meaning sequence/LSN, per unified.rs's identical + // base-record-only scoping). + record.sequence_number = header.sequence_number; + record.lsn = header.log_file_sequence_number; record.stdinfo = std_info; record.first_stream.size = SizeInfo { length: default_size, @@ -760,6 +784,17 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf // `file_ref`; without it a delete-then-reuse of an MFT slot is invisible to // the snapshot diff (the slot number alone is stable across reuse). record.sequence_number = header.sequence_number; + record.lsn = header.log_file_sequence_number; + // $FILE_NAME's own namespace/timestamps for the primary name (often + // differ from $STANDARD_INFORMATION — e.g. timestomping alters + // STD_INFO but leaves FILE_NAME original). Captured above per-name; + // only the primary's values are stored since FileRecord carries just + // one set. + record.namespace = primary_namespace; + record.fn_created = primary_fn_created; + record.fn_modified = primary_fn_modified; + record.fn_accessed = primary_fn_accessed; + record.fn_mft_changed = primary_fn_mft_changed; record.stdinfo = std_info; record.first_stream.size = SizeInfo { length: default_size, diff --git a/scripts/ci/file_size_exceptions.txt b/scripts/ci/file_size_exceptions.txt index 40be1cc88..c30174703 100644 --- a/scripts/ci/file_size_exceptions.txt +++ b/scripts/ci/file_size_exceptions.txt @@ -8,6 +8,7 @@ crates/uffs-core/src/search/filters/tests.rs|PERMANENT: Integration test suite f crates/uffs-core/src/search/filters/mod.rs|PERMANENT: Cohesive SearchFilters/SearchFilterParams definitions + from_params construction; kept together so the full per-field filter contract is auditable in one place crates/uffs-client/src/schema/field_metadata.rs|PERMANENT: Single const fn match table — one FieldMeta per FieldId variant; mirrors uffs-core version crates/uffs-mft/src/parse/direct_index.rs|PERMANENT: Performance-critical single-pass MFT record parser; monolithic loop for cache locality +crates/uffs-mft/src/io/parser/unified.rs|PERMANENT: Single-pass unified MFT record processor; monolithic attribute loop for cache locality, same reasoning as direct_index.rs crates/uffs-mft/src/parse/direct_index_extension.rs|PERMANENT: Extension record parser for the direct-to-index pipeline; same structural reasoning as its base-record counterpart crates/uffs-mft/src/reader/index_read.rs|PERMANENT: Single impl MftReader block with tightly coupled cfg-gated pipeline stages crates/uffs-diag/src/bin/compare_scan_parity.rs|PERMANENT: Standalone diagnostic binary; single-file readability outweighs LOC policy for tooling From 74cb16fe8b2eec5b202dc697ee7e640b10b91b02 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:56:22 -0700 Subject: [PATCH 5/7] refactor(mft): bring direct_index.rs and unified.rs under the 800-LOC policy for real The previous three commits' fixes pushed both files over the file-size policy threshold. Fixed it properly instead of adding policy exceptions: - direct_index.rs: extracted four small helpers (is_primary_attribute/extract_attr_name/read_size_allocated/ resident_and_sparse) that were each duplicated verbatim across 4-5 match arms (Data, ReparsePoint, IndexRoot-family, ObjectId-family, catch-all) -- genuine deduplication, not just fewer lines. 856 -> 724 lines, and every one of those match arms is now shorter and less repetitive to read, not just smaller on a line-count report. - unified.rs: split the NTFS UTF-16/WTF-8 name-decoding cluster (decode_utf16le_into, decode_name_u16, wtf8_from_utf16le, store_name_lossless, the LOSSY_NAME_COUNT tally) into a new sibling module, unified/name_codec.rs. That cluster has zero dependency on process_record's attribute-loop state -- it only needs MftIndex -- so it was always a separable concern, just never separated. Re-exported decode_name_u16/lossy_name_count from unified.rs so the ~9 other modules that call them via crate::io::parser::unified::* keep working unchanged. 803 -> 559 lines in unified.rs; name_codec.rs is 270 lines, comfortably under the threshold on its own. Also extracted a small resident_value_offset(data, attr_offset) helper in unified.rs for the value-offset-field read duplicated between the $FILE_NAME and $REPARSE_POINT arms. All prior documentation is preserved verbatim -- this is a structural split, not a comment trim. cargo test -p uffs-mft: 260/260 passing, identical to before the split (the extraction is behavior-preserving by construction: same logic, moved, not rewritten). Neither file needs a scripts/ci/file_size_exceptions.txt entry anymore. --- crates/uffs-mft/src/io/parser/unified.rs | 466 ++++-------------- .../src/io/parser/unified/name_codec.rs | 270 ++++++++++ crates/uffs-mft/src/parse/direct_index.rs | 352 +++++-------- scripts/ci/file_size_exceptions.txt | 2 - 4 files changed, 489 insertions(+), 601 deletions(-) create mode 100644 crates/uffs-mft/src/io/parser/unified/name_codec.rs diff --git a/crates/uffs-mft/src/io/parser/unified.rs b/crates/uffs-mft/src/io/parser/unified.rs index 0e8edd26f..951933ef6 100644 --- a/crates/uffs-mft/src/io/parser/unified.rs +++ b/crates/uffs-mft/src/io/parser/unified.rs @@ -3,10 +3,6 @@ //! Unified MFT record processor. //! -//! Exception: single-pass MFT record processor; the monolithic attribute -//! loop is kept together for cache locality and to mirror the NTFS on-disk -//! attribute layout one arm at a time. -//! //! ONE function processes ALL records (base AND extension) through the SAME //! attribute loop. This eliminates the dual-parser architecture that caused //! name-ordering and stream-counting discrepancies. @@ -36,261 +32,16 @@ use crate::ntfs::{ file_reference_to_frs, }; -/// Decode a UTF-16LE byte slice into `out`, replacing unpaired surrogates -/// with U+FFFD. Returns the number of U+FFFD replacements emitted -/// (`0` = lossless). -/// -/// This avoids the per-call `SmallVec` + `String` allocation that -/// `String::from_utf16_lossy` requires, and — unlike `from_utf16_lossy` — -/// surfaces the substitution count so name loss at the NTFS boundary is -/// measured, not silent (Category 4, WI-4.1). -#[inline] -fn decode_utf16le_into(bytes: &[u8], out: &mut String) -> u32 { - out.clear(); - let mut replacements: u32 = 0; - let mut i = 0_usize; - while let Some(pair) = i - .checked_add(2) - .and_then(|end| bytes.get(i..end)) - .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) - { - let code = u16::from_le_bytes(pair); - // `i` indexes a &[u8]; it cannot exceed `bytes.len()` (≤ isize::MAX), - // so `+= 2` cannot overflow usize. saturating_add keeps it total. - i = i.saturating_add(2); - match code { - // High surrogate - 0xD800..=0xDBFF => { - if let Some(low_pair) = i - .checked_add(2) - .and_then(|end| bytes.get(i..end)) - .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) - { - let low = u16::from_le_bytes(low_pair); - if (0xDC00..=0xDFFF).contains(&low) { - i = i.saturating_add(2); - // Bounds-proven: `code ∈ 0xD800..=0xDBFF` and - // `low ∈ 0xDC00..=0xDFFF`, so both subtractions are - // non-negative and the result is ≤ 0x10FFFF — no - // overflow/underflow is reachable. - let cp = 0x1_0000_u32 - .saturating_add((u32::from(code).saturating_sub(0xD800_u32)) << 10_u32) - .saturating_add(u32::from(low).saturating_sub(0xDC00_u32)); - if let Some(ch) = char::from_u32(cp) { - out.push(ch); - } else { - out.push(char::REPLACEMENT_CHARACTER); - replacements = replacements.saturating_add(1); - } - } else { - out.push(char::REPLACEMENT_CHARACTER); - replacements = replacements.saturating_add(1); - } - } else { - out.push(char::REPLACEMENT_CHARACTER); - replacements = replacements.saturating_add(1); - } - } - // Low surrogate without preceding high - 0xDC00..=0xDFFF => { - out.push(char::REPLACEMENT_CHARACTER); - replacements = replacements.saturating_add(1); - } - _ => { - // All non-surrogate u16 values are valid Unicode scalar values. - // `char::from_u32` is cheap for the common BMP case. - if let Some(ch) = char::from_u32(u32::from(code)) { - out.push(ch); - } - } - } - } - replacements -} - -/// Decode a `&[u16]` UTF-16 name into a fresh `String`, returning -/// `(String, replacement_count)`. Use this instead of -/// `String::from_utf16_lossy` at NTFS name boundaries so loss is counted, -/// not silent (Category 4, WI-4.1). -/// -/// Most NTFS-name call sites already hold a `Vec` / `SmallVec<[u16; N]>` -/// (the attribute decoder collects code units before stringifying), so this -/// `&[u16]` entry point avoids re-deriving a byte slice. There is exactly -/// ONE surrogate-handling implementation: this re-encodes to LE bytes and -/// routes through `decode_utf16le_into`. -#[inline] -pub(crate) fn decode_name_u16(units: &[u16]) -> (String, u32) { - let mut bytes = Vec::with_capacity(units.len().saturating_mul(2)); - for unit in units { - bytes.extend_from_slice(&unit.to_le_bytes()); - } - let mut out = String::new(); - let count = decode_utf16le_into(&bytes, &mut out); - if count > 0 { - LOSSY_NAME_COUNT.fetch_add(u64::from(count), core::sync::atomic::Ordering::Relaxed); - } - (out, count) -} +mod name_codec; -/// Re-encode a UTF-16LE byte slice **losslessly** as WTF-8 into `out`. -/// -/// Unlike [`decode_utf16le_into`] (which replaces unpaired surrogates with -/// U+FFFD for a valid-UTF-8 `String`), this preserves *every* code unit — -/// well-formed text becomes ordinary UTF-8, and an **unpaired surrogate** -/// (`0xD800..=0xDFFF` with no valid pairing) is emitted as its 3-byte WTF-8 -/// encoding (`1110_xxxx 10xx_xxxx 10xx_xxxx` over the raw 16-bit value). The -/// result is therefore byte-faithful to the on-disk NTFS name and is what the -/// byte-native search/trigram path matches against, so a file with an -/// ill-formed name remains **findable by its true name** (WI-4.4). Surrogate -/// *pairs* are combined into their astral scalar (normal 4-byte UTF-8). -/// -/// Only called on the rare lossy path (when `decode_utf16le_into` reported a -/// replacement), so its modest cost never touches the well-formed hot path. -#[expect( - clippy::arithmetic_side_effects, - reason = "all arithmetic is on values masked to ≤ 0x10FFFF / 6-bit groups; \ - the WTF-8 byte composition cannot overflow u8/u32" -)] -fn wtf8_from_utf16le(bytes: &[u8], out: &mut Vec) { - /// Low 6 bits of `x`, as a UTF-8 continuation byte (`10xx_xxxx`). - /// - /// `x & 0x3F` is in `0..=0x3F` and `0x80 | _` is in `0x80..=0xBF`, so the - /// `u8` cast is exact, never truncating. - #[expect( - clippy::cast_possible_truncation, - reason = "value is masked to 6 bits (≤ 0x3F) then OR'd with 0x80 → always ≤ 0xBF" - )] - const fn cont(x: u32) -> u8 { - (0x80_u32 | (x & 0x3F)) as u8 - } - - /// Leading byte: `prefix` OR the low `mask` bits of `cp_shifted`. - /// Callers pass a `mask` (5/4/3 bits) that bounds the residual to the - /// prefix's free bits, so the `u8` cast is exact. - #[expect( - clippy::cast_possible_truncation, - reason = "masked to ≤ 5 bits then OR'd with a fixed prefix → always ≤ 0xFF" - )] - const fn lead(prefix: u8, cp_shifted: u32, mask: u32) -> u8 { - prefix | (cp_shifted & mask) as u8 - } - - /// Push a single code point (or lone surrogate) as WTF-8 bytes. - fn push_wtf8(cp: u32, out: &mut Vec) { - match cp { - 0x0000..=0x007F => { - // ASCII: single byte, value ≤ 0x7F fits u8 exactly. - #[expect( - clippy::cast_possible_truncation, - reason = "cp ≤ 0x7F in this arm → exact u8" - )] - out.push(cp as u8); - } - // 2-byte: 110x_xxxx 10xx_xxxx (5 payload bits in the lead). - 0x0080..=0x07FF => { - out.push(lead(0xC0, cp >> 6, 0x1F)); - out.push(cont(cp)); - } - // 3-byte: BMP incl. lone surrogates 0xD800..=0xDFFF (4 lead bits). - 0x0800..=0xFFFF => { - out.push(lead(0xE0, cp >> 12, 0x0F)); - out.push(cont(cp >> 6)); - out.push(cont(cp)); - } - // 4-byte: astral from a valid surrogate pair (3 lead bits). - _ => { - out.push(lead(0xF0, cp >> 18, 0x07)); - out.push(cont(cp >> 12)); - out.push(cont(cp >> 6)); - out.push(cont(cp)); - } - } - } - - let mut i = 0_usize; - while let Some(pair) = i - .checked_add(2) - .and_then(|end| bytes.get(i..end)) - .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) - { - let code = u16::from_le_bytes(pair); - i = i.saturating_add(2); - if (0xD800..=0xDBFF).contains(&code) { - // High surrogate: combine with a following low surrogate if present. - if let Some(low) = i - .checked_add(2) - .and_then(|end| bytes.get(i..end)) - .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) - .map(u16::from_le_bytes) - .filter(|low| (0xDC00..=0xDFFF).contains(low)) - { - i = i.saturating_add(2); - let cp = 0x1_0000_u32 - + ((u32::from(code) - 0xD800_u32) << 10_u32) - + (u32::from(low) - 0xDC00_u32); - push_wtf8(cp, out); - } else { - // Unpaired high surrogate — preserve verbatim as WTF-8. - push_wtf8(u32::from(code), out); - } - } else { - // BMP scalar or unpaired low surrogate — both preserved verbatim. - push_wtf8(u32::from(code), out); - } - } -} - -/// Store a just-decoded name into the index's name buffer **losslessly**, -/// returning `(byte_offset, stored_byte_len)`. -/// -/// - `display` is the lossy `String` produced by [`decode_utf16le_into`] -/// (U+FFFD for ill-formed parts) — used as-is for the common well-formed -/// case, where its bytes are identical to the name's WTF-8. -/// - `raw_utf16le` is the original on-disk UTF-16LE byte slice for the name. -/// - `lossy` is the replacement count `decode_utf16le_into` reported. -/// -/// When `lossy == 0` (the overwhelming common case) the `display` bytes are -/// stored directly — zero extra work on the hot path. When `lossy > 0`, the -/// raw UTF-16 is re-encoded to byte-faithful WTF-8 and *those* bytes are -/// stored, so the file is findable by its true name (WI-4.4). The returned -/// length is the **stored** byte length (WTF-8 length on the lossy path), -/// which the caller records in the `IndexNameRef` so `get_name_bytes` slices -/// exactly the stored name. -fn store_name_lossless( - index: &mut MftIndex, - display: &str, - raw_utf16le: &[u8], - lossy: u32, -) -> (u32, usize) { - if lossy == 0 { - let bytes = display.as_bytes(); - (index.add_name_bytes(bytes), bytes.len()) - } else { - let mut wtf8 = Vec::with_capacity(raw_utf16le.len()); - wtf8_from_utf16le(raw_utf16le, &mut wtf8); - (index.add_name_bytes(&wtf8), wtf8.len()) - } -} - -/// Process-global tally of U+FFFD substitutions emitted by -/// [`decode_name_u16`] across all NTFS-name decodes (Category 4, WI-4.1). -/// -/// The parser call sites are spread across nine modules and do not thread a -/// stats accumulator through their (hot-path) signatures, so the count is -/// gathered here with a single relaxed atomic — cheap, lock-free, and read -/// at index-build time into the `lossy_name_count` field of -/// [`crate::index::MftStats`] for the "N filenames were stored with -/// U+FFFD" warning. `Relaxed` is -/// sufficient: it is a monotonic diagnostic counter, not a synchronisation -/// point. -pub(crate) static LOSSY_NAME_COUNT: core::sync::atomic::AtomicU64 = - core::sync::atomic::AtomicU64::new(0); - -/// Snapshot the current global lossy-name tally. -#[inline] -pub(crate) fn lossy_name_count() -> u64 { - LOSSY_NAME_COUNT.load(core::sync::atomic::Ordering::Relaxed) -} +// `decode_name_u16`/`lossy_name_count` are consumed crate-wide as +// `crate::io::parser::unified::{decode_name_u16, lossy_name_count}` (nine +// other modules); re-exporting keeps every existing call site unchanged +// even though the implementation now lives in the `name_codec` submodule. +#[cfg(test)] +use name_codec::wtf8_from_utf16le; +pub(crate) use name_codec::{decode_name_u16, lossy_name_count}; +use name_codec::{decode_utf16le_into, store_name_lossless}; /// Process a single MFT record (base OR extension) in one pass. /// @@ -418,106 +169,99 @@ pub fn process_record(data: &[u8], frs: u64, index: &mut MftIndex, name_buf: &mu // ── $FILE_NAME (0x30) ───────────────────────────────────── // Push-to-front: each new $FILE_NAME overwrites first_name. Some(AttributeType::FileName) => { - if attr_header.is_non_resident == 0 { - let vo = usize::from(rd_u16(data, offset.saturating_add(20))); - if let Some(fn_off) = offset.checked_add(vo) - && let Some(fn_slice) = data.get(fn_off..) - && let Ok((fn_attr, _)) = FileNameAttribute::read_from_prefix(fn_slice) - && fn_attr.file_name_namespace != 2 + if attr_header.is_non_resident == 0 + && let Some(fn_off) = resident_value_offset(data, offset) + && let Some(fn_slice) = data.get(fn_off..) + && let Ok((fn_attr, _)) = FileNameAttribute::read_from_prefix(fn_slice) + && fn_attr.file_name_namespace != 2 + { + // Skip DOS-only names (namespace 2) + let parent_frs = file_reference_to_frs(fn_attr.parent_directory); + let name_len = usize::from(fn_attr.file_name_length); + let ns = fn_off.saturating_add(size_of::()); + + // `name_len` is a u16 (≤ 65535); `*2` and `+ ns` cannot + // overflow usize on any supported target, but use the + // checked form so the parser is provably total, and let + // `data.get(..)` do the bounds check (None on a + // declared-length that overruns the record → skip name). + if let Some(nb) = name_len + .checked_mul(2) + .and_then(|byte_len| ns.checked_add(byte_len)) + .and_then(|name_end| data.get(ns..name_end)) { - // Skip DOS-only names (namespace 2) - let parent_frs = file_reference_to_frs(fn_attr.parent_directory); - let name_len = usize::from(fn_attr.file_name_length); - let ns = fn_off.saturating_add(size_of::()); - - // `name_len` is a u16 (≤ 65535); `*2` and `+ ns` cannot - // overflow usize on any supported target, but use the - // checked form so the parser is provably total, and let - // `data.get(..)` do the bounds check (None on a - // declared-length that overruns the record → skip name). - if let Some(nb) = name_len - .checked_mul(2) - .and_then(|byte_len| ns.checked_add(byte_len)) - .and_then(|name_end| data.get(ns..name_end)) - { - let lossy = decode_utf16le_into(nb, name_buf); - - // Push old first_name to chain - // Copy first_name before mutating (borrow checker) - let old_valid = index.records[base_ri].first_name.name.is_valid(); - let old_first = index.records[base_ri].first_name; // Copy - if old_valid { - let link_idx = len_to_u32(index.links.len()); - index.links.push(old_first); - index.records[base_ri].first_name.next_entry = link_idx; - } - - // Overwrite first_name with the new name. Store the - // name LOSSLESSLY (WI-4.4): a well-formed name's - // `String` bytes already equal its WTF-8, so the - // common path is unchanged; an ill-formed name - // (lossy > 0) is stored as byte-faithful WTF-8 of - // the raw UTF-16 so it stays findable. `is_ascii` / - // extension still derive from the lossy display - // `name_buf` (a U+FFFD name is not ASCII and has no - // meaningful extension). - let (name_off, stored_len) = - store_name_lossless(index, name_buf, nb, lossy); - let is_ascii = name_buf.is_ascii(); - let ext_id = index.intern_extension(name_buf); - let name_ref = IndexNameRef::new( - name_off, - len_to_u16(stored_len), - is_ascii, - ext_id, - ); - - index.records[base_ri].first_name.name = name_ref; - // Typed `ParentFrs` slot — lift parser-local raw `u64`. - index.records[base_ri].first_name.parent_frs = - crate::frs::ParentFrs::new(parent_frs); - - // $FILE_NAME's own namespace/timestamps (often - // differ from $STANDARD_INFORMATION — e.g. - // timestomping leaves STD_INFO altered but - // FILE_NAME original). `fn_attr` is already fully - // decoded above; these are free reads of already- - // resident memory. Push-to-front: whichever name - // is currently "first" also owns these fields. - let rec = &mut index.records[base_ri]; - rec.namespace = fn_attr.file_name_namespace; - rec.fn_created = fn_attr.creation_time; - rec.fn_modified = fn_attr.modification_time; - rec.fn_accessed = fn_attr.access_time; - rec.fn_mft_changed = fn_attr.mft_change_time; - - // Build parent-child relationship. - // name_index = name_count BEFORE increment - let name_index = index.records[base_ri].name_count; - - if parent_frs != frs_base && parent_frs != u64::from(NO_ENTRY) { - let parent_ri = u32_as_usize( - index.ensure_record(crate::frs::Frs::new(parent_frs)), - ); - let child_idx = len_to_u32(index.children.len()); - let old_fc = index.records[parent_ri].first_child; - index.records[parent_ri].first_child = child_idx; - - index.children.push(ChildInfo { - next_entry: old_fc, - _pad0: [0; 4], - // Typed `Frs` slot — reuse cached typed FRS. - child_frs: frs_base_typed, - name_index, - _pad1: [0; 6], - }); - } + let lossy = decode_utf16le_into(nb, name_buf); + + // Push old first_name to chain + // Copy first_name before mutating (borrow checker) + let old_valid = index.records[base_ri].first_name.name.is_valid(); + let old_first = index.records[base_ri].first_name; // Copy + if old_valid { + let link_idx = len_to_u32(index.links.len()); + index.links.push(old_first); + index.records[base_ri].first_name.next_entry = link_idx; + } - // Increment name_count (zero-based, always increment) - // (including the first name). - index.records[base_ri].name_count = - index.records[base_ri].name_count.saturating_add(1); + // Overwrite first_name with the new name. Store the + // name LOSSLESSLY (WI-4.4): a well-formed name's + // `String` bytes already equal its WTF-8, so the + // common path is unchanged; an ill-formed name + // (lossy > 0) is stored as byte-faithful WTF-8 of + // the raw UTF-16 so it stays findable. `is_ascii` / + // extension still derive from the lossy display + // `name_buf` (a U+FFFD name is not ASCII and has no + // meaningful extension). + let (name_off, stored_len) = + store_name_lossless(index, name_buf, nb, lossy); + let is_ascii = name_buf.is_ascii(); + let ext_id = index.intern_extension(name_buf); + let name_ref = + IndexNameRef::new(name_off, len_to_u16(stored_len), is_ascii, ext_id); + + index.records[base_ri].first_name.name = name_ref; + // Typed `ParentFrs` slot — lift parser-local raw `u64`. + index.records[base_ri].first_name.parent_frs = + crate::frs::ParentFrs::new(parent_frs); + + // $FILE_NAME's own namespace/timestamps (often + // differ from $STANDARD_INFORMATION — e.g. + // timestomping leaves STD_INFO altered but + // FILE_NAME original). `fn_attr` is already fully + // decoded above; these are free reads of already- + // resident memory. Push-to-front: whichever name + // is currently "first" also owns these fields. + let rec = &mut index.records[base_ri]; + rec.namespace = fn_attr.file_name_namespace; + rec.fn_created = fn_attr.creation_time; + rec.fn_modified = fn_attr.modification_time; + rec.fn_accessed = fn_attr.access_time; + rec.fn_mft_changed = fn_attr.mft_change_time; + + // Build parent-child relationship. + // name_index = name_count BEFORE increment + let name_index = index.records[base_ri].name_count; + + if parent_frs != frs_base && parent_frs != u64::from(NO_ENTRY) { + let parent_ri = + u32_as_usize(index.ensure_record(crate::frs::Frs::new(parent_frs))); + let child_idx = len_to_u32(index.children.len()); + let old_fc = index.records[parent_ri].first_child; + index.records[parent_ri].first_child = child_idx; + + index.children.push(ChildInfo { + next_entry: old_fc, + _pad0: [0; 4], + // Typed `Frs` slot — reuse cached typed FRS. + child_frs: frs_base_typed, + name_index, + _pad1: [0; 6], + }); } + + // Increment name_count (zero-based, always increment) + // (including the first name). + index.records[base_ri].name_count = + index.records[base_ri].name_count.saturating_add(1); } } } @@ -748,8 +492,9 @@ pub fn process_record(data: &[u8], frs: u64, index: &mut MftIndex, name_buf: &mu { // Fallible: a value offset that overruns the record // leaves the reparse tag unset rather than panicking. - let vo = usize::from(rd_u16(data, offset.saturating_add(20))); - if let Some(tag) = offset.checked_add(vo).map(|rp| rd_u32(data, rp)) { + if let Some(tag) = + resident_value_offset(data, offset).map(|rp| rd_u32(data, rp)) + { index.records[base_ri].reparse_tag = tag; } } @@ -803,5 +548,12 @@ fn rd_u64(buf: &[u8], off: usize) -> u64 { .map_or(0, u64::from_le_bytes) } +/// Absolute offset of a resident attribute's value, from its own +/// `value_offset` field (`attr_offset + 20`, a `u16`). +#[inline] +fn resident_value_offset(data: &[u8], attr_offset: usize) -> Option { + attr_offset.checked_add(usize::from(rd_u16(data, attr_offset.saturating_add(20)))) +} + #[cfg(test)] mod tests; diff --git a/crates/uffs-mft/src/io/parser/unified/name_codec.rs b/crates/uffs-mft/src/io/parser/unified/name_codec.rs new file mode 100644 index 000000000..c8cd230ab --- /dev/null +++ b/crates/uffs-mft/src/io/parser/unified/name_codec.rs @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! NTFS name decoding: UTF-16LE → `String` with counted, non-silent loss +//! (Category 4, WI-4.1), plus lossless WTF-8 storage for ill-formed names +//! (WI-4.4). Split out of `unified.rs` — this cluster has zero dependency +//! on `process_record`'s attribute-loop state, only on `MftIndex`. +//! +//! `arithmetic_side_effects` is enabled module-wide as a regression guard, +//! matching `unified.rs`'s own hardening posture: every offset here is +//! derived from attacker-controllable on-disk bytes. +#![warn(clippy::arithmetic_side_effects)] + +use crate::index::MftIndex; + +/// Decode a UTF-16LE byte slice into `out`, replacing unpaired surrogates +/// with U+FFFD. Returns the number of U+FFFD replacements emitted +/// (`0` = lossless). +/// +/// This avoids the per-call `SmallVec` + `String` allocation that +/// `String::from_utf16_lossy` requires, and — unlike `from_utf16_lossy` — +/// surfaces the substitution count so name loss at the NTFS boundary is +/// measured, not silent (Category 4, WI-4.1). +#[inline] +pub(super) fn decode_utf16le_into(bytes: &[u8], out: &mut String) -> u32 { + out.clear(); + let mut replacements: u32 = 0; + let mut i = 0_usize; + while let Some(pair) = i + .checked_add(2) + .and_then(|end| bytes.get(i..end)) + .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) + { + let code = u16::from_le_bytes(pair); + // `i` indexes a &[u8]; it cannot exceed `bytes.len()` (≤ isize::MAX), + // so `+= 2` cannot overflow usize. saturating_add keeps it total. + i = i.saturating_add(2); + match code { + // High surrogate + 0xD800..=0xDBFF => { + if let Some(low_pair) = i + .checked_add(2) + .and_then(|end| bytes.get(i..end)) + .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) + { + let low = u16::from_le_bytes(low_pair); + if (0xDC00..=0xDFFF).contains(&low) { + i = i.saturating_add(2); + // Bounds-proven: `code ∈ 0xD800..=0xDBFF` and + // `low ∈ 0xDC00..=0xDFFF`, so both subtractions are + // non-negative and the result is ≤ 0x10FFFF — no + // overflow/underflow is reachable. + let cp = 0x1_0000_u32 + .saturating_add((u32::from(code).saturating_sub(0xD800_u32)) << 10_u32) + .saturating_add(u32::from(low).saturating_sub(0xDC00_u32)); + if let Some(ch) = char::from_u32(cp) { + out.push(ch); + } else { + out.push(char::REPLACEMENT_CHARACTER); + replacements = replacements.saturating_add(1); + } + } else { + out.push(char::REPLACEMENT_CHARACTER); + replacements = replacements.saturating_add(1); + } + } else { + out.push(char::REPLACEMENT_CHARACTER); + replacements = replacements.saturating_add(1); + } + } + // Low surrogate without preceding high + 0xDC00..=0xDFFF => { + out.push(char::REPLACEMENT_CHARACTER); + replacements = replacements.saturating_add(1); + } + _ => { + // All non-surrogate u16 values are valid Unicode scalar values. + // `char::from_u32` is cheap for the common BMP case. + if let Some(ch) = char::from_u32(u32::from(code)) { + out.push(ch); + } + } + } + } + replacements +} + +/// Decode a `&[u16]` UTF-16 name into a fresh `String`, returning +/// `(String, replacement_count)`. Use this instead of +/// `String::from_utf16_lossy` at NTFS name boundaries so loss is counted, +/// not silent (Category 4, WI-4.1). +/// +/// Most NTFS-name call sites already hold a `Vec` / `SmallVec<[u16; N]>` +/// (the attribute decoder collects code units before stringifying), so this +/// `&[u16]` entry point avoids re-deriving a byte slice. There is exactly +/// ONE surrogate-handling implementation: this re-encodes to LE bytes and +/// routes through `decode_utf16le_into`. +#[inline] +pub(crate) fn decode_name_u16(units: &[u16]) -> (String, u32) { + let mut bytes = Vec::with_capacity(units.len().saturating_mul(2)); + for unit in units { + bytes.extend_from_slice(&unit.to_le_bytes()); + } + let mut out = String::new(); + let count = decode_utf16le_into(&bytes, &mut out); + if count > 0 { + LOSSY_NAME_COUNT.fetch_add(u64::from(count), core::sync::atomic::Ordering::Relaxed); + } + (out, count) +} + +/// Re-encode a UTF-16LE byte slice **losslessly** as WTF-8 into `out`. +/// +/// Unlike [`decode_utf16le_into`] (which replaces unpaired surrogates with +/// U+FFFD for a valid-UTF-8 `String`), this preserves *every* code unit — +/// well-formed text becomes ordinary UTF-8, and an **unpaired surrogate** +/// (`0xD800..=0xDFFF` with no valid pairing) is emitted as its 3-byte WTF-8 +/// encoding (`1110_xxxx 10xx_xxxx 10xx_xxxx` over the raw 16-bit value). The +/// result is therefore byte-faithful to the on-disk NTFS name and is what the +/// byte-native search/trigram path matches against, so a file with an +/// ill-formed name remains **findable by its true name** (WI-4.4). Surrogate +/// *pairs* are combined into their astral scalar (normal 4-byte UTF-8). +/// +/// Only called on the rare lossy path (when `decode_utf16le_into` reported a +/// replacement), so its modest cost never touches the well-formed hot path. +#[expect( + clippy::arithmetic_side_effects, + reason = "all arithmetic is on values masked to ≤ 0x10FFFF / 6-bit groups; \ + the WTF-8 byte composition cannot overflow u8/u32" +)] +pub(super) fn wtf8_from_utf16le(bytes: &[u8], out: &mut Vec) { + /// Low 6 bits of `x`, as a UTF-8 continuation byte (`10xx_xxxx`). + /// + /// `x & 0x3F` is in `0..=0x3F` and `0x80 | _` is in `0x80..=0xBF`, so the + /// `u8` cast is exact, never truncating. + #[expect( + clippy::cast_possible_truncation, + reason = "value is masked to 6 bits (≤ 0x3F) then OR'd with 0x80 → always ≤ 0xBF" + )] + const fn cont(x: u32) -> u8 { + (0x80_u32 | (x & 0x3F)) as u8 + } + + /// Leading byte: `prefix` OR the low `mask` bits of `cp_shifted`. + /// Callers pass a `mask` (5/4/3 bits) that bounds the residual to the + /// prefix's free bits, so the `u8` cast is exact. + #[expect( + clippy::cast_possible_truncation, + reason = "masked to ≤ 5 bits then OR'd with a fixed prefix → always ≤ 0xFF" + )] + const fn lead(prefix: u8, cp_shifted: u32, mask: u32) -> u8 { + prefix | (cp_shifted & mask) as u8 + } + + /// Push a single code point (or lone surrogate) as WTF-8 bytes. + fn push_wtf8(cp: u32, out: &mut Vec) { + match cp { + 0x0000..=0x007F => { + // ASCII: single byte, value ≤ 0x7F fits u8 exactly. + #[expect( + clippy::cast_possible_truncation, + reason = "cp ≤ 0x7F in this arm → exact u8" + )] + out.push(cp as u8); + } + // 2-byte: 110x_xxxx 10xx_xxxx (5 payload bits in the lead). + 0x0080..=0x07FF => { + out.push(lead(0xC0, cp >> 6, 0x1F)); + out.push(cont(cp)); + } + // 3-byte: BMP incl. lone surrogates 0xD800..=0xDFFF (4 lead bits). + 0x0800..=0xFFFF => { + out.push(lead(0xE0, cp >> 12, 0x0F)); + out.push(cont(cp >> 6)); + out.push(cont(cp)); + } + // 4-byte: astral from a valid surrogate pair (3 lead bits). + _ => { + out.push(lead(0xF0, cp >> 18, 0x07)); + out.push(cont(cp >> 12)); + out.push(cont(cp >> 6)); + out.push(cont(cp)); + } + } + } + + let mut i = 0_usize; + while let Some(pair) = i + .checked_add(2) + .and_then(|end| bytes.get(i..end)) + .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) + { + let code = u16::from_le_bytes(pair); + i = i.saturating_add(2); + if (0xD800..=0xDBFF).contains(&code) { + // High surrogate: combine with a following low surrogate if present. + if let Some(low) = i + .checked_add(2) + .and_then(|end| bytes.get(i..end)) + .and_then(|sl| <[u8; 2]>::try_from(sl).ok()) + .map(u16::from_le_bytes) + .filter(|low| (0xDC00..=0xDFFF).contains(low)) + { + i = i.saturating_add(2); + let cp = 0x1_0000_u32 + + ((u32::from(code) - 0xD800_u32) << 10_u32) + + (u32::from(low) - 0xDC00_u32); + push_wtf8(cp, out); + } else { + // Unpaired high surrogate — preserve verbatim as WTF-8. + push_wtf8(u32::from(code), out); + } + } else { + // BMP scalar or unpaired low surrogate — both preserved verbatim. + push_wtf8(u32::from(code), out); + } + } +} + +/// Store a just-decoded name into the index's name buffer **losslessly**, +/// returning `(byte_offset, stored_byte_len)`. +/// +/// - `display` is the lossy `String` produced by [`decode_utf16le_into`] +/// (U+FFFD for ill-formed parts) — used as-is for the common well-formed +/// case, where its bytes are identical to the name's WTF-8. +/// - `raw_utf16le` is the original on-disk UTF-16LE byte slice for the name. +/// - `lossy` is the replacement count `decode_utf16le_into` reported. +/// +/// When `lossy == 0` (the overwhelming common case) the `display` bytes are +/// stored directly — zero extra work on the hot path. When `lossy > 0`, the +/// raw UTF-16 is re-encoded to byte-faithful WTF-8 and *those* bytes are +/// stored, so the file is findable by its true name (WI-4.4). The returned +/// length is the **stored** byte length (WTF-8 length on the lossy path), +/// which the caller records in the `IndexNameRef` so `get_name_bytes` slices +/// exactly the stored name. +pub(super) fn store_name_lossless( + index: &mut MftIndex, + display: &str, + raw_utf16le: &[u8], + lossy: u32, +) -> (u32, usize) { + if lossy == 0 { + let bytes = display.as_bytes(); + (index.add_name_bytes(bytes), bytes.len()) + } else { + let mut wtf8 = Vec::with_capacity(raw_utf16le.len()); + wtf8_from_utf16le(raw_utf16le, &mut wtf8); + (index.add_name_bytes(&wtf8), wtf8.len()) + } +} + +/// Process-global tally of U+FFFD substitutions emitted by +/// [`decode_name_u16`] across all NTFS-name decodes (Category 4, WI-4.1). +/// +/// The parser call sites are spread across nine modules and do not thread a +/// stats accumulator through their (hot-path) signatures, so the count is +/// gathered here with a single relaxed atomic — cheap, lock-free, and read +/// at index-build time into the `lossy_name_count` field of +/// [`crate::index::MftStats`] for the "N filenames were stored with +/// U+FFFD" warning. `Relaxed` is +/// sufficient: it is a monotonic diagnostic counter, not a synchronisation +/// point. +pub(crate) static LOSSY_NAME_COUNT: core::sync::atomic::AtomicU64 = + core::sync::atomic::AtomicU64::new(0); + +/// Snapshot the current global lossy-name tally. +#[inline] +pub(crate) fn lossy_name_count() -> u64 { + LOSSY_NAME_COUNT.load(core::sync::atomic::Ordering::Relaxed) +} diff --git a/crates/uffs-mft/src/parse/direct_index.rs b/crates/uffs-mft/src/parse/direct_index.rs index dd0e7a8e7..862451a6c 100644 --- a/crates/uffs-mft/src/parse/direct_index.rs +++ b/crates/uffs-mft/src/parse/direct_index.rs @@ -3,10 +3,6 @@ //! Single-pass direct-to-index parser. //! -//! Exception: Performance-critical single-pass MFT record parser; monolithic -//! attribute-dispatch loop kept together for cache locality and to mirror the -//! NTFS on-disk attribute layout one arm at a time. -//! //! This module implements the high-performance single-pass parser that builds //! an `MftIndex` directly from raw MFT records without creating intermediate //! `ParsedRecord` allocations. @@ -75,6 +71,87 @@ fn rd_u32(buf: &[u8], off: usize) -> u32 { .map_or(0, u32::from_le_bytes) } +/// Whether an attribute is the "primary" copy for stream-counting purposes. +/// Resident attributes are always primary; a non-resident attribute is +/// primary only when its LowestVCN is 0 — continuation extents of a larger +/// non-resident attribute must not be double-counted as new streams. +#[inline] +fn is_primary_attribute( + data: &[u8], + offset: usize, + attr_header: &crate::ntfs::AttributeRecordHeader, +) -> bool { + if attr_header.is_non_resident == 0 { + return true; + } + let nr_offset = offset + 16; + data.get(nr_offset..nr_offset + 8) + .and_then(|sl| <[u8; 8]>::try_from(sl).ok()) + .is_some_and(|bytes| i64::from_le_bytes(bytes) == 0) +} + +/// Extracts an attribute's own name (the NTFS "attribute name", e.g. `$I30` +/// on `$INDEX_ROOT` or an ADS name on `$DATA` — not a `$FILE_NAME`). Empty +/// string if unnamed or the declared length overruns the record. +#[inline] +fn extract_attr_name( + data: &[u8], + offset: usize, + attr_header: &crate::ntfs::AttributeRecordHeader, +) -> String { + if attr_header.name_length == 0 { + return String::new(); + } + let name_offset = offset + usize::from(attr_header.name_offset); + let name_len = usize::from(attr_header.name_length); + if name_offset + name_len * 2 > data.len() { + return String::new(); + } + let name_bytes = &data[name_offset..name_offset + name_len * 2]; + let name_u16: SmallVec<[u16; 64]> = name_bytes + .as_chunks::<2>() + .0 + .iter() + .map(|c| u16::from_le_bytes(*c)) + .collect(); + crate::io::parser::unified::decode_name_u16(&name_u16).0 +} + +/// Resident/non-resident `(size, allocated)` for the generic "count as a +/// stream" attribute types (`$OBJECT_ID`, `$EA`, non-`$I30` index +/// attributes, the unknown-type catch-all): resident → `(value_length, 0)`; +/// non-resident → `(DataSize, AllocatedSize)` from the +/// `NonResidentAttributeData` block at `offset + 16`. +#[inline] +fn read_size_allocated( + data: &[u8], + offset: usize, + attr_header: &crate::ntfs::AttributeRecordHeader, +) -> (u64, u64) { + if attr_header.is_non_resident == 0 { + return (u64::from(rd_u32(data, offset + 16)), 0_u64); + } + let nr_offset = offset + 16; + if nr_offset + 48 > data.len() { + return (0_u64, 0_u64); + } + let alloc_bytes = &data[nr_offset + 24..nr_offset + 32]; + let allocated = i64::from_le_bytes(alloc_bytes.try_into().unwrap_or([0; 8])); + let size_bytes = &data[nr_offset + 32..nr_offset + 40]; + let data_size = i64::from_le_bytes(size_bytes.try_into().unwrap_or([0; 8])); + (nonneg_to_u64(data_size), nonneg_to_u64(allocated)) +} + +/// `(is_resident, is_sparse)` for an attribute, from already-parsed header +/// fields: `is_resident` from `is_non_resident`, `is_sparse` from the +/// `ATTRIBUTE_FLAG_SPARSE` (`0x8000`) header flag bit — free, no new I/O. +#[inline] +const fn resident_and_sparse(attr_header: &crate::ntfs::AttributeRecordHeader) -> (bool, bool) { + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + (is_resident, is_sparse) +} + /// Parses a record directly into `MftIndex` (single-pass inline parsing). /// /// This function parses the record and adds it directly to the index, @@ -261,21 +338,7 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf // legacy-output parity: Only primary attributes (LowestVCN == 0) count as // streams. Continuation extents (LowestVCN > 0) are skipped. // See ntfs_index_load.hpp:358 - let is_primary = if attr_header.is_non_resident == 0 { - true // Resident attributes are always primary - } else { - let nr_offset = offset + 16; - if nr_offset + 8 <= data.len() { - let lowest_vcn = i64::from_le_bytes( - data[nr_offset..nr_offset + 8].try_into().unwrap_or([0; 8]), - ); - lowest_vcn == 0 - } else { - false // Can't verify, skip to be safe - } - }; - - if !is_primary { + if !is_primary_attribute(data, offset, &attr_header) { // Skip continuation extents - they don't count as new streams offset += u32_as_usize(attr_header.length); continue; @@ -283,49 +346,8 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf // Parse $DATA - track both default stream and ADS let name_len = usize::from(attr_header.name_length); - let (size, allocated) = if attr_header.is_non_resident != 0 { - // Non-resident: size at offset 48, allocated at offset 40 - let alloc_offset = offset + 40; - let size_offset = offset + 48; - if size_offset + 8 <= data.len() { - let allocated = u64::from_le_bytes( - data[alloc_offset..alloc_offset + 8] - .try_into() - .unwrap_or([0; 8]), - ); - let size = u64::from_le_bytes( - data[size_offset..size_offset + 8] - .try_into() - .unwrap_or([0; 8]), - ); - (size, allocated) - } else { - (0, 0) - } - } else { - // Resident: value_length at offset 16 - // Resident files have no clusters allocated — data is stored in the MFT record. - // allocated_size=0 for resident files. - let len_offset = offset + 16; - if len_offset + 4 <= data.len() { - let len = u64::from(u32::from_le_bytes( - data[len_offset..len_offset + 4] - .try_into() - .unwrap_or([0; 4]), - )); - (len, 0) // allocated_size = 0 for resident files - } else { - (0, 0) - } - }; - - // WI-5.2-adjacent correctness fix: `is_resident` was already - // computed above (`attr_header.is_non_resident == 0`) but - // never carried into the index; `is_sparse` lives in the - // attribute header's own `flags` (ATTR_IS_SPARSE = 0x8000), - // already-parsed data — both are free to read, no new I/O. - let is_resident = attr_header.is_non_resident == 0; - let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + let (size, allocated) = read_size_allocated(data, offset, &attr_header); + let (is_resident, is_sparse) = resident_and_sparse(&attr_header); if name_len == 0 { // Default stream @@ -390,8 +412,7 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf }; // Add $REPARSE_POINT as a stream (contributes to stream counting) - let is_resident = attr_header.is_non_resident == 0; - let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + let (is_resident, is_sparse) = resident_and_sparse(&attr_header); additional_streams.push(( String::from("$REPARSE"), rp_size, @@ -437,78 +458,26 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf if is_i30 { // Accumulate $I30 sizes for directories - if attr_header.is_non_resident == 0 { - dir_index_size += u64::from(rd_u32(data, offset + 16)); - } else { - let nr_offset = offset + 16; - if nr_offset + 48 <= data.len() { - let alloc_bytes = &data[nr_offset + 24..nr_offset + 32]; - let allocated = - i64::from_le_bytes(alloc_bytes.try_into().unwrap_or([0; 8])); - let size_bytes = &data[nr_offset + 32..nr_offset + 40]; - let data_size = - i64::from_le_bytes(size_bytes.try_into().unwrap_or([0; 8])); - dir_index_size += nonneg_to_u64(data_size); - dir_index_allocated += nonneg_to_u64(allocated); - } - } - } else { + let (size, allocated) = read_size_allocated(data, offset, &attr_header); + dir_index_size += size; + dir_index_allocated += allocated; + } else if is_primary_attribute(data, offset, &attr_header) { // Non-$I30 index - count as stream - // Check if primary attribute (LowestVCN == 0) - let is_primary = if attr_header.is_non_resident == 0 { - true - } else { - let nr_offset = offset + 16; - if nr_offset + 8 <= data.len() { - let lowest_vcn = i64::from_le_bytes( - data[nr_offset..nr_offset + 8].try_into().unwrap_or([0; 8]), - ); - lowest_vcn == 0 - } else { - false + let (size, allocated) = read_size_allocated(data, offset, &attr_header); + let stream_name = if attr_name.is_empty() { + match attr_type { + Some(AttributeType::Bitmap) => String::from("$BITMAP"), + Some(AttributeType::IndexRoot) => String::from("$INDEX_ROOT"), + Some(AttributeType::IndexAllocation) => { + String::from("$INDEX_ALLOCATION") + } + _ => String::new(), } + } else { + attr_name }; - - if is_primary { - let (size, allocated) = if attr_header.is_non_resident == 0 { - (u64::from(rd_u32(data, offset + 16)), 0_u64) - } else { - let nr_offset = offset + 16; - if nr_offset + 48 <= data.len() { - let alloc_bytes = &data[nr_offset + 24..nr_offset + 32]; - let allocated = - i64::from_le_bytes(alloc_bytes.try_into().unwrap_or([0; 8])); - let size_bytes = &data[nr_offset + 32..nr_offset + 40]; - let data_size = - i64::from_le_bytes(size_bytes.try_into().unwrap_or([0; 8])); - (nonneg_to_u64(data_size), nonneg_to_u64(allocated)) - } else { - (0_u64, 0_u64) - } - }; - - let stream_name = if attr_name.is_empty() { - match attr_type { - Some(AttributeType::Bitmap) => String::from("$BITMAP"), - Some(AttributeType::IndexRoot) => String::from("$INDEX_ROOT"), - Some(AttributeType::IndexAllocation) => { - String::from("$INDEX_ALLOCATION") - } - _ => String::new(), - } - } else { - attr_name - }; - let is_resident = attr_header.is_non_resident == 0; - let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; - additional_streams.push(( - stream_name, - size, - allocated, - is_sparse, - is_resident, - )); - } + let (is_resident, is_sparse) = resident_and_sparse(&attr_header); + additional_streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } Some( @@ -523,59 +492,9 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf | AttributeType::AttributeList, ) => { // All these attribute types are counted as individual streams. - // Check if primary attribute (LowestVCN == 0) - let is_primary = if attr_header.is_non_resident == 0 { - true - } else { - let nr_offset = offset + 16; - if nr_offset + 8 <= data.len() { - let lowest_vcn = i64::from_le_bytes( - data[nr_offset..nr_offset + 8].try_into().unwrap_or([0; 8]), - ); - lowest_vcn == 0 - } else { - false - } - }; - - if is_primary { - // Extract attribute name (if any) - let attr_name = if attr_header.name_length > 0 { - let name_offset = offset + usize::from(attr_header.name_offset); - let name_len = usize::from(attr_header.name_length); - if name_offset + name_len * 2 <= data.len() { - let name_bytes = &data[name_offset..name_offset + name_len * 2]; - let name_u16: SmallVec<[u16; 64]> = name_bytes - .as_chunks::<2>() - .0 - .iter() - .map(|c| u16::from_le_bytes(*c)) - .collect(); - crate::io::parser::unified::decode_name_u16(&name_u16).0 - } else { - String::new() - } - } else { - String::new() - }; - - let (size, allocated) = if attr_header.is_non_resident == 0 { - (u64::from(rd_u32(data, offset + 16)), 0_u64) - } else { - let nr_offset = offset + 16; - if nr_offset + 48 <= data.len() { - let alloc_bytes = &data[nr_offset + 24..nr_offset + 32]; - let allocated = - i64::from_le_bytes(alloc_bytes.try_into().unwrap_or([0; 8])); - let size_bytes = &data[nr_offset + 32..nr_offset + 40]; - let data_size = - i64::from_le_bytes(size_bytes.try_into().unwrap_or([0; 8])); - (nonneg_to_u64(data_size), nonneg_to_u64(allocated)) - } else { - (0_u64, 0_u64) - } - }; - + if is_primary_attribute(data, offset, &attr_header) { + let attr_name = extract_attr_name(data, offset, &attr_header); + let (size, allocated) = read_size_allocated(data, offset, &attr_header); let stream_name = if attr_name.is_empty() { match attr_type { Some(AttributeType::ObjectId) => String::from("$OBJECT_ID"), @@ -598,8 +517,7 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf } else { attr_name }; - let is_resident = attr_header.is_non_resident == 0; - let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + let (is_resident, is_sparse) = resident_and_sparse(&attr_header); additional_streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } @@ -608,66 +526,16 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf // This includes truly unknown types let type_code = attr_header.type_code; - // Check if primary attribute (LowestVCN == 0) - let is_primary = if attr_header.is_non_resident == 0 { - true - } else { - let nr_offset = offset + 16; - if nr_offset + 8 <= data.len() { - let lowest_vcn = i64::from_le_bytes( - data[nr_offset..nr_offset + 8].try_into().unwrap_or([0; 8]), - ); - lowest_vcn == 0 - } else { - false - } - }; - - if is_primary { - // Extract attribute name (if any) - let attr_name = if attr_header.name_length > 0 { - let name_offset = offset + usize::from(attr_header.name_offset); - let name_len = usize::from(attr_header.name_length); - if name_offset + name_len * 2 <= data.len() { - let name_bytes = &data[name_offset..name_offset + name_len * 2]; - let name_u16: SmallVec<[u16; 64]> = name_bytes - .as_chunks::<2>() - .0 - .iter() - .map(|c| u16::from_le_bytes(*c)) - .collect(); - crate::io::parser::unified::decode_name_u16(&name_u16).0 - } else { - String::new() - } - } else { - String::new() - }; - - let (size, allocated) = if attr_header.is_non_resident == 0 { - (u64::from(rd_u32(data, offset + 16)), 0_u64) - } else { - let nr_offset = offset + 16; - if nr_offset + 48 <= data.len() { - let alloc_bytes = &data[nr_offset + 24..nr_offset + 32]; - let allocated = - i64::from_le_bytes(alloc_bytes.try_into().unwrap_or([0; 8])); - let size_bytes = &data[nr_offset + 32..nr_offset + 40]; - let data_size = - i64::from_le_bytes(size_bytes.try_into().unwrap_or([0; 8])); - (nonneg_to_u64(data_size), nonneg_to_u64(allocated)) - } else { - (0_u64, 0_u64) - } - }; + if is_primary_attribute(data, offset, &attr_header) { + let attr_name = extract_attr_name(data, offset, &attr_header); + let (size, allocated) = read_size_allocated(data, offset, &attr_header); let stream_name = if attr_name.is_empty() { format!("$UNKNOWN_0x{type_code:X}") } else { attr_name }; - let is_resident = attr_header.is_non_resident == 0; - let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + let (is_resident, is_sparse) = resident_and_sparse(&attr_header); additional_streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } diff --git a/scripts/ci/file_size_exceptions.txt b/scripts/ci/file_size_exceptions.txt index c30174703..7c0b95561 100644 --- a/scripts/ci/file_size_exceptions.txt +++ b/scripts/ci/file_size_exceptions.txt @@ -7,8 +7,6 @@ crates/uffs-core/src/search/field/field_metadata.rs|PERMANENT: Single const fn m crates/uffs-core/src/search/filters/tests.rs|PERMANENT: Integration test suite for filter pipeline; splitting further would scatter related test fixtures crates/uffs-core/src/search/filters/mod.rs|PERMANENT: Cohesive SearchFilters/SearchFilterParams definitions + from_params construction; kept together so the full per-field filter contract is auditable in one place crates/uffs-client/src/schema/field_metadata.rs|PERMANENT: Single const fn match table — one FieldMeta per FieldId variant; mirrors uffs-core version -crates/uffs-mft/src/parse/direct_index.rs|PERMANENT: Performance-critical single-pass MFT record parser; monolithic loop for cache locality -crates/uffs-mft/src/io/parser/unified.rs|PERMANENT: Single-pass unified MFT record processor; monolithic attribute loop for cache locality, same reasoning as direct_index.rs crates/uffs-mft/src/parse/direct_index_extension.rs|PERMANENT: Extension record parser for the direct-to-index pipeline; same structural reasoning as its base-record counterpart crates/uffs-mft/src/reader/index_read.rs|PERMANENT: Single impl MftReader block with tightly coupled cfg-gated pipeline stages crates/uffs-diag/src/bin/compare_scan_parity.rs|PERMANENT: Standalone diagnostic binary; single-file readability outweighs LOC policy for tooling From b8f3d17569fe713754dfa95fa2a496de861a7d72 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:10:11 -0700 Subject: [PATCH 6/7] fix(mft): populate namespace/fn_* timestamps when a name arrives only via an extension record FileRecord.base_frs audit: confirmed correct as-is. Traced parse/merger.rs (the legacy pipeline) directly -- extension ParsedRecords are fully merged into their base record and never survive into the final Vec, so base_frs is genuinely Frs::ZERO for every record that reaches MftIndex, in both the legacy and modern pipelines. Not a bug; no change needed. Continuing the same audit surfaced a real one: direct_index_extension.rs's "base record has no $FILE_NAME, promote the first extension name to primary" merge path (the case where a file has enough attributes to overflow its base MFT record and $FILE_NAME lands in an extension record) copied only the name text and parent FRS into first_name, silently dropping namespace and all four $FILE_NAME timestamps -- even though `$FILE_NAME`'s own attribute bytes were already fully decoded and even already read once by the same function. Every such record read back namespace=0 and fn_created/fn_modified/fn_accessed/fn_mft_changed=0 forever, with nothing else in the pipeline able to fix it later. Root cause: the `names` collection only ever carried (name, parent_frs) -- namespace/timestamps were decoded from `fn_attr` and then discarded before even reaching the promotion site. Extended it to a 7-field ExtNameEntry tuple (name, parent_frs, namespace, fn_created, fn_modified, fn_accessed, fn_mft_changed) and copy those fields onto the record alongside the name whenever it's promoted to primary. `LinkInfo` (storage for non-primary hard links) still has no room for these fields, so this only applies to whichever name ends up primary -- the same structural constraint as FileRecord's own fn_* fields. While touching this file, applied the same helper-extraction refactor already done for direct_index.rs (is_primary_attribute/extract_attr_name/ read_size_allocated/resident_and_sparse) -- same duplicated logic, same fix. 821 -> 762 lines, genuinely under the file-size policy threshold now instead of relying on its pre-existing PERMANENT exception, which is removed along with the stale "same reasoning as direct_index.rs" claim (direct_index.rs no longer needs one either). Added a regression test: base record with zero $FILE_NAME attributes, followed by an extension record carrying the file's only name, asserting namespace/fn_* land on the merged record. Verified red before the fix, green after. --- crates/uffs-mft/src/io/parser/mod.rs | 92 +++++ .../src/parse/direct_index_extension.rs | 364 +++++++----------- scripts/ci/file_size_exceptions.txt | 1 - 3 files changed, 230 insertions(+), 227 deletions(-) diff --git a/crates/uffs-mft/src/io/parser/mod.rs b/crates/uffs-mft/src/io/parser/mod.rs index acec683d1..26835751b 100644 --- a/crates/uffs-mft/src/io/parser/mod.rs +++ b/crates/uffs-mft/src/io/parser/mod.rs @@ -447,6 +447,98 @@ mod tests { assert_eq!(direct_rec.lsn, lsn); } + /// Regression pin: when a file's *only* `$FILE_NAME` lives in an + /// extension record (the base MFT record segment has none at all -- + /// the normal case once a record has enough attributes to overflow the + /// base segment), `direct_index_extension.rs`'s "promote to primary + /// name" merge path used to copy only the name text and parent FRS, + /// silently dropping namespace and all four `$FILE_NAME` timestamps + /// even though they were already fully decoded from the extension + /// record's own bytes. + #[test] + fn extension_only_file_name_sets_namespace_and_fn_timestamps_on_base_record() { + let namespace = 1_u8; // Win32 + let fn_created = 111_i64; + let fn_modified = 222_i64; + let fn_accessed = 333_i64; + let fn_mft_changed = 444_i64; + + // Base record: no $FILE_NAME attribute at all. + let mut base_record = RecordBuilder::new(56).build(); + let base_len = u32::try_from(base_record.len()).expect("fits in u32"); + base_record + .get_mut(24..28) + .expect("record well over 28 bytes") + .copy_from_slice(&base_len.to_le_bytes()); + + let mut index = MftIndex::new(crate::platform::DriveLetter::C); + assert!(!crate::parse::parse_record_to_index( + &base_record, + 42, + &mut index + )); + let base_rec = index + .find(crate::frs::Frs::new(42)) + .expect("the no-name early-return path must still create the record"); + assert_eq!(base_rec.namespace, 0); + assert_eq!(base_rec.fn_created, 0); + + // Extension record: carries the file's only $FILE_NAME. + let mut fn_payload = Vec::new(); + fn_payload.extend_from_slice(&0_u64.to_le_bytes()); // parent_directory + fn_payload.extend_from_slice(&fn_created.to_le_bytes()); + fn_payload.extend_from_slice(&fn_modified.to_le_bytes()); + fn_payload.extend_from_slice(&fn_mft_changed.to_le_bytes()); + fn_payload.extend_from_slice(&fn_accessed.to_le_bytes()); + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // allocated_size + fn_payload.extend_from_slice(&0_i64.to_le_bytes()); // data_size + fn_payload.extend_from_slice(&0_u32.to_le_bytes()); // file_attributes + fn_payload.extend_from_slice(&0_u16.to_le_bytes()); // packed_ea_size + fn_payload.extend_from_slice(&0_u16.to_le_bytes()); // reserved + fn_payload.push(1); // file_name_length = 1 char + fn_payload.push(namespace); + fn_payload.extend_from_slice(&0x0062_u16.to_le_bytes()); // "b" + let file_name_total_len = u32::try_from(24 + fn_payload.len()).expect("fits in u32"); + + let mut ext_record = RecordBuilder::new(56) + .attr(0x30, file_name_total_len, 0, 0, 0) + .raw( + &u32::try_from(fn_payload.len()) + .expect("fits in u32") + .to_le_bytes(), + ) + .raw(&24_u16.to_le_bytes()) + .raw(&[0_u8; 2]) + .raw(&fn_payload) + .build(); + let ext_len = u32::try_from(ext_record.len()).expect("fits in u32"); + ext_record + .get_mut(24..28) + .expect("record well over 28 bytes") + .copy_from_slice(&ext_len.to_le_bytes()); + // base_file_record_segment @ header offset 32 (u64): nonzero makes + // `is_base_record()` false and routes this to the extension-merge + // path, targeting FRS 42 (the base record created above). + ext_record + .get_mut(32..40) + .expect("record well over 40 bytes") + .copy_from_slice(&42_u64.to_le_bytes()); + + assert!(crate::parse::parse_record_to_index( + &ext_record, + 99, + &mut index + )); + let merged_rec = index + .find(crate::frs::Frs::new(42)) + .expect("extension merge must keep the base record"); + assert_eq!(merged_rec.namespace, namespace); + assert_eq!(merged_rec.fn_created, fn_created); + assert_eq!(merged_rec.fn_modified, fn_modified); + assert_eq!(merged_rec.fn_accessed, fn_accessed); + assert_eq!(merged_rec.fn_mft_changed, fn_mft_changed); + } + // ── WI-5.2 panic-resistance corpus ────────────────────────────── // // The daemon builds with `panic = "abort"`: a single parser panic on a diff --git a/crates/uffs-mft/src/parse/direct_index_extension.rs b/crates/uffs-mft/src/parse/direct_index_extension.rs index 118e9b4a0..a6b6d67cb 100644 --- a/crates/uffs-mft/src/parse/direct_index_extension.rs +++ b/crates/uffs-mft/src/parse/direct_index_extension.rs @@ -3,8 +3,6 @@ //! Extension record parser for direct-to-index path. //! -//! Exception: Direct index extension record parser for offline mode. -//! //! This module handles extension records for the single-pass parser, extracting //! names, streams, and all attribute types from extension records and merging //! them into base records in the index. @@ -66,6 +64,90 @@ fn rd_u32(buf: &[u8], off: usize) -> u32 { .map_or(0, u32::from_le_bytes) } +/// Whether an attribute is the "primary" copy for stream-counting purposes. +/// Mirrors `direct_index.rs`'s helper of the same name. +#[inline] +fn is_primary_attribute( + data: &[u8], + offset: usize, + attr_header: &crate::ntfs::AttributeRecordHeader, +) -> bool { + if attr_header.is_non_resident == 0 { + return true; + } + let nr_offset = offset + 16; + data.get(nr_offset..nr_offset + 8) + .and_then(|sl| <[u8; 8]>::try_from(sl).ok()) + .is_some_and(|bytes| i64::from_le_bytes(bytes) == 0) +} + +/// Extracts an attribute's own name (e.g. `$I30` on `$INDEX_ROOT`, an ADS +/// name on `$DATA` — not a `$FILE_NAME`). Mirrors `direct_index.rs`'s +/// helper of the same name. +#[inline] +fn extract_attr_name( + data: &[u8], + offset: usize, + attr_header: &crate::ntfs::AttributeRecordHeader, +) -> String { + if attr_header.name_length == 0 { + return String::new(); + } + let name_offset = offset + usize::from(attr_header.name_offset); + let name_len = usize::from(attr_header.name_length); + if name_offset + name_len * 2 > data.len() { + return String::new(); + } + let name_bytes = &data[name_offset..name_offset + name_len * 2]; + let name_u16: SmallVec<[u16; 64]> = name_bytes + .as_chunks::<2>() + .0 + .iter() + .map(|c| u16::from_le_bytes(*c)) + .collect(); + crate::io::parser::unified::decode_name_u16(&name_u16).0 +} + +/// Resident/non-resident `(size, allocated)` for the generic "count as a +/// stream" attribute types. Mirrors `direct_index.rs`'s helper of the same +/// name. +#[inline] +fn read_size_allocated( + data: &[u8], + offset: usize, + attr_header: &crate::ntfs::AttributeRecordHeader, +) -> (u64, u64) { + if attr_header.is_non_resident == 0 { + return (u64::from(rd_u32(data, offset + 16)), 0_u64); + } + let nr_offset = offset + 16; + if nr_offset + 48 > data.len() { + return (0_u64, 0_u64); + } + let alloc_bytes = &data[nr_offset + 24..nr_offset + 32]; + let allocated = i64::from_le_bytes(alloc_bytes.try_into().unwrap_or([0; 8])); + let size_bytes = &data[nr_offset + 32..nr_offset + 40]; + let data_size = i64::from_le_bytes(size_bytes.try_into().unwrap_or([0; 8])); + (nonneg_to_u64(data_size), nonneg_to_u64(allocated)) +} + +/// `(is_resident, is_sparse)` for an attribute. Mirrors `direct_index.rs`'s +/// helper of the same name. +#[inline] +const fn resident_and_sparse(attr_header: &crate::ntfs::AttributeRecordHeader) -> (bool, bool) { + let is_resident = attr_header.is_non_resident == 0; + let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + (is_resident, is_sparse) +} + +/// A pending extension-record `$FILE_NAME`: +/// `(name, parent_frs, namespace, fn_created, fn_modified, fn_accessed, +/// fn_mft_changed)`. `LinkInfo` (the storage for non-primary hard links) has +/// no room for namespace/timestamps, so those only ever reach the index if +/// this name gets promoted to the base record's primary name — see the +/// "base record has no name" merge path below. +type ExtNameEntry = (String, u64, u8, i64, i64, i64, i64); + /// Parses an extension record and adds its names/streams to the base record. /// /// Extension records contain additional `$FILE_NAME` attributes (hard links) @@ -121,7 +203,7 @@ pub(super) fn parse_extension_to_index( let max_offset = core::cmp::min(u32_as_usize(header.bytes_in_use), data.len()); // Collect names and streams from extension record - let mut names: SmallVec<[(String, u64); 4]> = SmallVec::new(); + let mut names: SmallVec<[ExtNameEntry; 4]> = SmallVec::new(); let mut streams: SmallVec<[StreamEntry; 4]> = SmallVec::new(); let ext_internal_streams: SmallVec<[(u64, u64); 4]> = SmallVec::new(); let mut dir_index_size: u64 = 0; @@ -175,7 +257,15 @@ pub(super) fn parse_extension_to_index( .collect(); let name = crate::io::parser::unified::decode_name_u16(&name_u16).0; let parent_frs = fn_attr.parent_directory & 0x0000_FFFF_FFFF_FFFF; - names.push((name, parent_frs)); + names.push(( + name, + parent_frs, + fn_attr.file_name_namespace, + fn_attr.creation_time, + fn_attr.modification_time, + fn_attr.access_time, + fn_attr.mft_change_time, + )); } } } @@ -185,21 +275,7 @@ pub(super) fn parse_extension_to_index( // legacy-output parity: Only primary attributes (LowestVCN == 0) count as // streams. Continuation extents (LowestVCN > 0) are skipped. // See ntfs_index_load.hpp:358 - let is_primary = if attr_header.is_non_resident == 0 { - true // Resident attributes are always primary - } else { - let nr_offset = offset + 16; - if nr_offset + 8 <= data.len() { - let lowest_vcn = i64::from_le_bytes( - data[nr_offset..nr_offset + 8].try_into().unwrap_or([0; 8]), - ); - lowest_vcn == 0 - } else { - false // Can't verify, skip to be safe - } - }; - - if !is_primary { + if !is_primary_attribute(data, offset, &attr_header) { // Skip continuation extents - they don't count as new streams offset += u32_as_usize(attr_header.length); continue; @@ -207,41 +283,8 @@ pub(super) fn parse_extension_to_index( // Parse $DATA attribute — default stream (unnamed) or ADS (named) let name_len = usize::from(attr_header.name_length); - let (size, allocated) = if attr_header.is_non_resident != 0 { - let nr_offset = offset + 16; - if nr_offset + 48 <= data.len() { - let allocated = i64::from_le_bytes( - data[nr_offset + 24..nr_offset + 32] - .try_into() - .unwrap_or([0; 8]), - ); - let size = i64::from_le_bytes( - data[nr_offset + 32..nr_offset + 40] - .try_into() - .unwrap_or([0; 8]), - ); - (nonneg_to_u64(size), nonneg_to_u64(allocated)) - } else { - (0, 0) - } - } else { - let len_offset = offset + 16; - if len_offset + 4 <= data.len() { - let len = u64::from(u32::from_le_bytes( - data[len_offset..len_offset + 4] - .try_into() - .unwrap_or([0; 4]), - )); - (len, 0) - } else { - (0, 0) - } - }; - - // Already-parsed attribute-header data, free to read: see - // `direct_index.rs`'s identical fix for is_sparse/is_resident. - let is_resident = attr_header.is_non_resident == 0; - let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + let (size, allocated) = read_size_allocated(data, offset, &attr_header); + let (is_resident, is_sparse) = resident_and_sparse(&attr_header); if name_len == 0 { // Default $DATA stream — update base record size @@ -286,8 +329,7 @@ pub(super) fn parse_extension_to_index( (0_u64, 0_u64) } }; - let is_resident = attr_header.is_non_resident == 0; - let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + let (is_resident, is_sparse) = resident_and_sparse(&attr_header); streams.push(( String::from("$REPARSE"), rp_size, @@ -328,71 +370,26 @@ pub(super) fn parse_extension_to_index( if is_i30 { // Accumulate $I30 sizes - if attr_header.is_non_resident == 0 { - dir_index_size += u64::from(rd_u32(data, offset + 16)); - } else { - let nr_offset = offset + 16; - if nr_offset + 48 <= data.len() { - let alloc_bytes = &data[nr_offset + 24..nr_offset + 32]; - let allocated = - i64::from_le_bytes(alloc_bytes.try_into().unwrap_or([0; 8])); - let size_bytes = &data[nr_offset + 32..nr_offset + 40]; - let data_size = - i64::from_le_bytes(size_bytes.try_into().unwrap_or([0; 8])); - dir_index_size += nonneg_to_u64(data_size); - dir_index_allocated += nonneg_to_u64(allocated); - } - } - } else { + let (size, allocated) = read_size_allocated(data, offset, &attr_header); + dir_index_size += size; + dir_index_allocated += allocated; + } else if is_primary_attribute(data, offset, &attr_header) { // Non-$I30 index - count as stream - let is_primary = if attr_header.is_non_resident == 0 { - true - } else { - let nr_offset = offset + 16; - if nr_offset + 8 <= data.len() { - let lowest_vcn = i64::from_le_bytes( - data[nr_offset..nr_offset + 8].try_into().unwrap_or([0; 8]), - ); - lowest_vcn == 0 - } else { - false + let (size, allocated) = read_size_allocated(data, offset, &attr_header); + let stream_name = if attr_name.is_empty() { + match attr_type { + Some(AttributeType::Bitmap) => String::from("$BITMAP"), + Some(AttributeType::IndexRoot) => String::from("$INDEX_ROOT"), + Some(AttributeType::IndexAllocation) => { + String::from("$INDEX_ALLOCATION") + } + _ => String::new(), } + } else { + attr_name }; - - if is_primary { - let (size, allocated) = if attr_header.is_non_resident == 0 { - (u64::from(rd_u32(data, offset + 16)), 0_u64) - } else { - let nr_offset = offset + 16; - if nr_offset + 48 <= data.len() { - let alloc_bytes = &data[nr_offset + 24..nr_offset + 32]; - let allocated = - i64::from_le_bytes(alloc_bytes.try_into().unwrap_or([0; 8])); - let size_bytes = &data[nr_offset + 32..nr_offset + 40]; - let data_size = - i64::from_le_bytes(size_bytes.try_into().unwrap_or([0; 8])); - (nonneg_to_u64(data_size), nonneg_to_u64(allocated)) - } else { - (0_u64, 0_u64) - } - }; - - let stream_name = if attr_name.is_empty() { - match attr_type { - Some(AttributeType::Bitmap) => String::from("$BITMAP"), - Some(AttributeType::IndexRoot) => String::from("$INDEX_ROOT"), - Some(AttributeType::IndexAllocation) => { - String::from("$INDEX_ALLOCATION") - } - _ => String::new(), - } - } else { - attr_name - }; - let is_resident = attr_header.is_non_resident == 0; - let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; - streams.push((stream_name, size, allocated, is_sparse, is_resident)); - } + let (is_resident, is_sparse) = resident_and_sparse(&attr_header); + streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } Some( @@ -407,57 +404,9 @@ pub(super) fn parse_extension_to_index( | AttributeType::AttributeList, ) => { // All counted as streams - let is_primary = if attr_header.is_non_resident == 0 { - true - } else { - let nr_offset = offset + 16; - if nr_offset + 8 <= data.len() { - let lowest_vcn = i64::from_le_bytes( - data[nr_offset..nr_offset + 8].try_into().unwrap_or([0; 8]), - ); - lowest_vcn == 0 - } else { - false - } - }; - - if is_primary { - let attr_name = if attr_header.name_length > 0 { - let name_offset = offset + usize::from(attr_header.name_offset); - let name_len = usize::from(attr_header.name_length); - if name_offset + name_len * 2 <= data.len() { - let name_bytes = &data[name_offset..name_offset + name_len * 2]; - let name_u16: SmallVec<[u16; 64]> = name_bytes - .as_chunks::<2>() - .0 - .iter() - .map(|c| u16::from_le_bytes(*c)) - .collect(); - crate::io::parser::unified::decode_name_u16(&name_u16).0 - } else { - String::new() - } - } else { - String::new() - }; - - let (size, allocated) = if attr_header.is_non_resident == 0 { - (u64::from(rd_u32(data, offset + 16)), 0_u64) - } else { - let nr_offset = offset + 16; - if nr_offset + 48 <= data.len() { - let alloc_bytes = &data[nr_offset + 24..nr_offset + 32]; - let allocated = - i64::from_le_bytes(alloc_bytes.try_into().unwrap_or([0; 8])); - let size_bytes = &data[nr_offset + 32..nr_offset + 40]; - let data_size = - i64::from_le_bytes(size_bytes.try_into().unwrap_or([0; 8])); - (nonneg_to_u64(data_size), nonneg_to_u64(allocated)) - } else { - (0_u64, 0_u64) - } - }; - + if is_primary_attribute(data, offset, &attr_header) { + let attr_name = extract_attr_name(data, offset, &attr_header); + let (size, allocated) = read_size_allocated(data, offset, &attr_header); let stream_name = if attr_name.is_empty() { match attr_type { Some(AttributeType::ObjectId) => String::from("$OBJECT_ID"), @@ -480,8 +429,7 @@ pub(super) fn parse_extension_to_index( } else { attr_name }; - let is_resident = attr_header.is_non_resident == 0; - let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + let (is_resident, is_sparse) = resident_and_sparse(&attr_header); streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } @@ -492,64 +440,16 @@ pub(super) fn parse_extension_to_index( // Unknown attribute types — counted as streams (catch-all). let type_code = attr_header.type_code; - let is_primary = if attr_header.is_non_resident == 0 { - true - } else { - let nr_offset = offset + 16; - if nr_offset + 8 <= data.len() { - let lowest_vcn = i64::from_le_bytes( - data[nr_offset..nr_offset + 8].try_into().unwrap_or([0; 8]), - ); - lowest_vcn == 0 - } else { - false - } - }; - - if is_primary { - let attr_name = if attr_header.name_length > 0 { - let name_offset = offset + usize::from(attr_header.name_offset); - let name_len = usize::from(attr_header.name_length); - if name_offset + name_len * 2 <= data.len() { - let name_bytes = &data[name_offset..name_offset + name_len * 2]; - let name_u16: SmallVec<[u16; 64]> = name_bytes - .as_chunks::<2>() - .0 - .iter() - .map(|c| u16::from_le_bytes(*c)) - .collect(); - crate::io::parser::unified::decode_name_u16(&name_u16).0 - } else { - String::new() - } - } else { - String::new() - }; - - let (size, allocated) = if attr_header.is_non_resident == 0 { - (u64::from(rd_u32(data, offset + 16)), 0_u64) - } else { - let nr_offset = offset + 16; - if nr_offset + 48 <= data.len() { - let alloc_bytes = &data[nr_offset + 24..nr_offset + 32]; - let allocated = - i64::from_le_bytes(alloc_bytes.try_into().unwrap_or([0; 8])); - let size_bytes = &data[nr_offset + 32..nr_offset + 40]; - let data_size = - i64::from_le_bytes(size_bytes.try_into().unwrap_or([0; 8])); - (nonneg_to_u64(data_size), nonneg_to_u64(allocated)) - } else { - (0_u64, 0_u64) - } - }; + if is_primary_attribute(data, offset, &attr_header) { + let attr_name = extract_attr_name(data, offset, &attr_header); + let (size, allocated) = read_size_allocated(data, offset, &attr_header); let stream_name = if attr_name.is_empty() { format!("$UNKNOWN_0x{type_code:X}") } else { attr_name }; - let is_resident = attr_header.is_non_resident == 0; - let is_sparse = !is_resident && (attr_header.flags & 0x8000) != 0; + let (is_resident, is_sparse) = resident_and_sparse(&attr_header); streams.push((stream_name, size, allocated, is_sparse, is_resident)); } } @@ -575,7 +475,7 @@ pub(super) fn parse_extension_to_index( // Add names/streams using helpers let link_indices: Vec = names .iter() - .map(|(name, parent)| add_link_to_index(index, name, *parent)) + .map(|(name, parent, ..)| add_link_to_index(index, name, *parent)) .collect(); let stream_indices: Vec = streams .iter() @@ -586,7 +486,7 @@ pub(super) fn parse_extension_to_index( // Ensure parent directories exist for the new names. Parser-local // raw `u64` lifts to typed `Frs` at the typed-API boundary. - for (_, parent_frs) in &names { + for (_, parent_frs, ..) in &names { if *parent_frs != base_frs && *parent_frs != 0 { let _ = index.get_or_create(crate::frs::Frs::new(*parent_frs)); } @@ -622,6 +522,18 @@ pub(super) fn parse_extension_to_index( let first_link = &index.links[u32_as_usize(link_indices[0])]; record.first_name.name = first_link.name; record.first_name.parent_frs = first_link.parent_frs; + // `LinkInfo` has no room for namespace/timestamps, so pull + // them from the original `names` entry instead (same index, + // `link_indices` was built via `names.iter().map(..)`). + // Base record has no $FILE_NAME of its own, so nothing else + // in the pipeline will ever set these otherwise. + let (_, _, namespace, fn_created, fn_modified, fn_accessed, fn_mft_changed) = + names[0]; + record.namespace = namespace; + record.fn_created = fn_created; + record.fn_modified = fn_modified; + record.fn_accessed = fn_accessed; + record.fn_mft_changed = fn_mft_changed; // Don't increment name_count for the first name (it's already counted as 1) // Chain remaining links (if any) to first_name.next_entry @@ -789,7 +701,7 @@ pub(super) fn parse_extension_to_index( // Use the name_count from BEFORE link-chaining to avoid overflow let existing_name_count = pre_chain_name_count; - for (name_idx, (_, parent_frs)) in names.iter().enumerate() { + for (name_idx, (_, parent_frs, ..)) in names.iter().enumerate() { let p_frs = *parent_frs; if p_frs == base_frs || p_frs == u64::from(NO_ENTRY) { continue; diff --git a/scripts/ci/file_size_exceptions.txt b/scripts/ci/file_size_exceptions.txt index 7c0b95561..3eb427eb6 100644 --- a/scripts/ci/file_size_exceptions.txt +++ b/scripts/ci/file_size_exceptions.txt @@ -7,7 +7,6 @@ crates/uffs-core/src/search/field/field_metadata.rs|PERMANENT: Single const fn m crates/uffs-core/src/search/filters/tests.rs|PERMANENT: Integration test suite for filter pipeline; splitting further would scatter related test fixtures crates/uffs-core/src/search/filters/mod.rs|PERMANENT: Cohesive SearchFilters/SearchFilterParams definitions + from_params construction; kept together so the full per-field filter contract is auditable in one place crates/uffs-client/src/schema/field_metadata.rs|PERMANENT: Single const fn match table — one FieldMeta per FieldId variant; mirrors uffs-core version -crates/uffs-mft/src/parse/direct_index_extension.rs|PERMANENT: Extension record parser for the direct-to-index pipeline; same structural reasoning as its base-record counterpart crates/uffs-mft/src/reader/index_read.rs|PERMANENT: Single impl MftReader block with tightly coupled cfg-gated pipeline stages crates/uffs-diag/src/bin/compare_scan_parity.rs|PERMANENT: Standalone diagnostic binary; single-file readability outweighs LOC policy for tooling crates/uffs-mcp/src/cookbook.rs|PERMANENT: Declarative JSON data (curated agent cookbook examples); splitting by line count would fragment the cohesive narrative From cd1bce6f082c7e9c4c9779bc6b82eaecfa716718 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:42:03 -0700 Subject: [PATCH 7/7] docs: fix stale "Fast vs Full Mode" claim in CLAUDE.md Found during this same MFT-parser audit: CLAUDE.md described a fast/full mode toggle where the default path "skips extension MFT records" and --full "merges" them. Neither exists anymore -- confirmed via direct grep that no CLI flag, no fast_mode/full_mode concept, and no skip-extension-records behavior exists anywhere in uffs-cli or the current reader code. Both production parsers (unified.rs, direct_index.rs + direct_index_extension.rs) always merge extension records in the same pass; this was pre-unification legacy behavior that the "one function processes ALL records" consolidation (unified.rs's own module doc) already superseded. --- CLAUDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6b7a80c28..cf157d4f5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -106,8 +106,8 @@ MFT reading supports multiple modes auto-selected by drive type: Records are parsed with `parse_record_zero_alloc` (thread-local buffers, zero heap allocation per record). Output uses SoA (Struct-of-Arrays) layout — parse directly into column vectors, not `Vec`. -### Fast vs Full Mode -Default ("fast") skips extension MFT records (~1% of files with many hard links/ADS), giving 15–25% faster reads. `--full` mode merges extension records for complete data. +### Extension Record Merging +The default record parsers (`io::parser::unified::process_record`, `parse::direct_index::parse_record_to_index` + `direct_index_extension.rs`) always merge extension MFT records (hard links/ADS beyond what fits in the base record) into their base record in the same pass — there is no fast/full toggle on this path; that split was a pre-unification legacy behavior and no longer exists in the CLI or the live query pipeline. ### Access Broker (Windows — non-elevated MFT reads) Reading the live MFT normally needs Administrator. The **Access Broker** (`uffs-broker`, a `LocalSystem` Windows service) lets the daemon run **non-elevated**: the broker opens the volume, `DuplicateHandle`s an elevated, `FILE_FLAG_OVERLAPPED` handle into the daemon over a named pipe (after verifying the client is `uffsd` + Authenticode via `WinVerifyTrust`), and the daemon adopts a duplicate of it for every MFT/USN/`$MFT`-extent read. The handle registry + `try_adopt_broker_handle` live in `uffs-mft::platform::volume`; the daemon warms up handles in `warm_up_broker_handles` only when **not** already elevated. On the broker handle, use overlapped-offset reads (`read_handle_at`), not `SetFilePointerEx` (which has no synchronous file pointer there). One-time setup: `uffs-broker --install` → no UAC on any later search. Full design + the production follow-ups (all landed) are in `docs/architecture/access-broker-followups.md`.