From 7e7da9c490306b0d23763534478ff897a1a2cfec Mon Sep 17 00:00:00 2001 From: Thomas Zarebczan Date: Wed, 14 Oct 2026 17:19:38 -0400 Subject: [PATCH 1/4] Follow $MFT's $ATTRIBUTE_LIST when the table is fragmented Record 0 only holds the first $DATA run list once $MFT spills into extension records. Read those records from the runs already in hand, append the rest in starting-VCN order, and warn on stderr when the joined map is still short of MftValidDataLength. --- native/diskhound-native-scanner/src/main.rs | 2 + native/diskhound-native-scanner/src/mft.rs | 235 +++---- .../diskhound-native-scanner/src/mft_runs.rs | 660 ++++++++++++++++++ 3 files changed, 758 insertions(+), 139 deletions(-) create mode 100644 native/diskhound-native-scanner/src/mft_runs.rs diff --git a/native/diskhound-native-scanner/src/main.rs b/native/diskhound-native-scanner/src/main.rs index d1ee8334..e162a895 100644 --- a/native/diskhound-native-scanner/src/main.rs +++ b/native/diskhound-native-scanner/src/main.rs @@ -26,6 +26,8 @@ mod usn_aggregate; #[cfg(windows)] mod mft; +#[cfg(any(windows, test))] +mod mft_runs; mod sample; mod clone_attrs; diff --git a/native/diskhound-native-scanner/src/mft.rs b/native/diskhound-native-scanner/src/mft.rs index a8571c9c..943c68bc 100644 --- a/native/diskhound-native-scanner/src/mft.rs +++ b/native/diskhound-native-scanner/src/mft.rs @@ -31,10 +31,10 @@ //! //! ## What this file is NOT //! -//! - We don't support $ATTRIBUTE_LIST (0x20) expansion for records whose -//! attributes overflow a single record. These are rare (very fragmented -//! files, giant directories); we log and skip them. This is fine -//! because the goal is "fast common-case", not "archivally complete". +//! - `$MFT`'s own run map follows `$ATTRIBUTE_LIST` (0x20) on record 0 +//! when the table's `$DATA` extents do not all fit in that record. +//! Other files still do not expand `$ATTRIBUTE_LIST` for names; an +//! extension record is read only to recover `$DATA` size. //! - Encrypted files use the same $DATA allocated_size as compressed/ //! sparse files (cluster occupancy). We do not decrypt content. //! - We don't process $FILE_NAME entries with DOS-only namespace (we @@ -251,13 +251,6 @@ fn volume_handle_to_file(volume: VolumeHandle) -> std::fs::File { unsafe { std::fs::File::from_raw_handle(raw as _) } } -/// Parsed MFT record header + pointer offsets. `record_buf` contains the -/// full record bytes after USA fixup. -struct RecordHeader { - flags: u16, - first_attr_offset: u16, -} - /// Apply the Update Sequence Array fixup in-place. Each 512-byte sector's /// last 2 bytes get replaced with the corresponding USA entry. fn apply_usa_fixup(buf: &mut [u8], bytes_per_sector: u32) -> bool { @@ -297,30 +290,6 @@ fn apply_usa_fixup(buf: &mut [u8], bytes_per_sector: u32) -> bool { true } -fn parse_record_header(buf: &[u8]) -> Option { - if buf.len() < 48 || &buf[0..4] != FILE_RECORD_MAGIC { - return None; - } - let first_attr_offset = u16::from_le_bytes([buf[20], buf[21]]); - let flags = u16::from_le_bytes([buf[22], buf[23]]); - // BaseFileRecordSegment at offset 32 (u64). If non-zero this record - // is an extension of another record — we skip those for simplicity. - let base_frn = u64::from_le_bytes([ - buf[32], buf[33], buf[34], buf[35], buf[36], buf[37], buf[38], buf[39], - ]) & 0x0000_FFFF_FFFF_FFFF; - if base_frn != 0 { - return None; - } - // We do not read the header's MFTRecordNumber field (offset 44) — - // it's only present on NTFS v3.0+ and some records carry zero here. - // Instead the caller computes the record number from the record's - // position within the logical MFT stream. - Some(RecordHeader { - flags, - first_attr_offset, - }) -} - struct ParsedAttrs { mtime_ms: Option, /// Every `$FILE_NAME` attribute in the record, pre-dedup. Namespace @@ -507,110 +476,98 @@ fn ntfs_time_to_unix_ms(ntfs_ticks: u64) -> u64 { (ntfs_ticks - WINDOWS_TO_UNIX_EPOCH_TICKS) / 10_000 } -/// A single (LCN, cluster_count) extent of a non-resident attribute's -/// data. Returned by `parse_data_runs` when we follow $MFT's own $DATA -/// attribute to discover where on disk each piece of the MFT lives. -#[derive(Debug, Clone, Copy)] -struct DataRun { - lcn: i64, // signed; relative accumulator can go negative for sparse runs - cluster_count: u64, -} +/// Record 0's unnamed `$DATA` runs, plus every extension record named by +/// its `$ATTRIBUTE_LIST`. Extension records sit in the part of `$MFT` the +/// base runs already cover; a later fragment can reveal the next one. +fn resolve_mft_runs( + volume: &mut std::fs::File, + record_0: &[u8], + record_size: usize, + bytes_per_cluster: u64, + bytes_per_sector: u32, + valid_data_length: i64, +) -> Result, MftError> { + use crate::mft_runs::{ + absorb_extension_records, extension_records_from_attribute_list, join_run_segments, + logical_byte_length, read_logical, record_0_mft_map, + }; -/// Parse an NTFS data run list starting at `runs` and return the absolute -/// extents. The stream ends on a zero header byte. Sparse runs (zero -/// offset length) are represented with lcn=-1 and skipped by callers. -fn parse_data_runs(mut runs: &[u8]) -> Vec { - let mut out = Vec::new(); - let mut current_lcn: i64 = 0; - while !runs.is_empty() { - let header = runs[0]; - if header == 0 { - break; - } - let offset_len = (header >> 4) as usize; - let count_len = (header & 0x0F) as usize; - if count_len == 0 || 1 + count_len + offset_len > runs.len() { - break; - } - // Read unsigned cluster count (little-endian, count_len bytes). - let mut count: u64 = 0; - for i in 0..count_len { - count |= (runs[1 + i] as u64) << (8 * i); - } - // Read signed offset delta (little-endian, offset_len bytes, - // sign-extended from the top byte). A length of 0 means SPARSE. - if offset_len == 0 { - out.push(DataRun { - lcn: -1, - cluster_count: count, - }); + let invalid = || { + MftError::VolumeQueryFailed(io::Error::new( + io::ErrorKind::InvalidData, + "could not parse $MFT's $DATA run list from record 0", + )) + }; + let map = record_0_mft_map(record_0).ok_or_else(invalid)?; + let mut segments = map.segments; + let mut pending = map.extension_records; + if let Some(external) = map.external_attribute_list { + let available = logical_byte_length(&external.runs, bytes_per_cluster).unwrap_or(0); + let capped = (external.byte_length.min(available).min(1_u64 << 20)) as usize; + let mut list = vec![0u8; capped]; + let read = if capped == 0 { + Ok(false) } else { - let mut offset_delta: i64 = 0; - let delta_bytes = &runs[1 + count_len..1 + count_len + offset_len]; - for (i, byte) in delta_bytes.iter().enumerate() { - offset_delta |= (*byte as i64) << (8 * i); - } - // Sign-extend from offset_len bytes to 64 bits. - let sign_bit = 1_i64 << (8 * offset_len - 1); - if offset_delta & sign_bit != 0 { - offset_delta |= !((1_i64 << (8 * offset_len)) - 1); + read_logical(volume, &external.runs, 0, &mut list, bytes_per_cluster) + }; + match read { + Ok(true) => pending = extension_records_from_attribute_list(&list), + Ok(false) => { + eprintln!( + "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT could not be read; using record 0's run list only" + ); } - current_lcn = current_lcn.wrapping_add(offset_delta); - out.push(DataRun { - lcn: current_lcn, - cluster_count: count, - }); + Err(err) => return Err(err.into()), } - runs = &runs[1 + count_len + offset_len..]; } - out -} -/// Find the non-resident $DATA attribute in $MFT's own record and return -/// its run list. Record 0 is $MFT itself; its $DATA run list IS the map -/// of "where on disk are the MFT records". -fn extract_mft_data_runs( - mft_record_0: &[u8], -) -> Option> { - let header = parse_record_header(mft_record_0)?; - if header.flags & FLAG_IN_USE == 0 { - return None; - } - let mut pos = header.first_attr_offset as usize; - while pos + 16 <= mft_record_0.len() { - let attr_type = u32::from_le_bytes([ - mft_record_0[pos], mft_record_0[pos + 1], - mft_record_0[pos + 2], mft_record_0[pos + 3], - ]); - if attr_type == ATTR_END_MARKER { - break; - } - let attr_len = u32::from_le_bytes([ - mft_record_0[pos + 4], mft_record_0[pos + 5], - mft_record_0[pos + 6], mft_record_0[pos + 7], - ]) as usize; - if attr_len < 16 || pos + attr_len > mft_record_0.len() { - break; - } - let non_resident = mft_record_0[pos + 8]; - if attr_type == ATTR_DATA && non_resident != 0 { - // Non-resident attribute header: runlist offset at offset 32. - let runlist_offset = u16::from_le_bytes([ - mft_record_0[pos + 32], mft_record_0[pos + 33], - ]) as usize; - let run_start = pos + runlist_offset; - let run_end = pos + attr_len; - if run_start < run_end && run_end <= mft_record_0.len() { - return Some(parse_data_runs(&mft_record_0[run_start..run_end])); + let unresolved = absorb_extension_records( + &mut segments, + pending, + record_size, + |runs_now, logical, buf| { + let found = read_logical(volume, runs_now, logical, buf, bytes_per_cluster)?; + let readable = found + && buf.len() >= 4 + && &buf[0..4] == FILE_RECORD_MAGIC + && apply_usa_fixup(buf, bytes_per_sector); + if found && !readable { + buf[..4].fill(0); } - } - pos += attr_len; + Ok(found) + }, + )?; + + let (runs, had_gap) = join_run_segments(&segments); + if runs.iter().all(|run| run.lcn < 0) { + return Err(invalid()); } - None + // Same total the "data runs" log prints: real clusters, not holes + // inserted where an extension fragment was missing. + let covered_clusters: u64 = runs + .iter() + .filter(|run| run.lcn >= 0) + .map(|run| run.cluster_count) + .sum(); + let covered = covered_clusters.saturating_mul(bytes_per_cluster); + let valid = if valid_data_length > 0 { + valid_data_length as u64 + } else { + 0 + }; + if had_gap || !unresolved.is_empty() || (valid > 0 && covered < valid) { + eprintln!( + "[diskhound-native-scanner] mft: warning: $MFT run list covers {:.1} MB of {:.1} MB valid data ({} extension records unread)", + covered as f64 / (1024.0 * 1024.0), + valid as f64 / (1024.0 * 1024.0), + unresolved.len(), + ); + } + Ok(runs) } -/// Read the entire MFT into a FRN → MftRecordParsed map by following the -/// data runs discovered in $MFT's own record 0. +/// Read the entire MFT into a FRN → MftRecordParsed map by following +/// `$MFT`'s data runs, including extents stored in extension records. /// /// Why not just seek to MftStartLcn and read MftValidDataLength bytes? /// Because the MFT is typically fragmented on any drive that's seen real @@ -646,12 +603,14 @@ fn read_all_mft_records( return Err(MftError::InvalidMftRecord); } apply_usa_fixup(&mut record_0, bytes_per_sector); - let runs = extract_mft_data_runs(&record_0).ok_or_else(|| { - MftError::VolumeQueryFailed(io::Error::new( - io::ErrorKind::InvalidData, - "could not parse $MFT's $DATA run list from record 0", - )) - })?; + let runs = resolve_mft_runs( + &mut volume, + &record_0, + record_size, + bytes_per_cluster, + bytes_per_sector, + volume_data.MftValidDataLength, + )?; let total_clusters: u64 = runs .iter() .filter(|r| r.lcn >= 0) @@ -738,10 +697,8 @@ fn read_all_mft_records( continue; } - // Parse header WITHOUT skipping extension records — we - // need the size data they carry. parse_record_header - // used to return None for base_frn != 0; we pull that - // check inline here so we can see extensions. + // Keep extension records. Their $DATA size belongs to the + // base file; skipping a non-zero base reference would drop it. if rec_buf.len() < 48 || &rec_buf[0..4] != FILE_RECORD_MAGIC { continue; } diff --git a/native/diskhound-native-scanner/src/mft_runs.rs b/native/diskhound-native-scanner/src/mft_runs.rs new file mode 100644 index 00000000..f4fe05d5 --- /dev/null +++ b/native/diskhound-native-scanner/src/mft_runs.rs @@ -0,0 +1,660 @@ +//! Where the bytes of `$MFT` itself live. +//! +//! Record 0 is `$MFT`. Its unnamed `$DATA` run list is only the first +//! piece of the table. When that piece is not the whole file, NTFS adds +//! an `$ATTRIBUTE_LIST` (0x20) and stores the rest of the run list in +//! extension records. Those records are read through the runs gathered +//! so far and appended in starting-VCN order. + +use std::io::{self, Read, Seek, SeekFrom}; + +/// One extent of a non-resident attribute. `lcn == -1` is a sparse hole. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DataRun { + pub lcn: i64, + pub cluster_count: u64, +} + +/// Unnamed `$DATA` fragment: the logical cluster where it starts, then +/// its own run list (offsets in that list are relative to this fragment). +pub type DataSegment = (u64, Vec); + +/// A non-resident `$ATTRIBUTE_LIST` whose bytes live in these runs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExternalAttributeList { + pub runs: Vec, + pub byte_length: u64, +} + +/// What record 0 says about `$MFT`'s own mapping. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Record0Map { + pub segments: Vec, + /// Extension records that hold further unnamed `$DATA` (never 0). + pub extension_records: Vec, + pub external_attribute_list: Option, +} + +const FILE_RECORD_MAGIC: &[u8; 4] = b"FILE"; +const FLAG_IN_USE: u16 = 0x0001; +const ATTR_ATTRIBUTE_LIST: u32 = 0x20; +const ATTR_DATA: u32 = 0x80; +const ATTR_END_MARKER: u32 = 0xFFFF_FFFF; +const FILE_REF_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; + +/// Run list of an unnamed non-resident attribute, absolute LCNs. +pub fn parse_data_runs(mut runs: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut current_lcn: i64 = 0; + while !runs.is_empty() { + let header = runs[0]; + if header == 0 { + break; + } + let offset_len = (header >> 4) as usize; + let count_len = (header & 0x0F) as usize; + if count_len == 0 || 1 + count_len + offset_len > runs.len() { + break; + } + let mut count: u64 = 0; + for i in 0..count_len { + count |= (runs[1 + i] as u64) << (8 * i); + } + if offset_len == 0 { + out.push(DataRun { + lcn: -1, + cluster_count: count, + }); + } else { + let mut offset_delta: i64 = 0; + let delta_bytes = &runs[1 + count_len..1 + count_len + offset_len]; + for (i, byte) in delta_bytes.iter().enumerate() { + offset_delta |= (*byte as i64) << (8 * i); + } + let sign_bit = 1_i64 << (8 * offset_len - 1); + if offset_delta & sign_bit != 0 { + offset_delta |= !((1_i64 << (8 * offset_len)) - 1); + } + current_lcn = current_lcn.wrapping_add(offset_delta); + out.push(DataRun { + lcn: current_lcn, + cluster_count: count, + }); + } + runs = &runs[1 + count_len + offset_len..]; + } + out +} + +/// Record 0's unnamed `$DATA` pieces, plus where the rest of the run +/// list lives. `None` when this is not an in-use FILE record. +pub fn record_0_mft_map(record: &[u8]) -> Option { + let mut map = Record0Map { + segments: Vec::new(), + extension_records: Vec::new(), + external_attribute_list: None, + }; + if !walk_attributes(record, &mut map, true) { + return None; + } + if map.segments.is_empty() + && map.extension_records.is_empty() + && map.external_attribute_list.is_none() + { + return None; + } + Some(map) +} + +/// Unnamed `$DATA` fragments stored in one extension record. +pub fn unnamed_data_segments(record: &[u8]) -> Vec { + let mut map = Record0Map { + segments: Vec::new(), + extension_records: Vec::new(), + external_attribute_list: None, + }; + if !walk_attributes(record, &mut map, false) { + return Vec::new(); + } + map.segments +} + +/// Extension record numbers named by an `$ATTRIBUTE_LIST` value. +/// Entries for record 0, named attributes, and anything other than +/// `$DATA` are ignored. +pub fn extension_records_from_attribute_list(list: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut offset = 0; + while offset + 26 <= list.len() { + let rec_len = u16::from_le_bytes([list[offset + 4], list[offset + 5]]) as usize; + if rec_len < 26 || offset + rec_len > list.len() { + break; + } + let attr_type = u32::from_le_bytes([ + list[offset], + list[offset + 1], + list[offset + 2], + list[offset + 3], + ]); + let name_len = list[offset + 6]; + let file_ref = u64::from_le_bytes(list[offset + 16..offset + 24].try_into().unwrap()); + let record_number = file_ref & FILE_REF_MASK; + if attr_type == ATTR_DATA + && name_len == 0 + && record_number != 0 + && !out.contains(&record_number) + { + out.push(record_number); + } + offset += rec_len; + } + out +} + +/// Concatenate fragments in starting-VCN order. A hole becomes a sparse +/// run so later extents keep their real record numbers. Overlapping +/// fragments are dropped. `had_gap` is set for either. +pub fn join_run_segments(segments: &[DataSegment]) -> (Vec, bool) { + let mut ordered: Vec<&DataSegment> = segments.iter().collect(); + ordered.sort_by_key(|(vcn, _)| *vcn); + let mut runs = Vec::new(); + let mut cursor: u64 = 0; + let mut had_gap = false; + for (vcn, frag_runs) in ordered { + let clusters = cluster_span(frag_runs); + if clusters == 0 { + continue; + } + if *vcn < cursor { + had_gap = true; + continue; + } + if *vcn > cursor { + runs.push(DataRun { + lcn: -1, + cluster_count: *vcn - cursor, + }); + had_gap = true; + cursor = *vcn; + } + runs.extend_from_slice(frag_runs); + cursor = cursor.saturating_add(clusters); + } + (runs, had_gap) +} + +pub fn logical_byte_length(runs: &[DataRun], bytes_per_cluster: u64) -> Option { + let mut total = 0u64; + for run in runs { + let bytes = run.cluster_count.checked_mul(bytes_per_cluster)?; + total = total.checked_add(bytes)?; + } + Some(total) +} + +/// Pull unnamed `$DATA` fragments out of each extension record. +/// A record the current map cannot reach stays pending until a fragment +/// just loaded covers it. Records that never become readable, or that +/// are not FILE records, are returned unread. +pub fn absorb_extension_records( + segments: &mut Vec, + mut pending: Vec, + record_size: usize, + mut read_record: impl FnMut(&[DataRun], u64, &mut [u8]) -> io::Result, +) -> io::Result> { + let mut unresolved = Vec::new(); + loop { + let (runs_now, _) = join_run_segments(segments); + let mut still_out_of_range = Vec::new(); + let mut progress = false; + for frn in pending { + let Some(logical) = frn.checked_mul(record_size as u64) else { + unresolved.push(frn); + continue; + }; + let mut buf = vec![0u8; record_size]; + if !read_record(&runs_now, logical, &mut buf)? { + still_out_of_range.push(frn); + continue; + } + let more = unnamed_data_segments(&buf); + if more.is_empty() { + unresolved.push(frn); + } else { + segments.extend(more); + progress = true; + } + } + if still_out_of_range.is_empty() || !progress { + unresolved.extend(still_out_of_range); + break; + } + pending = still_out_of_range; + } + Ok(unresolved) +} + +/// Disk byte and how many contiguous bytes of this run follow it. +/// `None` past the end or inside a sparse hole. +pub fn locate_logical( + runs: &[DataRun], + logical: u64, + bytes_per_cluster: u64, +) -> Option<(u64, u64)> { + if bytes_per_cluster == 0 { + return None; + } + let mut cursor = 0u64; + for run in runs { + let run_bytes = run.cluster_count.checked_mul(bytes_per_cluster)?; + let end = cursor.checked_add(run_bytes)?; + if logical < end { + if run.lcn < 0 { + return None; + } + let into = logical - cursor; + let disk = (run.lcn as u64) + .checked_mul(bytes_per_cluster)? + .checked_add(into)?; + return Some((disk, end - logical)); + } + cursor = end; + } + None +} + +/// Read `buf` from the logical stream described by `runs`. `Ok(false)` +/// when any byte is past the map or in a sparse hole. +pub fn read_logical( + volume: &mut R, + runs: &[DataRun], + mut logical: u64, + buf: &mut [u8], + bytes_per_cluster: u64, +) -> io::Result { + let mut filled = 0; + while filled < buf.len() { + let Some((disk, available)) = locate_logical(runs, logical, bytes_per_cluster) else { + return Ok(false); + }; + let n = available.min((buf.len() - filled) as u64) as usize; + if n == 0 { + return Ok(false); + } + volume.seek(SeekFrom::Start(disk))?; + volume.read_exact(&mut buf[filled..filled + n])?; + filled += n; + logical += n as u64; + } + Ok(true) +} + +fn cluster_span(runs: &[DataRun]) -> u64 { + runs.iter() + .fold(0u64, |sum, run| sum.saturating_add(run.cluster_count)) +} + +fn walk_attributes(record: &[u8], map: &mut Record0Map, parse_list: bool) -> bool { + if record.len() < 48 || &record[0..4] != FILE_RECORD_MAGIC { + return false; + } + let flags = u16::from_le_bytes([record[22], record[23]]); + if flags & FLAG_IN_USE == 0 { + return false; + } + let mut pos = u16::from_le_bytes([record[20], record[21]]) as usize; + while pos + 16 <= record.len() { + let attr_type = u32::from_le_bytes([ + record[pos], + record[pos + 1], + record[pos + 2], + record[pos + 3], + ]); + if attr_type == ATTR_END_MARKER { + break; + } + let attr_len = u32::from_le_bytes([ + record[pos + 4], + record[pos + 5], + record[pos + 6], + record[pos + 7], + ]) as usize; + if attr_len < 16 || pos + attr_len > record.len() { + break; + } + let non_resident = record[pos + 8]; + let name_len = record[pos + 9]; + if name_len == 0 && attr_type == ATTR_DATA && non_resident != 0 { + if let Some(segment) = data_segment(record, pos, attr_len) { + map.segments.push(segment); + } + } else if parse_list && name_len == 0 && attr_type == ATTR_ATTRIBUTE_LIST { + if non_resident == 0 { + if let Some(value) = resident_value(record, pos, attr_len) { + map.extension_records + .extend(extension_records_from_attribute_list(value)); + } + } else if let Some(external) = external_list(record, pos, attr_len) { + map.external_attribute_list = Some(external); + } + } + pos += attr_len; + } + true +} + +fn data_segment(buf: &[u8], pos: usize, attr_len: usize) -> Option { + if pos + 24 > buf.len() { + return None; + } + let starting_vcn = u64::from_le_bytes(buf[pos + 16..pos + 24].try_into().ok()?); + let runs = runs_in_attribute(buf, pos, attr_len)?; + if runs.is_empty() { + return None; + } + Some((starting_vcn, runs)) +} + +fn external_list(buf: &[u8], pos: usize, attr_len: usize) -> Option { + if pos + 64 > buf.len() { + return None; + } + let real_size = u64::from_le_bytes(buf[pos + 48..pos + 56].try_into().ok()?); + let initialized = u64::from_le_bytes(buf[pos + 56..pos + 64].try_into().ok()?); + let byte_length = if initialized > 0 { + initialized + } else { + real_size + }; + if byte_length == 0 { + return None; + } + let runs = runs_in_attribute(buf, pos, attr_len)?; + if runs.is_empty() { + return None; + } + Some(ExternalAttributeList { runs, byte_length }) +} + +fn resident_value(buf: &[u8], pos: usize, attr_len: usize) -> Option<&[u8]> { + if pos + 22 > buf.len() { + return None; + } + let value_len = u32::from_le_bytes(buf[pos + 16..pos + 20].try_into().ok()?) as usize; + let value_off = u16::from_le_bytes([buf[pos + 20], buf[pos + 21]]) as usize; + let start = pos + value_off; + let end = start + value_len; + if value_off < 24 || end > pos + attr_len || end > buf.len() { + return None; + } + Some(&buf[start..end]) +} + +fn runs_in_attribute(buf: &[u8], pos: usize, attr_len: usize) -> Option> { + if pos + 34 > buf.len() { + return None; + } + let runlist_offset = u16::from_le_bytes([buf[pos + 32], buf[pos + 33]]) as usize; + let run_start = pos + runlist_offset; + let run_end = pos + attr_len; + if run_start >= run_end || run_end > buf.len() { + return None; + } + Some(parse_data_runs(&buf[run_start..run_end])) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn file_record(attrs: &[u8]) -> Vec { + let mut buf = vec![0u8; 56 + attrs.len() + 8]; + buf[0..4].copy_from_slice(b"FILE"); + buf[20..22].copy_from_slice(&56u16.to_le_bytes()); + buf[22..24].copy_from_slice(&FLAG_IN_USE.to_le_bytes()); + let start = 56; + buf[start..start + attrs.len()].copy_from_slice(attrs); + let end = start + attrs.len(); + buf[end..end + 4].copy_from_slice(&ATTR_END_MARKER.to_le_bytes()); + buf + } + + /// One non-resident attribute: `count` clusters at absolute `lcn`. + fn nonresident( + attr_type: u32, + starting_vcn: u64, + count: u64, + lcn: i64, + initialized: u64, + ) -> Vec { + let runs = vec![0x11, count as u8, lcn as u8, 0]; + let header = 64usize; + let mut attr = vec![0u8; header + runs.len()]; + let attr_len = (attr.len() as u32).to_le_bytes(); + attr[0..4].copy_from_slice(&attr_type.to_le_bytes()); + attr[4..8].copy_from_slice(&attr_len); + attr[8] = 1; + attr[16..24].copy_from_slice(&starting_vcn.to_le_bytes()); + attr[32..34].copy_from_slice(&(header as u16).to_le_bytes()); + attr[48..56].copy_from_slice(&initialized.to_le_bytes()); + attr[56..64].copy_from_slice(&initialized.to_le_bytes()); + attr[header..].copy_from_slice(&runs); + attr + } + + fn list_entry(attr_type: u32, record: u64, name_len: u8) -> Vec { + let mut entry = vec![0u8; 32]; + entry[0..4].copy_from_slice(&attr_type.to_le_bytes()); + entry[4..6].copy_from_slice(&32u16.to_le_bytes()); + entry[6] = name_len; + entry[16..24].copy_from_slice(&record.to_le_bytes()); + entry + } + + fn resident_list(entries: &[u8]) -> Vec { + let value_off = 24u16; + let mut attr = vec![0u8; value_off as usize + entries.len()]; + let attr_len = (attr.len() as u32).to_le_bytes(); + attr[0..4].copy_from_slice(&ATTR_ATTRIBUTE_LIST.to_le_bytes()); + attr[4..8].copy_from_slice(&attr_len); + attr[16..20].copy_from_slice(&(entries.len() as u32).to_le_bytes()); + attr[20..22].copy_from_slice(&value_off.to_le_bytes()); + attr[value_off as usize..].copy_from_slice(entries); + attr + } + + #[test] + fn base_record_without_a_list_keeps_its_own_runs() { + let record = file_record(&nonresident(ATTR_DATA, 0, 4, 10, 0)); + let map = record_0_mft_map(&record).unwrap(); + assert!(map.extension_records.is_empty()); + assert!(map.external_attribute_list.is_none()); + let (runs, gap) = join_run_segments(&map.segments); + assert!(!gap); + assert_eq!( + runs, + vec![DataRun { + lcn: 10, + cluster_count: 4 + }] + ); + } + + #[test] + fn attribute_list_names_extension_records_and_ignores_the_base_and_named_streams() { + let mut entries = Vec::new(); + entries.extend(list_entry(ATTR_DATA, 0, 0)); + // High 16 bits are the sequence number, not part of the record number. + entries.extend(list_entry(ATTR_DATA, 16 | (7u64 << 48), 0)); + entries.extend(list_entry(ATTR_DATA, 16, 0)); + entries.extend(list_entry(ATTR_DATA, 40, 1)); + entries.extend(list_entry(0x30, 16, 0)); + let list = resident_list(&entries); + let mut attrs = nonresident(ATTR_DATA, 0, 8, 3, 0); + attrs.extend(list); + let map = record_0_mft_map(&file_record(&attrs)).unwrap(); + assert_eq!(map.extension_records, vec![16]); + } + + #[test] + fn extension_run_is_appended_in_vcn_order_even_when_parsed_first() { + let base = unnamed_data_segments(&file_record(&nonresident(ATTR_DATA, 0, 100, 5, 0))); + let ext = unnamed_data_segments(&file_record(&nonresident(ATTR_DATA, 100, 80, 50, 0))); + let mut segments = ext; + segments.extend(base); + let (runs, gap) = join_run_segments(&segments); + assert!(!gap); + assert_eq!(runs.len(), 2); + assert_eq!(runs[0].lcn, 5); + assert_eq!(runs[0].cluster_count, 100); + assert_eq!(runs[1].lcn, 50); + assert_eq!(runs[1].cluster_count, 80); + assert_eq!(logical_byte_length(&runs, 4096), Some(180 * 4096)); + } + + #[test] + fn a_missing_middle_fragment_stays_a_hole_so_later_records_keep_their_numbers() { + let segments = vec![ + unnamed_data_segments(&file_record(&nonresident(ATTR_DATA, 0, 10, 1, 0))).remove(0), + unnamed_data_segments(&file_record(&nonresident(ATTR_DATA, 30, 10, 2, 0))).remove(0), + ]; + let (runs, gap) = join_run_segments(&segments); + assert!(gap); + assert_eq!( + runs, + vec![ + DataRun { + lcn: 1, + cluster_count: 10 + }, + DataRun { + lcn: -1, + cluster_count: 20 + }, + DataRun { + lcn: 2, + cluster_count: 10 + }, + ] + ); + } + + #[test] + fn nonresident_attribute_list_is_returned_for_the_caller_to_read() { + let list = nonresident(ATTR_ATTRIBUTE_LIST, 0, 1, 9, 128); + let data = nonresident(ATTR_DATA, 0, 2, 4, 0); + let mut attrs = data; + attrs.extend(list); + let map = record_0_mft_map(&file_record(&attrs)).unwrap(); + let external = map.external_attribute_list.unwrap(); + assert_eq!(external.byte_length, 128); + assert_eq!(external.runs[0].lcn, 9); + assert!(map.extension_records.is_empty()); + } + + #[test] + fn logical_read_crosses_a_run_boundary() { + let runs = vec![ + DataRun { + lcn: 0, + cluster_count: 1, + }, + DataRun { + lcn: 5, + cluster_count: 1, + }, + ]; + let disk = vec![0u8; 6 * 512]; + let mut image = disk; + image[0..512].fill(1); + image[5 * 512..6 * 512].fill(2); + let mut buf = [0u8; 600]; + assert_eq!( + read_logical(&mut Cursor::new(image), &runs, 400, &mut buf, 512).unwrap(), + true + ); + assert!(buf[..112].iter().all(|b| *b == 1)); + assert!(buf[112..].iter().all(|b| *b == 2)); + let mut hole = [0u8; 4]; + let sparse = vec![DataRun { + lcn: -1, + cluster_count: 1, + }]; + assert_eq!( + read_logical( + &mut Cursor::new(Vec::::new()), + &sparse, + 0, + &mut hole, + 512 + ) + .unwrap(), + false + ); + } + + #[test] + fn extension_records_are_read_from_the_base_map_and_then_from_each_other() { + let bpc = 1024u64; + let record_size = 1024usize; + // Records 0 and 1 live in the base extent. Record 1 points at a + // second extent, and record 2 lives only in that second extent. + let mut image = vec![0u8; 11 * record_size]; + let rec1 = file_record(&nonresident(ATTR_DATA, 2, 1, 10, 0)); + image[record_size..record_size + rec1.len()].copy_from_slice(&rec1); + let rec2 = file_record(&nonresident(ATTR_DATA, 3, 4, 20, 0)); + let rec2_at = 10 * record_size; + image[rec2_at..rec2_at + rec2.len()].copy_from_slice(&rec2); + + let mut segments = unnamed_data_segments(&file_record(&nonresident(ATTR_DATA, 0, 2, 0, 0))); + let unresolved = absorb_extension_records( + &mut segments, + vec![1, 2, 99], + record_size, + |runs, logical, buf| { + read_logical(&mut Cursor::new(image.clone()), runs, logical, buf, bpc) + }, + ) + .unwrap(); + assert_eq!(unresolved, vec![99]); + let (runs, gap) = join_run_segments(&segments); + assert!(!gap); + assert_eq!( + runs, + vec![ + DataRun { + lcn: 0, + cluster_count: 2 + }, + DataRun { + lcn: 10, + cluster_count: 1 + }, + DataRun { + lcn: 20, + cluster_count: 4 + }, + ] + ); + } + + #[test] + fn relative_run_offsets_accumulate() { + // Two runs: 3 clusters at LCN 10, then 2 clusters at delta -4 → LCN 6. + let bytes = [0x11, 3, 10, 0x11, 2, (-4i8) as u8, 0]; + assert_eq!( + parse_data_runs(&bytes), + vec![ + DataRun { + lcn: 10, + cluster_count: 3 + }, + DataRun { + lcn: 6, + cluster_count: 2 + }, + ] + ); + } +} From 23077cf636940fb19ce9052a08bd29a597b5fc34 Mon Sep 17 00:00:00 2001 From: Thomas Zarebczan Date: Wed, 14 Oct 2026 18:22:59 -0400 Subject: [PATCH 2/4] Keep a partial $MFT map when an extension read fails A non-resident $ATTRIBUTE_LIST is read in whole sectors, and only the initialized prefix is parsed. A failed list or extension read warns and continues with the runs already in hand. --- native/diskhound-native-scanner/src/mft.rs | 31 +++++++++++---- .../diskhound-native-scanner/src/mft_runs.rs | 39 +++++++++++++++++++ 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/native/diskhound-native-scanner/src/mft.rs b/native/diskhound-native-scanner/src/mft.rs index 943c68bc..64823b42 100644 --- a/native/diskhound-native-scanner/src/mft.rs +++ b/native/diskhound-native-scanner/src/mft.rs @@ -489,7 +489,7 @@ fn resolve_mft_runs( ) -> Result, MftError> { use crate::mft_runs::{ absorb_extension_records, extension_records_from_attribute_list, join_run_segments, - logical_byte_length, read_logical, record_0_mft_map, + logical_byte_length, read_logical, record_0_mft_map, sector_aligned_read_len, }; let invalid = || { @@ -503,21 +503,33 @@ fn resolve_mft_runs( let mut pending = map.extension_records; if let Some(external) = map.external_attribute_list { let available = logical_byte_length(&external.runs, bytes_per_cluster).unwrap_or(0); - let capped = (external.byte_length.min(available).min(1_u64 << 20)) as usize; - let mut list = vec![0u8; capped]; - let read = if capped == 0 { + // Initialized size is rarely a whole sector. `\\.\X:` rejects that + // read, which used to fail the scan instead of keeping record 0. + let want = external.byte_length.min(1_u64 << 20); + let read_len = + sector_aligned_read_len(want, bytes_per_sector as u64, available) as usize; + let mut list = vec![0u8; read_len]; + let read = if read_len == 0 { Ok(false) } else { read_logical(volume, &external.runs, 0, &mut list, bytes_per_cluster) }; match read { - Ok(true) => pending = extension_records_from_attribute_list(&list), + Ok(true) => { + let prefix = (external.byte_length as usize).min(list.len()); + pending = extension_records_from_attribute_list(&list[..prefix]); + } Ok(false) => { eprintln!( "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT could not be read; using record 0's run list only" ); } - Err(err) => return Err(err.into()), + Err(err) => { + eprintln!( + "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT could not be read ({}); using record 0's run list only", + err + ); + } } } @@ -526,7 +538,12 @@ fn resolve_mft_runs( pending, record_size, |runs_now, logical, buf| { - let found = read_logical(volume, runs_now, logical, buf, bytes_per_cluster)?; + // A bad sector in one extension must not drop the runs already + // in hand. The coverage warning reports what stayed unread. + let found = match read_logical(volume, runs_now, logical, buf, bytes_per_cluster) { + Ok(found) => found, + Err(_) => return Ok(false), + }; let readable = found && buf.len() >= 4 && &buf[0..4] == FILE_RECORD_MAGIC diff --git a/native/diskhound-native-scanner/src/mft_runs.rs b/native/diskhound-native-scanner/src/mft_runs.rs index f4fe05d5..53e5df1c 100644 --- a/native/diskhound-native-scanner/src/mft_runs.rs +++ b/native/diskhound-native-scanner/src/mft_runs.rs @@ -192,6 +192,23 @@ pub fn logical_byte_length(runs: &[DataRun], bytes_per_cluster: u64) -> Option u64 { + let sector = bytes_per_sector.max(1); + let room = available / sector * sector; + let want = logical_len.min(room); + if want == 0 { + return 0; + } + let aligned = want.saturating_add(sector - 1) / sector * sector; + aligned.min(room) +} + /// Pull unnamed `$DATA` fragments out of each extension record. /// A record the current map cannot reach stays pending until a fragment /// just loaded covers it. Records that never become readable, or that @@ -639,6 +656,28 @@ mod tests { ); } + #[test] + fn attribute_list_read_rounds_up_to_a_whole_sector_and_stops_at_the_runs() { + assert_eq!(sector_aligned_read_len(100, 512, 4096), 512); + assert_eq!(sector_aligned_read_len(512, 512, 4096), 512); + assert_eq!(sector_aligned_read_len(513, 512, 4096), 1024); + assert_eq!(sector_aligned_read_len(5_000, 512, 4096), 4096); + assert_eq!(sector_aligned_read_len(0, 512, 4096), 0); + // Shorter than one sector: refuse an unaligned transfer. + assert_eq!(sector_aligned_read_len(100, 512, 200), 0); + } + + #[test] + fn stale_bytes_past_the_initialized_list_are_not_entries() { + let entry = list_entry(ATTR_DATA, 16, 0); + let mut padded = entry.clone(); + padded.extend(list_entry(ATTR_DATA, 99, 0)); + assert_eq!( + extension_records_from_attribute_list(&padded[..entry.len()]), + vec![16] + ); + } + #[test] fn relative_run_offsets_accumulate() { // Two runs: 3 clusters at LCN 10, then 2 clusters at delta -4 → LCN 6. From 6643d01505b1f0ad46090bf2ec42f59d8c9b149b Mon Sep 17 00:00:00 2001 From: Thomas Zarebczan Date: Wed, 14 Oct 2026 18:48:30 -0400 Subject: [PATCH 3/4] Read a non-resident $MFT attribute list in full A 1 MiB cap dropped every extension record past the prefix and the scan still returned that partial map. Size the read from the list's initialized length and its own runs. Above 32 MiB, keep record 0 and warn instead of parsing a prefix. --- native/diskhound-native-scanner/src/mft.rs | 59 ++++++++++++------- .../diskhound-native-scanner/src/mft_runs.rs | 51 ++++++++++++++++ 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/native/diskhound-native-scanner/src/mft.rs b/native/diskhound-native-scanner/src/mft.rs index 64823b42..6c467e20 100644 --- a/native/diskhound-native-scanner/src/mft.rs +++ b/native/diskhound-native-scanner/src/mft.rs @@ -488,8 +488,9 @@ fn resolve_mft_runs( valid_data_length: i64, ) -> Result, MftError> { use crate::mft_runs::{ - absorb_extension_records, extension_records_from_attribute_list, join_run_segments, - logical_byte_length, read_logical, record_0_mft_map, sector_aligned_read_len, + AttributeListRead, absorb_extension_records, attribute_list_read, + extension_records_from_attribute_list, join_run_segments, logical_byte_length, + read_logical, record_0_mft_map, }; let invalid = || { @@ -503,33 +504,47 @@ fn resolve_mft_runs( let mut pending = map.extension_records; if let Some(external) = map.external_attribute_list { let available = logical_byte_length(&external.runs, bytes_per_cluster).unwrap_or(0); - // Initialized size is rarely a whole sector. `\\.\X:` rejects that - // read, which used to fail the scan instead of keeping record 0. - let want = external.byte_length.min(1_u64 << 20); - let read_len = - sector_aligned_read_len(want, bytes_per_sector as u64, available) as usize; - let mut list = vec![0u8; read_len]; - let read = if read_len == 0 { - Ok(false) - } else { - read_logical(volume, &external.runs, 0, &mut list, bytes_per_cluster) - }; - match read { - Ok(true) => { - let prefix = (external.byte_length as usize).min(list.len()); - pending = extension_records_from_attribute_list(&list[..prefix]); - } - Ok(false) => { + // Whole list only. A 1 MiB prefix used to drop every extension past + // the cut and the scan still looked successful. 32 MiB is far past + // a real $MFT attribute list; above that, keep record 0 and warn. + const MAX_ATTRIBUTE_LIST_BYTES: u64 = 32 * 1024 * 1024; + match attribute_list_read( + external.byte_length, + bytes_per_sector as u64, + available, + MAX_ATTRIBUTE_LIST_BYTES, + ) { + AttributeListRead::Missing => { eprintln!( "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT could not be read; using record 0's run list only" ); } - Err(err) => { + AttributeListRead::TooBig(bytes) => { eprintln!( - "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT could not be read ({}); using record 0's run list only", - err + "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT is {:.1} MB; using record 0's run list only", + bytes as f64 / (1024.0 * 1024.0) ); } + AttributeListRead::Ready(read_len) => { + let mut list = vec![0u8; read_len as usize]; + match read_logical(volume, &external.runs, 0, &mut list, bytes_per_cluster) { + Ok(true) => { + let prefix = (external.byte_length as usize).min(list.len()); + pending = extension_records_from_attribute_list(&list[..prefix]); + } + Ok(false) => { + eprintln!( + "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT could not be read; using record 0's run list only" + ); + } + Err(err) => { + eprintln!( + "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT could not be read ({}); using record 0's run list only", + err + ); + } + } + } } } diff --git a/native/diskhound-native-scanner/src/mft_runs.rs b/native/diskhound-native-scanner/src/mft_runs.rs index 53e5df1c..499eb716 100644 --- a/native/diskhound-native-scanner/src/mft_runs.rs +++ b/native/diskhound-native-scanner/src/mft_runs.rs @@ -209,6 +209,34 @@ pub fn sector_aligned_read_len(logical_len: u64, bytes_per_sector: u64, availabl aligned.min(room) } +/// A non-resident `$ATTRIBUTE_LIST` is read whole, or not at all. +/// A prefix would drop every extension record past the cut. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AttributeListRead { + /// Sector-aligned byte count to read. Parse only the initialized prefix. + Ready(u64), + /// The list is on disk and larger than `max_bytes`. + TooBig(u64), + /// No sector-aligned bytes are available. + Missing, +} + +pub fn attribute_list_read( + byte_length: u64, + bytes_per_sector: u64, + available: u64, + max_bytes: u64, +) -> AttributeListRead { + let read_len = sector_aligned_read_len(byte_length, bytes_per_sector, available); + if read_len == 0 { + AttributeListRead::Missing + } else if read_len > max_bytes { + AttributeListRead::TooBig(read_len) + } else { + AttributeListRead::Ready(read_len) + } +} + /// Pull unnamed `$DATA` fragments out of each extension record. /// A record the current map cannot reach stays pending until a fragment /// just loaded covers it. Records that never become readable, or that @@ -656,6 +684,29 @@ mod tests { ); } + #[test] + fn a_long_attribute_list_is_read_whole_or_refused() { + let sector = 512u64; + let available = 3 * 1024 * 1024; + let two_mb = 2 * 1024 * 1024; + assert_eq!( + attribute_list_read(two_mb, sector, available, 32 * 1024 * 1024), + AttributeListRead::Ready(two_mb) + ); + assert_eq!( + attribute_list_read(two_mb, sector, available, 1024 * 1024), + AttributeListRead::TooBig(two_mb) + ); + assert_eq!( + attribute_list_read(100, sector, 4096, 1024 * 1024), + AttributeListRead::Ready(512) + ); + assert_eq!( + attribute_list_read(0, sector, available, 1024 * 1024), + AttributeListRead::Missing + ); + } + #[test] fn attribute_list_read_rounds_up_to_a_whole_sector_and_stops_at_the_runs() { assert_eq!(sector_aligned_read_len(100, 512, 4096), 512); From 4049bdc3a8685603bdd5fa2c4f0d6c7f0cc1afe9 Mon Sep 17 00:00:00 2001 From: Thomas Zarebczan Date: Wed, 14 Oct 2026 19:01:20 -0400 Subject: [PATCH 4/4] Refuse a short $MFT attribute list and refresh extension runs as they load A non-resident list whose runs stop short of its initialized length was parsed as a prefix, so later $DATA extension records disappeared. Keep record 0 and warn instead. Rebuild the run map before each extension read so an in-order chain is loaded in one pass. --- native/diskhound-native-scanner/src/mft.rs | 12 ++- .../diskhound-native-scanner/src/mft_runs.rs | 76 +++++++++++++++++-- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/native/diskhound-native-scanner/src/mft.rs b/native/diskhound-native-scanner/src/mft.rs index 6c467e20..de8052c5 100644 --- a/native/diskhound-native-scanner/src/mft.rs +++ b/native/diskhound-native-scanner/src/mft.rs @@ -504,9 +504,10 @@ fn resolve_mft_runs( let mut pending = map.extension_records; if let Some(external) = map.external_attribute_list { let available = logical_byte_length(&external.runs, bytes_per_cluster).unwrap_or(0); - // Whole list only. A 1 MiB prefix used to drop every extension past - // the cut and the scan still looked successful. 32 MiB is far past - // a real $MFT attribute list; above that, keep record 0 and warn. + // Whole list only. A prefix drops every extension past the cut + // and the scan still looks successful. 32 MiB is far past a real + // $MFT attribute list. Above that, or when record 0's runs do not + // cover the initialized length, keep record 0 and warn. const MAX_ATTRIBUTE_LIST_BYTES: u64 = 32 * 1024 * 1024; match attribute_list_read( external.byte_length, @@ -519,6 +520,11 @@ fn resolve_mft_runs( "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT could not be read; using record 0's run list only" ); } + AttributeListRead::Incomplete => { + eprintln!( + "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT is only partly in record 0; using record 0's run list only" + ); + } AttributeListRead::TooBig(bytes) => { eprintln!( "[diskhound-native-scanner] mft: warning: $ATTRIBUTE_LIST for $MFT is {:.1} MB; using record 0's run list only", diff --git a/native/diskhound-native-scanner/src/mft_runs.rs b/native/diskhound-native-scanner/src/mft_runs.rs index 499eb716..65cd3445 100644 --- a/native/diskhound-native-scanner/src/mft_runs.rs +++ b/native/diskhound-native-scanner/src/mft_runs.rs @@ -213,10 +213,14 @@ pub fn sector_aligned_read_len(logical_len: u64, bytes_per_sector: u64, availabl /// A prefix would drop every extension record past the cut. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AttributeListRead { - /// Sector-aligned byte count to read. Parse only the initialized prefix. + /// Sector-aligned byte count covering `byte_length`. Parse only + /// the initialized prefix; the tail is sector padding. Ready(u64), /// The list is on disk and larger than `max_bytes`. TooBig(u64), + /// Record 0's runs cover only a prefix of the initialized length. + /// Parsing that prefix would drop the rest of the extension records. + Incomplete, /// No sector-aligned bytes are available. Missing, } @@ -230,6 +234,8 @@ pub fn attribute_list_read( let read_len = sector_aligned_read_len(byte_length, bytes_per_sector, available); if read_len == 0 { AttributeListRead::Missing + } else if read_len < byte_length { + AttributeListRead::Incomplete } else if read_len > max_bytes { AttributeListRead::TooBig(read_len) } else { @@ -238,9 +244,10 @@ pub fn attribute_list_read( } /// Pull unnamed `$DATA` fragments out of each extension record. -/// A record the current map cannot reach stays pending until a fragment -/// just loaded covers it. Records that never become readable, or that -/// are not FILE records, are returned unread. +/// The run map is rebuilt before each attempt, so a record loaded +/// earlier in this pass can cover the next one. A record the map still +/// cannot reach stays pending. Records that never become readable, or +/// that are not FILE records, are returned unread. pub fn absorb_extension_records( segments: &mut Vec, mut pending: Vec, @@ -249,10 +256,10 @@ pub fn absorb_extension_records( ) -> io::Result> { let mut unresolved = Vec::new(); loop { - let (runs_now, _) = join_run_segments(segments); let mut still_out_of_range = Vec::new(); let mut progress = false; for frn in pending { + let (runs_now, _) = join_run_segments(segments); let Some(logical) = frn.checked_mul(record_size as u64) else { unresolved.push(frn); continue; @@ -701,12 +708,71 @@ mod tests { attribute_list_read(100, sector, 4096, 1024 * 1024), AttributeListRead::Ready(512) ); + // Runs cover 4096 of an initialized 5000. Parsing that prefix + // would drop every extension record past the cut. + assert_eq!( + attribute_list_read(5_000, sector, 4096, 32 * 1024 * 1024), + AttributeListRead::Incomplete + ); assert_eq!( attribute_list_read(0, sector, available, 1024 * 1024), AttributeListRead::Missing ); } + #[test] + fn an_in_order_extension_chain_is_read_once_per_record() { + let bpc = 1024u64; + let record_size = 1024usize; + // Base covers records 0 and 1. Record 1 names the extent that + // holds record 2, and record 2 names the extent that holds record 3. + let mut image = vec![0u8; 21 * record_size]; + let rec1 = file_record(&nonresident(ATTR_DATA, 2, 1, 10, 0)); + image[record_size..record_size + rec1.len()].copy_from_slice(&rec1); + let rec2 = file_record(&nonresident(ATTR_DATA, 3, 1, 20, 0)); + image[10 * record_size..10 * record_size + rec2.len()].copy_from_slice(&rec2); + let rec3 = file_record(&nonresident(ATTR_DATA, 4, 1, 30, 0)); + image[20 * record_size..20 * record_size + rec3.len()].copy_from_slice(&rec3); + + let mut segments = unnamed_data_segments(&file_record(&nonresident(ATTR_DATA, 0, 2, 0, 0))); + let mut reads = Vec::new(); + let unresolved = absorb_extension_records( + &mut segments, + vec![1, 2, 3], + record_size, + |runs, logical, buf| { + reads.push(logical / record_size as u64); + read_logical(&mut Cursor::new(image.clone()), runs, logical, buf, bpc) + }, + ) + .unwrap(); + assert!(unresolved.is_empty()); + assert_eq!(reads, vec![1, 2, 3]); + let (runs, gap) = join_run_segments(&segments); + assert!(!gap); + assert_eq!( + runs, + vec![ + DataRun { + lcn: 0, + cluster_count: 2 + }, + DataRun { + lcn: 10, + cluster_count: 1 + }, + DataRun { + lcn: 20, + cluster_count: 1 + }, + DataRun { + lcn: 30, + cluster_count: 1 + }, + ] + ); + } + #[test] fn attribute_list_read_rounds_up_to_a_whole_sector_and_stops_at_the_runs() { assert_eq!(sector_aligned_read_len(100, 512, 4096), 512);