From b36bad6452c713f11cddb63e1aa624f86caaa566 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Sat, 26 Sep 2026 20:38:56 -0400 Subject: [PATCH] perf: batch macOS metadata during directory enumeration --- Cargo.lock | 1 + crates/disktree-core/Cargo.toml | 1 + crates/disktree-core/src/macos.rs | 950 ++++++++----------- crates/disktree-core/src/macos/tests.rs | 285 ++++++ crates/disktree-core/src/scan.rs | 149 +-- crates/disktree-core/src/scan/macos_tests.rs | 205 ++++ 6 files changed, 993 insertions(+), 598 deletions(-) create mode 100644 crates/disktree-core/src/macos/tests.rs create mode 100644 crates/disktree-core/src/scan/macos_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 71482b53..a05fb428 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1437,6 +1437,7 @@ name = "disktree-core" version = "0.11.0" dependencies = [ "chrono", + "libc", "rayon", "rustc-hash 2.1.3", "rustix", diff --git a/crates/disktree-core/Cargo.toml b/crates/disktree-core/Cargo.toml index 8f121ebf..12ba2740 100644 --- a/crates/disktree-core/Cargo.toml +++ b/crates/disktree-core/Cargo.toml @@ -34,6 +34,7 @@ windows-sys = { version = "0.61", features = [ # without asking to control Finder. Only this backend is used; the crate's # Linux and Windows code is not built. [target.'cfg(target_os = "macos")'.dependencies] +libc = "0.2" trash = { version = "5.2.9", default-features = false } [dev-dependencies] diff --git a/crates/disktree-core/src/macos.rs b/crates/disktree-core/src/macos.rs index 6cdac06d..12fb605f 100644 --- a/crates/disktree-core/src/macos.rs +++ b/crates/disktree-core/src/macos.rs @@ -1,623 +1,509 @@ -//! What a walk on macOS needs beyond the standard library. +//! Directory metadata in batches, through Darwin's `getattrlistbulk(2)`. //! -//! `std::fs::read_dir` lists a directory with `readdir`, whose records carry -//! a name and a type, and `DirEntry::metadata` is then an `lstat` of each -//! entry's full path. `getattrlistbulk(2)` returns every entry's size, -//! identity, age and flags for a whole buffer of entries, without the kernel -//! setting up per-file state for each one as a stat must. +//! Unsafe code is limited to the system call and its aligned buffer view. +//! Packed records use checked slices; missing attributes use no-follow stat. #![allow( unsafe_code, - reason = "getattrlistbulk, which neither std nor rustix wraps; each block \ - says why it is sound" + reason = "Darwin's bulk directory call has no std wrapper; the call documents its safety" )] -use std::cell::RefCell; -use std::ffi::{OsStr, c_int, c_void}; +use std::cell::OnceCell; +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions}; use std::io; -use std::os::fd::{AsRawFd as _, OwnedFd}; -use std::os::unix::ffi::OsStrExt as _; +use std::os::fd::AsRawFd as _; +use std::os::unix::ffi::OsStringExt as _; +use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; use std::path::{Path, PathBuf}; +use std::sync::Arc; -use rustix::fs::{AtFlags, FileType, Mode, OFlags}; - -/// Bytes the kernel fills per call: some hundreds of entries. +// A batch holds hundreds of records without retaining a whole directory. const BUFFER_BYTES: usize = 64 * 1024; - -// From and , which the libc crate does not cover. -const ATTR_BIT_MAP_COUNT: u16 = 5; -const ATTR_CMN_NAME: u32 = 0x0000_0001; -const ATTR_CMN_DEVID: u32 = 0x0000_0002; -const ATTR_CMN_OBJTYPE: u32 = 0x0000_0008; -const ATTR_CMN_MODTIME: u32 = 0x0000_0400; -const ATTR_CMN_FLAGS: u32 = 0x0004_0000; -const ATTR_CMN_FILEID: u32 = 0x0200_0000; +// Darwin sys/attr.h and sys/stat.h constants absent from libc. const ATTR_CMN_ERROR: u32 = 0x2000_0000; -const ATTR_CMN_RETURNED_ATTRS: u32 = 0x8000_0000; -const ATTR_FILE_LINKCOUNT: u32 = 0x0000_0001; -/// Every fork, which is what `st_blocks` counts: a resource fork is on the -/// disk too. -const ATTR_FILE_ALLOCSIZE: u32 = 0x0000_0004; -/// The data fork alone, which is what `st_size` is. -const ATTR_FILE_DATALENGTH: u32 = 0x0000_0200; -const VREG: u32 = 1; -const VDIR: u32 = 2; -const VLNK: u32 = 5; -/// `SF_DATALESS` from . -const SF_DATALESS: u32 = 0x4000_0000; - -const COMMON_ATTRIBUTES: u32 = ATTR_CMN_RETURNED_ATTRS - | ATTR_CMN_NAME - | ATTR_CMN_DEVID - | ATTR_CMN_OBJTYPE - | ATTR_CMN_MODTIME - | ATTR_CMN_FLAGS - | ATTR_CMN_FILEID - | ATTR_CMN_ERROR; -const FILE_ATTRIBUTES: u32 = - ATTR_FILE_LINKCOUNT | ATTR_FILE_ALLOCSIZE | ATTR_FILE_DATALENGTH; -/// What [`Cursor::entry`] reads after the name; a file system may leave -/// any of them out. -const MEASURED: u32 = ATTR_CMN_DEVID - | ATTR_CMN_OBJTYPE - | ATTR_CMN_MODTIME - | ATTR_CMN_FLAGS - | ATTR_CMN_FILEID; +const SF_FIRMLINK: u32 = 0x0080_0000; +const COMMON: u32 = libc::ATTR_CMN_RETURNED_ATTRS + | ATTR_CMN_ERROR + | libc::ATTR_CMN_NAME + | libc::ATTR_CMN_DEVID + | libc::ATTR_CMN_OBJTYPE + | libc::ATTR_CMN_MODTIME + | libc::ATTR_CMN_FLAGS + | libc::ATTR_CMN_FILEID; +const FILE: u32 = libc::ATTR_FILE_LINKCOUNT + | libc::ATTR_FILE_ALLOCSIZE + | libc::ATTR_FILE_DATALENGTH; +// Length followed by the five returned attribute bitmaps. +const HEADER_BYTES: usize = 24; -#[repr(C)] -struct AttrList { - bitmapcount: u16, - reserved: u16, - commonattr: u32, - volattr: u32, - dirattr: u32, - fileattr: u32, - forkattr: u32, +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + File, + Directory, + Symlink, + Other, } -unsafe extern "C" { - fn getattrlistbulk( - dirfd: c_int, - attr_list: *mut AttrList, - attr_buf: *mut c_void, - attr_buf_size: usize, - options: u64, - ) -> c_int; +impl Kind { + fn of(kind: fs::FileType) -> Self { + if kind.is_dir() { + Self::Directory + } else if kind.is_file() { + Self::File + } else if kind.is_symlink() { + Self::Symlink + } else { + Self::Other + } + } } -thread_local! { - /// The last finished listing's buffer, for the next one on this thread: - /// a walk lists millions of directories, one at a time per thread. - static SPARE: RefCell>> = const { RefCell::new(None) }; +#[derive(Debug)] +pub struct Metadata { + pub apparent: u64, + pub allocated: u64, + pub device: u64, + pub inode: u64, + pub modified: i64, + pub links: u64, } -/// What an entry is. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Kind { - Directory, - Link, - File, - /// A FIFO, a socket or a device node. - Other, +impl Metadata { + fn of(meta: &fs::Metadata) -> Self { + Self { + apparent: meta.len(), + allocated: meta.blocks().saturating_mul(512), + device: meta.dev(), + inode: meta.ino(), + modified: meta + .modified() + .ok() + .and_then(|time| { + time.duration_since(std::time::UNIX_EPOCH).ok() + }) + .map_or(0, |since| { + i64::try_from(since.as_secs()).unwrap_or(i64::MAX) + }), + links: meta.nlink(), + } + } } -/// One directory entry, with what a measurement needs. #[derive(Debug)] pub struct Entry { - name: Box, - /// The name as the file system spells it, only when `name` could not: - /// a name that is not UTF-8 is shown lossy but must still open. - exact: Option>, - kind: Kind, - apparent: u64, - allocated: u64, - modified: i64, - device: u64, - inode: u64, - links: u32, - flags: u32, + pub file_name: OsString, + pub kind: Kind, + pub metadata: OnceCell>, + parent: Arc, } impl Entry { - /// The entry's path inside `dir`, the directory it was listed from. - pub fn path(&self, dir: &Path) -> PathBuf { - dir.join(self.file_name()) - } - - pub fn file_name(&self) -> &OsStr { - self.exact - .as_deref() - .unwrap_or_else(|| OsStr::new(&*self.name)) - } - - /// The name as text; see [`Self::file_name`] for what it may lose. - pub fn name(&self) -> &str { - &self.name - } - - /// The name, moved out: the tree keeps it, so nothing copies it. The - /// entry's other accessors keep working; `name`, `file_name` and - /// `path` do not. - pub fn take_name(&mut self) -> Box { - std::mem::take(&mut self.name) - } - - pub const fn kind(&self) -> Kind { - self.kind - } - - /// `st_size`. - pub const fn apparent(&self) -> u64 { - self.apparent - } - - /// `st_blocks * 512`. - pub const fn allocated(&self) -> u64 { - self.allocated - } - - /// Last write, in Unix seconds. - pub const fn modified(&self) -> i64 { - self.modified - } - - /// `(st_dev, st_ino)`: the same for two names of one file. - pub const fn identity(&self) -> (u64, u64) { - (self.device, self.inode) - } - - pub const fn device(&self) -> u64 { - self.device - } - - /// Whether the file has more than one name. - pub const fn shared(&self) -> bool { - self.links > 1 + pub fn path(&self) -> PathBuf { + self.parent.join(&self.file_name) } - /// Whether this is a directory iCloud Drive or a File Provider has - /// evicted, which listing would make it fetch. - pub const fn evicted(&self) -> bool { - matches!(self.kind, Kind::Directory) && self.flags & SF_DATALESS != 0 + pub fn metadata(&self) -> Result<&Metadata, &io::Error> { + self.metadata + .get_or_init(|| { + fs::symlink_metadata(self.path()) + .map(|meta| Metadata::of(&meta)) + }) + .as_ref() } +} - fn named(name: &[u8]) -> (Box, Option>) { - match std::str::from_utf8(name) { - Ok(text) => (text.into(), None), - Err(_) => ( - String::from_utf8_lossy(name).into(), - Some(OsStr::from_bytes(name).into()), - ), - } - } +// Darwin requires each record to start at an eight-byte-aligned address. +#[derive(Debug)] +struct Buffer(Box<[u64]>); - fn from_stat(name: &[u8], stat: &rustix::fs::Stat) -> Self { - let (name, exact) = Self::named(name); - let kind = match FileType::from_raw_mode(stat.st_mode) { - FileType::Directory => Kind::Directory, - FileType::Symlink => Kind::Link, - FileType::RegularFile => Kind::File, - _ => Kind::Other, - }; - Self { - name, - exact, - kind, - apparent: stat.st_size.max(0) as u64, - allocated: (stat.st_blocks.max(0) as u64).saturating_mul(512), - modified: stat.st_mtime, - device: i64::from(stat.st_dev).cast_unsigned(), - inode: stat.st_ino, - links: u32::from(stat.st_nlink), - flags: stat.st_flags, +impl Buffer { + fn bytes(&mut self) -> &mut [u8] { + // SAFETY: all words are initialized, every bit pattern is valid for + // u64, and u8 needs no alignment. This exclusive byte view covers + // exactly the allocation and cannot outlive its mutable borrow. + unsafe { + std::slice::from_raw_parts_mut( + self.0.as_mut_ptr().cast(), + size_of_val(&*self.0), + ) } } } -/// A directory being listed; like `std::fs::ReadDir`, it leaves out `.` -/// and `..`. #[derive(Debug)] pub struct ReadDir { - fd: OwnedFd, - /// `u64`s for alignment; the records themselves are 4-byte aligned. - buffer: Option>, - /// Records in the buffer not yet handed out. - remaining: u32, - /// Where the next of them starts. + directory: File, + parent: Arc, + buffer: Buffer, offset: usize, + remaining: usize, + started: bool, done: bool, + fallback: Option, } -/// List `dir`, with each entry's size, identity and age. A link at `dir` -/// itself is followed, as `std::fs::read_dir` follows it. -pub fn read_dir(dir: &Path) -> io::Result { - let fd = rustix::fs::open( - dir, - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, - Mode::empty(), - )?; - let buffer = SPARE - .with_borrow_mut(Option::take) - .unwrap_or_else(|| vec![0; BUFFER_BYTES / 8].into_boxed_slice()); +pub fn read_dir(path: &Path) -> io::Result { Ok(ReadDir { - fd, - buffer: Some(buffer), - remaining: 0, + directory: OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY) + .open(path)?, + parent: Arc::from(path), + buffer: Buffer( + vec![0; BUFFER_BYTES / size_of::()].into_boxed_slice(), + ), offset: 0, + remaining: 0, + started: false, done: false, + fallback: None, }) } -impl Drop for ReadDir { - fn drop(&mut self) { - if let Some(buffer) = self.buffer.take() { - SPARE.with_borrow_mut(|spare| *spare = Some(buffer)); - } - } -} - -impl Iterator for ReadDir { - type Item = io::Result; - - fn next(&mut self) -> Option { - if self.remaining == 0 { - if self.done { - return None; - } - match self.fill() { - Ok(0) => { - self.done = true; - return None; - } - Ok(count) => { - self.remaining = count; - self.offset = 0; - } - Err(error) => { - self.done = true; - return Some(Err(error)); - } - } - } - self.remaining -= 1; - match self.record() { - Ok((entry, length)) => { - self.offset += length; - Some(entry) - } - Err(error) => { - self.remaining = 0; - self.done = true; - Some(Err(error)) - } - } - } -} - impl ReadDir { - /// Ask the kernel for the next buffer of records; how many it wrote. - fn fill(&mut self) -> io::Result { - let buffer = self.buffer.as_mut().expect("held until drop"); - let mut request = AttrList { - bitmapcount: ATTR_BIT_MAP_COUNT, - reserved: 0, - commonattr: COMMON_ATTRIBUTES, - volattr: 0, - dirattr: 0, - fileattr: FILE_ATTRIBUTES, - forkattr: 0, - }; + fn refill(&mut self) -> io::Result<()> { loop { - // SAFETY: the descriptor is an open directory owned by `self`; - // `request` is a valid attrlist for the call's duration; and the - // pointer and length describe `buffer`, which is writable for - // that many bytes and not otherwise borrowed during the call. + let mut attributes = libc::attrlist { + bitmapcount: libc::ATTR_BIT_MAP_COUNT, + reserved: 0, + commonattr: COMMON, + volattr: 0, + dirattr: 0, + fileattr: FILE, + forkattr: 0, + }; + // SAFETY: the File owns an open directory descriptor; attrlist + // is initialized; the aligned buffer is exclusively writable + // for exactly BUFFER_BYTES. No pointers escape this call. let count = unsafe { - getattrlistbulk( - self.fd.as_raw_fd(), - &raw mut request, - buffer.as_mut_ptr().cast(), + libc::getattrlistbulk( + self.directory.as_raw_fd(), + (&raw mut attributes).cast(), + self.buffer.bytes().as_mut_ptr().cast(), BUFFER_BYTES, 0, ) }; - if let Ok(count) = u32::try_from(count) { - return Ok(count); + if count >= 0 { + self.remaining = usize::try_from(count) + .map_err(|_| invalid("invalid directory record count"))?; + self.offset = 0; + self.done = count == 0; + self.started |= count > 0; + return Ok(()); } let error = io::Error::last_os_error(); - if error.kind() != io::ErrorKind::Interrupted { - return Err(error); + if error.kind() == io::ErrorKind::Interrupted { + continue; } + return self.refill_error(error); } } - /// The buffer as the bytes the kernel wrote. - fn bytes(&self) -> &[u8] { - let buffer = self.buffer.as_deref().expect("held until drop"); - // SAFETY: `[u64]` can be read as `u8`s of eight times its length: - // one allocation, no alignment requirement for `u8`, and every byte - // initialized, since the buffer starts zeroed. - unsafe { - std::slice::from_raw_parts( - buffer.as_ptr().cast::(), - size_of_val(buffer), + fn refill_error(&mut self, error: io::Error) -> io::Result<()> { + // A readable directory may deny metadata/search access. Let std + // enumerate its names and surface per-entry errors. Unsupported + // filesystems also retain the ordinary walk. Never restart after + // emitting entries: doing so could duplicate part of the tree. + if !self.started + && matches!( + error.raw_os_error(), + Some(libc::EACCES | libc::ENOTSUP | libc::ENOSYS) ) + { + self.fallback = Some(fs::read_dir(&self.parent)?); + Ok(()) + } else { + Err(error) } } - /// The entry at `self.offset` and the record's length. Every length and - /// offset the kernel wrote is checked against the buffer before it is - /// used. - /// - /// Layout per getattrlist(2): a `u32` length, the returned - /// `attribute_set_t`, `ATTR_CMN_ERROR` when it is set, then every other - /// attribute in bit order, each packed at 4-byte alignment. - fn record(&self) -> io::Result<(io::Result, usize)> { - let bytes = self.bytes(); - let length = read_u32(bytes, self.offset)? as usize; - let record = bytes - .get(self.offset..self.offset + length) - .ok_or_else(malformed)?; - let mut cursor = Cursor { record, at: 4 }; - let common = cursor.u32()?; - cursor.at += 8; - let file = cursor.u32()?; - cursor.at += 4; - let error = if common & ATTR_CMN_ERROR == 0 { - 0 - } else { - cursor.u32()? - }; - let name = cursor.name()?; - if error != 0 { - let error = io::Error::from_raw_os_error(error.cast_signed()); - return Ok((Err(error), length)); + fn next_record(&mut self) -> io::Result { + let remaining = + self.buffer.bytes().get(self.offset..).ok_or_else(|| { + invalid("directory record offset exceeds buffer") + })?; + let length = u32::from_ne_bytes(Cursor::new(remaining).take()?); + let length = usize::try_from(length).map_err(|_| { + invalid("directory record length exceeds address space") + })?; + if length < HEADER_BYTES || length % 8 != 0 { + return Err(invalid("invalid directory record length")); } - let entry = if common & MEASURED == MEASURED { - cursor.entry(file, name)? - } else { - None - }; - Ok((entry.map_or_else(|| self.stat(name), Ok), length)) + let bytes = remaining + .get(..length) + .ok_or_else(|| invalid("directory record exceeds buffer"))?; + let record = Record::parse(bytes)?; + self.offset += length; + self.remaining -= 1; + Ok(record.entry(Arc::clone(&self.parent))) } - /// An entry the file system described only in part, read the ordinary - /// way, relative to the directory already open. - fn stat(&self, name: &[u8]) -> io::Result { - let stat = - rustix::fs::statat(&self.fd, name, AtFlags::SYMLINK_NOFOLLOW)?; - Ok(Entry::from_stat(name, &stat)) + fn standard_entry(&self, entry: &fs::DirEntry) -> io::Result { + let kind = Kind::of(entry.file_type()?); + let metadata = if matches!(kind, Kind::Directory | Kind::Symlink) { + OnceCell::new() + } else { + OnceCell::from(entry.metadata().map(|meta| Metadata::of(&meta))) + }; + Ok(Entry { + file_name: entry.file_name(), + kind, + metadata, + parent: Arc::clone(&self.parent), + }) } } -fn malformed() -> io::Error { - io::Error::other("the kernel returned a malformed record") -} +impl Iterator for ReadDir { + type Item = io::Result; -fn read_u32(bytes: &[u8], at: usize) -> io::Result { - let field = at - .checked_add(4) - .and_then(|end| bytes.get(at..end)) - .ok_or_else(malformed)?; - Ok(u32::from_ne_bytes( - field.try_into().map_err(|_| malformed())?, - )) + fn next(&mut self) -> Option { + loop { + if self.done { + return None; + } + if let Some(fallback) = &mut self.fallback { + return fallback.next().map(|entry| { + entry.and_then(|entry| self.standard_entry(&entry)) + }); + } + if self.remaining == 0 { + if let Err(error) = self.refill() { + self.done = true; + return Some(Err(error)); + } + continue; + } + match self.next_record() { + Ok(entry) + if entry.file_name == "." || entry.file_name == ".." => {} + Ok(entry) => return Some(Ok(entry)), + Err(error) => { + self.done = true; + return Some(Err(error)); + } + } + } + } } -/// Reads a record's attributes in order; reading past its end is a -/// malformed record. -struct Cursor<'a> { - record: &'a [u8], - at: usize, +#[derive(Debug)] +struct Record { + name: OsString, + kind: Kind, + error: i32, + flags: Option, + metadata: Option, } -impl<'a> Cursor<'a> { - fn u32(&mut self) -> io::Result { - let value = read_u32(self.record, self.at)?; - self.at += 4; - Ok(value) - } - - fn u64(&mut self) -> io::Result { - let field = self - .at - .checked_add(8) - .and_then(|end| self.record.get(self.at..end)) - .ok_or_else(malformed)?; - self.at += 8; - Ok(u64::from_ne_bytes( - field.try_into().map_err(|_| malformed())?, - )) - } - - /// An `attrreference_t`: an offset from itself and a length that counts - /// the terminating NUL. - fn name(&mut self) -> io::Result<&'a [u8]> { - let reference = self.at; - let offset = isize::try_from(self.u32()?.cast_signed()) - .map_err(|_| malformed())?; - let length = self.u32()? as usize; - reference - .checked_add_signed(offset) - .zip(length.checked_sub(1)) - .and_then(|(start, length)| { - self.record.get(start..start.checked_add(length)?) - }) - .ok_or_else(malformed) - } - - /// The attributes after the name, every one of them asked for; `None` - /// for a file whose sizes the file system left out. - fn entry(&mut self, file: u32, name: &[u8]) -> io::Result> { - let device = self.u32()?.cast_signed(); - let kind = match self.u32()? { - VDIR => Kind::Directory, - VLNK => Kind::Link, - VREG => Kind::File, +impl Record { + fn parse(bytes: &[u8]) -> io::Result { + let mut cursor = Cursor::new(bytes); + cursor.take::<4>()?; + let common = u32::from_ne_bytes(cursor.take()?); + let volume = u32::from_ne_bytes(cursor.take()?); + let directory = u32::from_ne_bytes(cursor.take()?); + let file = u32::from_ne_bytes(cursor.take()?); + let fork = u32::from_ne_bytes(cursor.take()?); + if common & libc::ATTR_CMN_RETURNED_ATTRS == 0 + || common & !COMMON != 0 + || file & !FILE != 0 + || volume != 0 + || directory != 0 + || fork != 0 + { + return Err(invalid("unexpected returned directory attributes")); + } + // ATTR_CMN_ERROR precedes all other common fields, regardless of + // its bit position. Without PACK_INVAL_ATTRS, missing fields occupy + // no bytes; every read follows the returned bitmap. + let error = cursor + .attribute(common, ATTR_CMN_ERROR)? + .map_or(0, i32::from_ne_bytes); + let name = cursor.name(common)?; + let device = cursor + .attribute(common, libc::ATTR_CMN_DEVID)? + .map(|raw| i64::from(i32::from_ne_bytes(raw)).cast_unsigned()); + let kind = match cursor + .attribute(common, libc::ATTR_CMN_OBJTYPE)? + .map(u32::from_ne_bytes) + { + // Darwin's enum vtype, from sys/vnode.h. + Some(1) => Kind::File, + Some(2) => Kind::Directory, + Some(5) => Kind::Symlink, _ => Kind::Other, }; - let modified = self.u64()?.cast_signed(); - self.at += 8; - let flags = self.u32()?; - let inode = self.u64()?; - // A directory is stated the ordinary way. A mount point is listed - // with the attributes of the directory it covers, so its DEVID is - // the parent volume's, and the walk would cross into every disk, - // share and system volume mounted below the root. `statat` reads - // the mounted root, as `lstat` did; directories are few beside files. - let (links, allocated, apparent) = if kind == Kind::Directory { - return Ok(None); - } else if file & FILE_ATTRIBUTES == FILE_ATTRIBUTES { - (self.u32()?, self.u64()?, self.u64()?) - } else { - return Ok(None); - }; - let (name, exact) = Entry::named(name); - Ok(Some(Entry { + let modified = cursor + .attribute::<16>(common, libc::ATTR_CMN_MODTIME)? + .map(modified_seconds) + .transpose()?; + let flags = cursor + .attribute(common, libc::ATTR_CMN_FLAGS)? + .map(u32::from_ne_bytes); + let inode = cursor + .attribute(common, libc::ATTR_CMN_FILEID)? + .map(u64::from_ne_bytes); + let links = cursor + .attribute(file, libc::ATTR_FILE_LINKCOUNT)? + .map(|raw| u64::from(u32::from_ne_bytes(raw))); + let allocated = cursor + .attribute(file, libc::ATTR_FILE_ALLOCSIZE)? + .map(u64::from_ne_bytes); + let apparent = cursor + .attribute(file, libc::ATTR_FILE_DATALENGTH)? + .map(u64::from_ne_bytes); + let metadata = + match (device, inode, modified, links, allocated, apparent) { + ( + Some(device), + Some(inode), + Some(modified), + Some(links), + Some(allocated), + Some(apparent), + ) => { + Some(Metadata { + device, + inode, + modified, + links, + // Total allocation includes resource forks. Match the + // 512-byte stat accounting unit, rounding up as FTS does. + allocated: allocated.div_ceil(512).saturating_mul(512), + apparent, + }) + } + _ => None, + }; + Ok(Self { name, - exact, kind, - apparent, - allocated, - modified, - device: i64::from(device).cast_unsigned(), - inode, - links, + error, flags, - })) + metadata, + }) + } + + fn entry(self, parent: Arc) -> Entry { + let mut kind = self.kind; + let metadata = if self.error != 0 { + OnceCell::from(Err(io::Error::from_raw_os_error(self.error))) + } else if matches!(kind, Kind::Directory | Kind::Symlink) + && self.flags.is_some_and(|flags| flags & SF_FIRMLINK == 0) + { + // The scanner stats directories immediately before descent and + // links when deciding whether to follow them. Fetching their + // metadata here would repeat that work for every such entry. + OnceCell::new() + } else if kind == Kind::File + && self.flags.is_some_and(|flags| flags & SF_FIRMLINK == 0) + && let Some(metadata) = self.metadata + { + OnceCell::from(Ok(metadata)) + } else { + // Bulk metadata describes underlying mount/firmlink entries. + // A path lookup sees the visible directory; it also supplies + // missing attributes and metadata for special file types. + OnceCell::from(fs::symlink_metadata(parent.join(&self.name)).map( + |meta| { + kind = Kind::of(meta.file_type()); + Metadata::of(&meta) + }, + )) + }; + Entry { + file_name: self.name, + kind, + metadata, + parent, + } } } -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - use std::os::unix::fs::symlink; - use tempfile::TempDir; - - fn listed(dir: &Path) -> Vec { - let mut entries: Vec = read_dir(dir) - .expect("list") - .collect::>() - .expect("entries"); - entries.sort_by(|left, right| left.name.cmp(&right.name)); - entries - } - - /// What `lstat` says of each entry, the way the standard listing reads it. - fn stated(dir: &Path) -> Vec { - let mut entries: Vec = fs::read_dir(dir) - .expect("list") - .map(|entry| { - let entry = entry.expect("entry"); - let stat = rustix::fs::lstat(entry.path()).expect("lstat"); - Entry::from_stat(entry.file_name().as_bytes(), &stat) - }) - .collect(); - entries.sort_by(|left, right| left.name.cmp(&right.name)); - entries - } - - type Fields = (String, Kind, u64, u64, i64, (u64, u64), bool, bool); - - /// What the walk reads of a leaf. - fn fields(entry: &Entry) -> Fields { - ( - entry.name.to_string(), - entry.kind, - entry.apparent, - entry.allocated, - entry.modified, - entry.identity(), - entry.shared(), - entry.evicted(), - ) - } - - #[test] - fn a_listing_has_names_kinds_lengths_and_no_dot_entries() { - let temp = TempDir::new().expect("tempdir"); - fs::create_dir(temp.path().join("sub")).expect("mkdir"); - fs::write(temp.path().join("data.bin"), vec![7_u8; 100_000]) - .expect("write"); - symlink("data.bin", temp.path().join("link")).expect("symlink"); - - let entries = listed(temp.path()); - let names: Vec<&OsStr> = entries.iter().map(Entry::file_name).collect(); - assert_eq!(names, ["data.bin", "link", "sub"]); - assert_eq!(entries[0].kind(), Kind::File); - assert_eq!(entries[0].apparent(), 100_000); - assert_eq!(entries[0].path(temp.path()), temp.path().join("data.bin")); - assert_eq!(entries[1].kind(), Kind::Link); - assert_eq!(entries[2].kind(), Kind::Directory); - assert!(entries[0].modified() > 0, "written just now"); - } - - #[test] - fn every_leaf_reads_as_lstat_reads_it() { - let temp = TempDir::new().expect("tempdir"); - let root = temp.path(); - fs::write(root.join("plain"), b"hi").expect("write"); - fs::hard_link(root.join("plain"), root.join("hard")).expect("link"); - // Ten megabytes long, nothing written: no blocks. - fs::File::create(root.join("sparse")) - .and_then(|file| file.set_len(10 << 20)) - .expect("sparse"); - symlink("plain", root.join("link")).expect("symlink"); - // A resource fork, which `st_blocks` counts and `st_size` does not. - let forked = root.join("forked"); - fs::write(&forked, b"data").expect("write"); - rustix::fs::setxattr( - &forked, - "com.apple.ResourceFork", - &vec![b'r'; 20_000], - rustix::fs::XattrFlags::empty(), - ) - .expect("resource fork"); +fn modified_seconds(raw: [u8; 16]) -> io::Result { + let seconds = i64::from_ne_bytes(raw[..8].try_into().expect("eight bytes")); + let nanos = i64::from_ne_bytes(raw[8..].try_into().expect("eight bytes")); + if !(-1_000_000_000..1_000_000_000).contains(&nanos) { + return Err(invalid("invalid modification time")); + } + // Rust's existing scan maps every pre-epoch timestamp to zero. Darwin + // can represent the fractional part before the epoch with negative ns. + Ok(if seconds < 0 || (seconds == 0 && nanos < 0) { + 0 + } else if nanos < 0 { + seconds - 1 + } else { + seconds + }) +} - let listed: Vec = listed(root).iter().map(fields).collect(); - let stated: Vec = stated(root).iter().map(fields).collect(); - assert_eq!(listed, stated); - let forked = listed.iter().find(|leaf| leaf.0 == "forked").unwrap(); - assert_eq!(forked.2, 4, "the data fork's length"); - assert!(forked.3 >= 20_000, "the fork's blocks: {}", forked.3); - let plain = listed.iter().find(|leaf| leaf.0 == "plain").unwrap(); - assert!(plain.6, "two names"); - } +struct Cursor<'a> { + bytes: &'a [u8], + offset: usize, +} - #[test] - fn a_directory_larger_than_the_buffer_is_listed_whole() { - let temp = TempDir::new().expect("tempdir"); - // About a hundred bytes a record: several fills. - let count = 3 * BUFFER_BYTES / 100; - for index in 0..count { - fs::write(temp.path().join(format!("file-{index:05}")), b"") - .expect("write"); +impl<'a> Cursor<'a> { + const fn new(bytes: &'a [u8]) -> Self { + Self { bytes, offset: 0 } + } + + fn take(&mut self) -> io::Result<[u8; N]> { + let value = self + .bytes + .get(self.offset..) + .and_then(|bytes| bytes.first_chunk::()) + .ok_or_else(|| invalid("attribute exceeds directory record"))?; + self.offset += N; + Ok(*value) + } + + fn attribute( + &mut self, + mask: u32, + bit: u32, + ) -> io::Result> { + if mask & bit == 0 { + Ok(None) + } else { + self.take().map(Some) } - let entries = listed(temp.path()); - assert_eq!(entries.len(), count); - assert_eq!(entries[0].name(), "file-00000"); - assert_eq!(entries[count - 1].name(), format!("file-{:05}", count - 1)); } - #[test] - fn names_are_text_and_open() { - let temp = TempDir::new().expect("tempdir"); - for name in ["café", "日本語", "with space", "a\nb"] { - fs::write(temp.path().join(name), b"x").expect("write"); - } - let entries = listed(temp.path()); - assert_eq!(entries.len(), 4); - for entry in entries { - assert!(entry.exact.is_none(), "{}", entry.name()); - assert!(fs::metadata(entry.path(temp.path())).is_ok()); + fn name(&mut self, common: u32) -> io::Result { + if common & libc::ATTR_CMN_NAME == 0 { + return Err(invalid("directory record has no name")); } + let reference = self.offset; + let relative = i32::from_ne_bytes(self.take()?); + let length = u32::from_ne_bytes(self.take()?); + let start = isize::try_from(relative) + .ok() + .and_then(|relative| reference.checked_add_signed(relative)); + let name = start + .and_then(|start| { + usize::try_from(length) + .ok() + .and_then(|length| start.checked_add(length)) + .and_then(|end| self.bytes.get(start..end)) + }) + .ok_or_else(|| invalid("filename exceeds directory record"))?; + let name = name + .strip_suffix(&[0]) + .filter(|name| { + !name.is_empty() && !name.contains(&0) && !name.contains(&b'/') + }) + .ok_or_else(|| invalid("invalid directory entry name"))?; + Ok(OsString::from_vec(name.to_vec())) } +} - #[test] - fn a_missing_directory_is_an_error() { - let temp = TempDir::new().expect("tempdir"); - let error = read_dir(&temp.path().join("gone")).expect_err("missing"); - assert_eq!(error.kind(), io::ErrorKind::NotFound); - } +fn invalid(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) } + +#[cfg(test)] +#[path = "macos/tests.rs"] +mod tests; diff --git a/crates/disktree-core/src/macos/tests.rs b/crates/disktree-core/src/macos/tests.rs new file mode 100644 index 00000000..16a0e3b5 --- /dev/null +++ b/crates/disktree-core/src/macos/tests.rs @@ -0,0 +1,285 @@ +use super::*; +use std::collections::BTreeSet; + +// Build the documented wire format, including omitted attributes. Values +// distinguish adjacent fields so misaligned reads cannot pass by accident. +fn record(common: u32, file: u32, name: &[u8]) -> Vec { + let mut bytes = Vec::new(); + for word in [0, common, 0, 0, file, 0] { + bytes.extend(word.to_ne_bytes()); + } + if common & ATTR_CMN_ERROR != 0 { + bytes.extend(0_i32.to_ne_bytes()); + } + let name_at = bytes.len(); + if common & libc::ATTR_CMN_NAME != 0 { + bytes.extend([0_u8; 8]); + } + for (bit, value) in [ + (libc::ATTR_CMN_DEVID, (-3_i32).to_ne_bytes().to_vec()), + (libc::ATTR_CMN_OBJTYPE, 1_u32.to_ne_bytes().to_vec()), + ( + libc::ATTR_CMN_MODTIME, + [123_i64.to_ne_bytes(), 456_i64.to_ne_bytes()].concat(), + ), + (libc::ATTR_CMN_FLAGS, 0_u32.to_ne_bytes().to_vec()), + (libc::ATTR_CMN_FILEID, u64::MAX.to_ne_bytes().to_vec()), + ] { + if common & bit != 0 { + bytes.extend(value); + } + } + for (bit, value) in [ + (libc::ATTR_FILE_LINKCOUNT, 2_u32.to_ne_bytes().to_vec()), + (libc::ATTR_FILE_ALLOCSIZE, 513_u64.to_ne_bytes().to_vec()), + (libc::ATTR_FILE_DATALENGTH, 321_u64.to_ne_bytes().to_vec()), + ] { + if file & bit != 0 { + bytes.extend(value); + } + } + if common & libc::ATTR_CMN_NAME != 0 { + let relative = i32::try_from(bytes.len() - name_at).unwrap(); + let length = u32::try_from(name.len() + 1).unwrap(); + bytes[name_at..name_at + 4].copy_from_slice(&relative.to_ne_bytes()); + bytes[name_at + 4..name_at + 8].copy_from_slice(&length.to_ne_bytes()); + bytes.extend(name); + bytes.push(0); + } + bytes.resize(bytes.len().next_multiple_of(8), 0); + let length = u32::try_from(bytes.len()).unwrap(); + bytes[..4].copy_from_slice(&length.to_ne_bytes()); + bytes +} + +#[test] +fn packed_fields_keep_signed_devices_sizes_and_exact_names() { + let parsed = Record::parse(&record(COMMON, FILE, b"n\xff")).unwrap(); + assert_eq!(parsed.name, OsString::from_vec(b"n\xff".to_vec())); + assert_eq!(parsed.kind, Kind::File); + let meta = parsed.metadata.unwrap(); + assert_eq!(meta.device, (-3_i64).cast_unsigned()); + assert_eq!(meta.inode, u64::MAX); + assert_eq!(meta.links, 2); + assert_eq!(meta.modified, 123); + assert_eq!(meta.allocated, 1024); + assert_eq!(meta.apparent, 321); +} + +#[test] +fn missing_attributes_use_stat_without_following_symlinks() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("file"), b"actual file").unwrap(); + symlink("file", temp.path().join("link")).unwrap(); + let common = libc::ATTR_CMN_RETURNED_ATTRS | libc::ATTR_CMN_NAME; + for name in [b"file".as_slice(), b"link".as_slice()] { + let parsed = Record::parse(&record(common, 0, name)).unwrap(); + assert!(parsed.metadata.is_none()); + let entry = parsed.entry(Arc::from(temp.path())); + let stat = fs::symlink_metadata(entry.path()).unwrap(); + assert_eq!(entry.kind, Kind::of(stat.file_type())); + let metadata = entry.metadata().unwrap(); + assert_eq!(metadata.apparent, stat.len()); + assert_eq!(metadata.allocated, stat.blocks() * 512); + assert_eq!(metadata.inode, stat.ino()); + } +} + +#[test] +fn returned_errors_keep_errno_even_when_the_path_exists() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("file"), b"data").unwrap(); + let mut bytes = record(COMMON, FILE, b"file"); + bytes[HEADER_BYTES..HEADER_BYTES + 4] + .copy_from_slice(&libc::EACCES.to_ne_bytes()); + let entry = Record::parse(&bytes).unwrap().entry(Arc::from(temp.path())); + assert_eq!( + entry.metadata().unwrap_err().raw_os_error(), + Some(libc::EACCES) + ); +} + +#[test] +fn invalid_records_stop_iteration_once_without_panicking() { + let temp = tempfile::tempdir().unwrap(); + let valid = record(COMMON, FILE, b"file"); + let mut bad_reference = valid.clone(); + // NAME's attrreference follows the header and entry error. + bad_reference[28..32].copy_from_slice(&i32::MAX.to_ne_bytes()); + let mut bad_length = valid.clone(); + bad_length[..4].copy_from_slice(&u32::MAX.to_ne_bytes()); + let mut bad_bitmap = valid.clone(); + bad_bitmap[4..8].copy_from_slice(&0_u32.to_ne_bytes()); + let mut bad_alignment = valid.clone(); + bad_alignment[..4].copy_from_slice(&25_u32.to_ne_bytes()); + for bytes in [ + bad_reference, + bad_length, + bad_bitmap, + bad_alignment, + record(COMMON, FILE, b"../escape"), + record(COMMON, FILE, b"nul\0name"), + ] { + let mut reader = read_dir(temp.path()).unwrap(); + reader.buffer.bytes()[..bytes.len()].copy_from_slice(&bytes); + reader.remaining = 1; + assert_eq!( + reader.next().unwrap().unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + assert!(reader.next().is_none()); + } + // Every truncation before the filename terminator is rejected. + let name_end = 28 + + usize::try_from(i32::from_ne_bytes( + valid[28..32].try_into().unwrap(), + )) + .unwrap() + + usize::try_from(u32::from_ne_bytes( + valid[32..36].try_into().unwrap(), + )) + .unwrap(); + for length in 0..name_end { + assert!(Record::parse(&valid[..length]).is_err(), "length {length}"); + } +} + +#[test] +fn fallback_lists_entries_but_never_restarts_a_partial_directory() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("file"), b"data").unwrap(); + for code in [libc::ENOTSUP, libc::ENOSYS, libc::EACCES] { + let mut reader = read_dir(temp.path()).unwrap(); + reader + .refill_error(io::Error::from_raw_os_error(code)) + .unwrap(); + let entries: Vec<_> = reader.map(Result::unwrap).collect(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].file_name, "file"); + assert_eq!(entries[0].metadata().unwrap().apparent, 4); + + let mut reader = read_dir(temp.path()).unwrap(); + reader.started = true; + assert_eq!( + reader + .refill_error(io::Error::from_raw_os_error(code)) + .unwrap_err() + .raw_os_error(), + Some(code) + ); + assert!(reader.fallback.is_none()); + } +} + +#[test] +fn several_buffers_return_each_name_once() { + let temp = tempfile::tempdir().unwrap(); + for index in 0..1200 { + fs::write(temp.path().join(format!("file-{index:04}")), b"data") + .unwrap(); + } + let mut reader = read_dir(temp.path()).unwrap(); + let first = reader.next().unwrap().unwrap(); + assert!( + reader.remaining < 1199, + "the fixture spans multiple refills" + ); + let mut names = BTreeSet::from([first.file_name]); + for entry in reader.by_ref() { + let entry = entry.unwrap(); + assert_eq!(entry.metadata().unwrap().apparent, 4); + assert!(names.insert(entry.file_name), "no duplicate names"); + } + assert_eq!(names.len(), 1200); + assert!(reader.next().is_none()); +} + +#[test] +fn firmlinks_and_missing_flags_use_visible_path_metadata() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir(temp.path().join("directory")).unwrap(); + for flags in [Some(SF_FIRMLINK), None] { + // Underlying records can disagree with the visible path. Even a + // complete regular-file record must not bypass this lookup. + let mut parsed = + Record::parse(&record(COMMON, FILE, b"directory")).unwrap(); + parsed.flags = flags; + let entry = parsed.entry(Arc::from(temp.path())); + assert_eq!(entry.kind, Kind::Directory); + let stat = fs::symlink_metadata(entry.path()).unwrap(); + assert_eq!(entry.metadata().unwrap().inode, stat.ino()); + } +} + +#[test] +fn readable_directory_without_search_matches_standard_errors() { + use std::os::unix::fs::PermissionsExt as _; + + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("restricted"); + fs::create_dir(&path).unwrap(); + fs::write(path.join("file"), b"data").unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap(); + let native: io::Result> = + read_dir(&path).and_then(Iterator::collect); + let standard: io::Result> = fs::read_dir(&path) + .map(|entries| { + entries + .map(|entry| { + entry.map(|entry| (entry.file_name(), entry.metadata())) + }) + .collect() + }) + .and_then(std::convert::identity); + // Restore access before assertions so a failed assertion cannot leave + // cleanup unable to enter the temporary fixture. + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); + let native = native.unwrap(); + let standard = standard.unwrap(); + assert_eq!(native.len(), 1); + assert_eq!(standard.len(), 1); + assert_eq!(native[0].file_name, standard[0].0); + match (native[0].metadata(), &standard[0].1) { + (Err(left), Err(right)) => { + assert_eq!(left.raw_os_error(), right.raw_os_error()); + assert_eq!(left.raw_os_error(), Some(libc::EACCES)); + } + // Elevated test runners can still stat through this directory. + (Ok(left), Ok(right)) => assert_eq!(left.apparent, right.len()), + _ => panic!("bulk and stat disagree on metadata access"), + } +} + +#[test] +fn directory_and_link_stats_are_deferred_until_needed() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::create_dir(root.join("directory")).unwrap(); + symlink("missing", root.join("link")).unwrap(); + let entries: Vec<_> = read_dir(root).unwrap().map(Result::unwrap).collect(); + assert_eq!(entries.len(), 2); + for entry in &entries { + assert!( + entry.metadata.get().is_none(), + "enumeration must not stat this entry" + ); + } + // No earlier cached stat can hide a directory disappearing before the + // consumer requests its metadata. Links still use no-follow metadata. + fs::remove_dir(root.join("directory")).unwrap(); + for entry in entries { + if entry.kind == Kind::Directory { + assert_eq!( + entry.metadata().unwrap_err().kind(), + io::ErrorKind::NotFound + ); + } else { + assert_eq!(entry.kind, Kind::Symlink); + assert_eq!(entry.metadata().unwrap().apparent, 7); + } + } +} diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rs index 693253a7..d2e6730a 100644 --- a/crates/disktree-core/src/scan.rs +++ b/crates/disktree-core/src/scan.rs @@ -15,7 +15,7 @@ //! poll without locking, and cooperative cancellation so a re-scan can abandon //! a walk of a large home directory instead of queueing behind it. -#[cfg(not(any(windows, target_os = "macos")))] +#[cfg(all(not(windows), any(test, not(target_os = "macos"))))] use std::fs::DirEntry; use std::fs::{self, Metadata}; use std::io; @@ -495,12 +495,10 @@ impl WalkContext { /// What the walk reads about one directory entry. /// -/// `std::fs::DirEntry` everywhere but Windows and macOS. On Windows the -/// standard listing has neither the allocated size nor a file id, so -/// measuring like `du` would cost an open per file; [`crate::windows`] lists -/// a directory with both instead. On macOS it would cost an `lstat` per -/// entry, and [`crate::macos`] lists a directory with everything a stat -/// says. +/// Native bulk metadata on macOS and Windows, `std::fs::DirEntry` elsewhere. +/// Bulk enumeration avoids a metadata syscall per file on macOS. On Windows +/// the standard listing lacks allocation and identity; [`crate::windows`] +/// returns both without opening each file. trait Listed { /// The entry's path inside `dir`, the directory it was listed from. fn entry_path(&self, dir: &Path) -> PathBuf; @@ -568,15 +566,15 @@ impl Facts { } } -/// A standard directory entry with its name decoded once, up front. -#[cfg(not(any(windows, target_os = "macos")))] -struct Named { - entry: DirEntry, +/// A directory entry with its name decoded once, up front. +#[cfg(not(windows))] +struct Named { + entry: E, name: Box, } -#[cfg(not(any(windows, target_os = "macos")))] -impl Listed for Named { +#[cfg(all(not(windows), any(test, not(target_os = "macos"))))] +impl Listed for Named { fn entry_path(&self, _dir: &Path) -> PathBuf { self.entry.path() } @@ -616,50 +614,6 @@ impl Listed for Named { } } -#[cfg(target_os = "macos")] -impl Listed for crate::macos::Entry { - fn entry_path(&self, dir: &Path) -> PathBuf { - self.path(dir) - } - - fn name(&self) -> &str { - self.name() - } - - fn take_name(&mut self) -> Box { - self.take_name() - } - - fn listing(&self) -> io::Result { - Ok(match self.kind() { - crate::macos::Kind::Link => Listing::Symlink, - crate::macos::Kind::Directory => Listing::Directory, - crate::macos::Kind::File => Listing::Leaf(NodeKind::File), - crate::macos::Kind::Other => Listing::Leaf(NodeKind::Other), - }) - } - - fn facts(&self, apparent_size: bool) -> io::Result { - Ok(Facts { - size: if apparent_size { - self.apparent() - } else { - self.allocated() - }, - identity: Some(self.identity()), - modified: self.modified(), - shared: self.shared(), - }) - } - - fn directory(&self) -> io::Result { - Ok(Directory { - device: self.device(), - evicted: self.evicted(), - }) - } -} - #[cfg(windows)] impl Listed for crate::windows::Entry { fn entry_path(&self, dir: &Path) -> PathBuf { @@ -717,7 +671,7 @@ impl Listed for crate::windows::Entry { fn list( path: &Path, _volume: Option, -) -> io::Result>> { +) -> io::Result>>> { Ok(fs::read_dir(path)?.map(|entry| { entry.map(|entry| Named { name: entry.file_name().into_string().map_or_else( @@ -729,14 +683,6 @@ fn list( })) } -#[cfg(target_os = "macos")] -fn list( - path: &Path, - _volume: Option, -) -> io::Result { - crate::macos::read_dir(path) -} - #[cfg(windows)] fn list( path: &Path, @@ -745,6 +691,77 @@ fn list( crate::windows::read_dir(path, volume) } +#[cfg(target_os = "macos")] +impl Listed for Named { + fn entry_path(&self, _dir: &Path) -> PathBuf { + self.entry.path() + } + + fn name(&self) -> &str { + &self.name + } + + fn take_name(&mut self) -> Box { + std::mem::take(&mut self.name) + } + + fn listing(&self) -> io::Result { + Ok(match self.entry.kind { + crate::macos::Kind::Directory => Listing::Directory, + crate::macos::Kind::Symlink => Listing::Symlink, + crate::macos::Kind::File => Listing::Leaf(NodeKind::File), + crate::macos::Kind::Other => Listing::Leaf(NodeKind::Other), + }) + } + + fn facts(&self, apparent_size: bool) -> io::Result { + let meta = self.entry.metadata().map_err(|error| { + error.raw_os_error().map_or_else( + || io::Error::new(error.kind(), error.to_string()), + io::Error::from_raw_os_error, + ) + })?; + Ok(Facts { + size: if apparent_size { + meta.apparent + } else { + meta.allocated + }, + identity: Some((meta.device, meta.inode)), + modified: meta.modified, + shared: meta.links > 1, + }) + } + + fn directory(&self) -> io::Result { + // Recheck SF_DATALESS before descent: provider state can change + // after enumeration, and listing it could download cloud contents. + fs::symlink_metadata(self.entry.path()).map(|meta| Directory { + device: device_of(&meta), + evicted: is_dataless(&meta), + }) + } +} + +#[cfg(target_os = "macos")] +fn list( + path: &Path, + _volume: Option, +) -> io::Result>>> { + // Only enumeration changes: Rayon still owns directory scheduling and + // completion. Request total allocation, without clone queries. + Ok(crate::macos::read_dir(path)?.map(|entry| { + entry.map(|entry| Named { + name: entry.file_name.to_string_lossy().into(), + entry, + }) + })) +} + +#[cfg(all(test, target_os = "macos"))] +#[path = "scan/macos_tests.rs"] +mod macos_tests; + /// What a directory entry turned out to be. enum Classified { /// Descend into this directory on a new task. diff --git a/crates/disktree-core/src/scan/macos_tests.rs b/crates/disktree-core/src/scan/macos_tests.rs new file mode 100644 index 00000000..03089aa4 --- /dev/null +++ b/crates/disktree-core/src/scan/macos_tests.rs @@ -0,0 +1,205 @@ +use super::*; +use std::collections::HashMap; +use std::ffi::OsString; +use std::fs::{File, FileTimes}; +use std::os::unix::ffi::OsStringExt as _; +use std::os::unix::fs::symlink; +use std::time::{Duration, UNIX_EPOCH}; + +fn native(path: &Path) -> Vec> { + list(path, None) + .expect("list") + .map(Result::unwrap) + .collect() +} + +fn standard(entry: DirEntry) -> Named { + Named { + name: entry.file_name().to_string_lossy().into(), + entry, + } +} + +fn same_facts(native: &Named, standard: &Named) { + assert_eq!( + native.entry_path(Path::new("")), + standard.entry_path(Path::new("")) + ); + assert_eq!(native.name(), standard.name()); + for apparent in [false, true] { + let left = native.facts(apparent).expect("native facts"); + let right = standard.facts(apparent).expect("standard facts"); + assert_eq!( + left.size, + right.size, + "{}", + native.entry.file_name.display() + ); + assert_eq!(left.identity, right.identity); + assert_eq!(left.modified, right.modified); + assert_eq!(left.shared, right.shared); + } + match (native.listing().unwrap(), standard.listing().unwrap()) { + (Listing::Directory, Listing::Directory) => { + let left = native.directory().unwrap(); + let right = standard.directory().unwrap(); + assert_eq!(left.device, right.device); + assert_eq!(left.evicted, right.evicted); + } + (Listing::Symlink, Listing::Symlink) => {} + (Listing::Leaf(left), Listing::Leaf(right)) => assert_eq!(left, right), + _ => panic!("native and standard file types differ"), + } +} + +#[test] +fn bulk_metadata_matches_stat_accounting_and_names() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::create_dir(root.join("directory")).unwrap(); + fs::write(root.join("file"), vec![7_u8; 8193]).unwrap(); + fs::hard_link(root.join("file"), root.join("hardlink")).unwrap(); + symlink("file", root.join("symlink")).unwrap(); + symlink("missing", root.join("dangling")).unwrap(); + fs::write(root.join(".hidden"), b"hidden").unwrap(); + fs::write(root.join("café"), b"unicode").unwrap(); + let sparse = File::create(root.join("sparse")).unwrap(); + sparse.set_len(8 * 1024 * 1024).unwrap(); + fs::write(root.join("forked"), b"data").unwrap(); + fs::write(root.join("forked/..namedfork/rsrc"), vec![9_u8; 16385]) + .expect("resource fork"); + let standard: HashMap<_, _> = fs::read_dir(root) + .unwrap() + .map(|entry| { + let entry = entry.unwrap(); + (entry.file_name(), standard(entry)) + }) + .collect(); + let entries = native(root); + assert_eq!(entries.len(), standard.len()); + for entry in entries { + same_facts(&entry, &standard[&entry.entry.file_name]); + } +} + +#[test] +fn native_paths_preserve_non_utf8_names() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("file"), b"data").unwrap(); + let mut entry = native(temp.path()).pop().unwrap(); + // APFS rejects malformed UTF-8 at creation. Inject a returned name to + // verify that only display, never path resolution, is lossy. + entry.entry.file_name = OsString::from_vec(vec![b'n', 0xff]); + entry.name = entry.entry.file_name.to_string_lossy().into(); + assert_eq!( + entry.entry_path(temp.path()), + temp.path().join(&entry.entry.file_name) + ); + assert_eq!(entry.name(), "n�"); + assert_eq!(&*entry.take_name(), "n�"); + assert!(entry.name().is_empty()); +} + +#[test] +fn bulk_timestamps_keep_pre_epoch_semantics() { + let temp = tempfile::tempdir().unwrap(); + for (name, time) in [ + ("before", UNIX_EPOCH - Duration::from_millis(500)), + ("after", UNIX_EPOCH + Duration::from_millis(1500)), + ] { + let file = File::create(temp.path().join(name)).unwrap(); + file.set_times(FileTimes::new().set_modified(time)).unwrap(); + } + for entry in native(temp.path()) { + let standard = fs::read_dir(temp.path()) + .unwrap() + .map(Result::unwrap) + .find(|other| other.file_name() == entry.entry.file_name) + .unwrap(); + same_facts(&entry, &self::standard(standard)); + assert_eq!( + entry.facts(false).unwrap().modified, + i64::from(entry.entry.file_name == "after") + ); + } +} + +#[test] +fn native_metadata_errors_keep_os_codes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("file"), b"data").unwrap(); + let mut entry = native(temp.path()).pop().unwrap(); + // Inject a per-entry failure, without + // depending on the test runner's privileges or racing another thread. + for code in [rustix::io::Errno::ACCESS, rustix::io::Errno::NOENT] { + entry.entry.metadata = std::cell::OnceCell::from(Err( + io::Error::from_raw_os_error(code.raw_os_error()), + )); + assert_eq!( + entry.facts(false).err().unwrap().raw_os_error(), + Some(code.raw_os_error()) + ); + } +} + +#[test] +fn directory_facts_are_rechecked_before_descent() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("directory"); + fs::create_dir(&path).unwrap(); + let entry = native(temp.path()).pop().unwrap(); + assert!(!entry.directory().unwrap().evicted); + fs::remove_dir(path).unwrap(); + assert_eq!( + entry.directory().err().unwrap().kind(), + io::ErrorKind::NotFound + ); +} + +#[test] +fn evicted_directory_facts_prevent_descent() { + struct Evicted(PathBuf); + impl Listed for Evicted { + fn entry_path(&self, _: &Path) -> PathBuf { + self.0.clone() + } + fn name(&self) -> &'static str { + "cloud" + } + fn take_name(&mut self) -> Box { + "cloud".into() + } + fn listing(&self) -> io::Result { + Ok(Listing::Directory) + } + fn facts(&self, _: bool) -> io::Result { + panic!("directory must not be measured as a leaf") + } + fn directory(&self) -> io::Result { + Ok(Directory { + device: 0, + evicted: true, + }) + } + } + let context = WalkContext { + known: None, + options: ScanOptions::default(), + progress: Arc::new(ScanProgress::default()), + root_device: Mutex::new(None), + foreign_mounts: OnceLock::new(), + never_scanned: OnceLock::new(), + visited_dirs: Mutex::new(FxHashSet::default()), + root: Mutex::new(None), + volume: OnceLock::new(), + }; + let temp = tempfile::tempdir().unwrap(); + fs::create_dir(temp.path().join("cloud")).unwrap(); + fs::write(temp.path().join("cloud/local-child"), b"not visited").unwrap(); + // Synthetic facts exercise the descent decision, not a live provider. + assert!(matches!( + context.classify(temp.path(), &mut Evicted(temp.path().join("cloud"))), + Classified::Skipped + )); + assert_eq!(context.progress.snapshot().dirs, 0); +}