From e9b4996c0acfe78f5e2aa317d2f79533e35742c2 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Sun, 27 Sep 2026 09:43:20 -0400 Subject: [PATCH 01/19] Report APFS clone sharing and estimate private-byte savings --- AGENTS.md | 12 +- Cargo.lock | 1 + README.md | 14 +- crates/disktree-app/src/state.rs | 9 +- crates/disktree-app/src/tests.rs | 71 ++++ crates/disktree-app/src/views.rs | 54 ++- crates/disktree-core/Cargo.toml | 1 + .../disktree-core/examples/scan_accounting.rs | 36 ++ crates/disktree-core/src/lib.rs | 1 + crates/disktree-core/src/macos.rs | 365 +++++++++++++++++- crates/disktree-core/src/removal.rs | 60 ++- crates/disktree-core/src/scan.rs | 49 ++- crates/disktree-core/src/sharing.rs | 213 ++++++++++ crates/disktree-core/src/tree.rs | 16 +- licenses/diskhound-MIT.txt | 21 + 15 files changed, 879 insertions(+), 44 deletions(-) create mode 100644 crates/disktree-core/examples/scan_accounting.rs create mode 100644 crates/disktree-core/src/sharing.rs create mode 100644 licenses/diskhound-MIT.txt diff --git a/AGENTS.md b/AGENTS.md index a567ce80..4ceabdb6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,11 +62,11 @@ and `cargo build --release` directly; CI runs the gate on both systems. ## Invariants 1. **Sizes come from `st_blocks * 512` unless apparent size was asked for.** - That is the number that comes back when a file is deleted. On macOS the - directory listing reports it and `st_size` for a whole buffer of entries: - every fork's allocation, and the data fork's length; see `macos.rs`. On - Windows it is the allocation the directory listing reports; see - `windows.rs`. An + APFS clones can share those blocks; private-byte estimates are reported + separately without changing tile weights. On macOS the directory listing + reports every fork's allocation and the data fork's length in bulk; see + `macos.rs`. On Windows it is the allocation the directory listing reports; + see `windows.rs`. An elevated scan of a whole NTFS drive reads it from the master file table instead, keeping the walk's rules for hidden entries, links, cloud folders and depth; see `mft.rs`. That path counts every stream's @@ -119,6 +119,8 @@ and `cargo build --release` directly; CI runs the gate on both systems. | what macOS lists and measures in bulk | `crates/disktree-core/src/macos.rs` — `unsafe` for `getattrlistbulk` | | what Windows lists, measures and compares differently | `crates/disktree-core/src/windows.rs` — `unsafe` for Win32 | | reading a whole NTFS drive from its file table | `crates/disktree-core/src/mft.rs` | +| APFS clone hints and private-size system calls | `crates/disktree-core/src/macos.rs` — isolated `unsafe` | +| clone sharing totals and conservative savings | `crates/disktree-core/src/sharing.rs` | | a key, a screen transition, a mark | `crates/disktree-app/src/state.rs` | | spacing, type and size | `crates/disktree-app/src/ui.rs` — tokens only, no `px` in layout | | the mosaic's painting or labels | `crates/disktree-app/src/treemap_view.rs` | diff --git a/Cargo.lock b/Cargo.lock index 71482b53..a05fb428 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1437,6 +1437,7 @@ name = "disktree-core" version = "0.11.0" dependencies = [ "chrono", + "libc", "rayon", "rustc-hash 2.1.3", "rustix", diff --git a/README.md b/README.md index 80442695..c7135bce 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,18 @@ What is different from Linux: - **Free space** is what `df` reports. Finder's figure is larger: it counts purgeable space (caches and local snapshots macOS will clear on its own). - **Cloned files** (copies APFS shares blocks between, as Finder's Duplicate - makes) are each counted in full, so a total can exceed what deleting them - frees. + makes) remain counted in full in the treemap. The scan line separately + reports bytes in cloned files and estimated repeated data-fork allocation. + The selection, review and permanent-delete confirmation estimate savings + using private bytes for clones. Unknown private sizes contribute zero. + These estimates exclude shared extents even when all copies are marked, + so deleting copies together can free more. Snapshots can retain blocks, + and changing files can stale the estimate; rescan to refresh it. The final + result still measures the volume's actual free-space change. + Clone hints come from bulk directory reads; only potentially shared files + need a private-size query. Unsupported filesystems or kernels fall back to + the ordinary listing without clone detection. Apparent-size scans do not + collect clone metadata. - **Time Machine's local snapshots** are not files and do not appear; they are part of the gap between the scan and the disk's used space. - **Cloud-only folders** (iCloud Drive, Dropbox and the like, evicted to the diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs index ee351f3a..02ba8b49 100644 --- a/crates/disktree-app/src/state.rs +++ b/crates/disktree-app/src/state.rs @@ -1767,7 +1767,14 @@ impl Disktree { /// The plan the review screen shows and the removal runs. pub fn plan(&self) -> Plan { - plan(self.marks.items(), &self.root_path) + let mut plan = plan(self.marks.items(), &self.root_path); + if let Some(tree) = self.tree() { + plan.estimate_reclaim(|target| { + crate::marks::find(&self.root_path, tree, &target.path) + .map(|node| node.sharing().reclaimable(node.bytes)) + }); + } + plan } // ── layout and hit-testing ────────────────────────────────────────── diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index 3d7f151b..142c33e4 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -1721,6 +1721,77 @@ fn restart_arguments_parse_back_to_the_same_scan() { } } +#[cfg(target_os = "macos")] +#[gpui_kit::test] +fn apfs_clones_use_private_bytes_in_review_and_refresh( + cx: &mut TestAppContext, +) { + cx.update(gpui_omarchy::init); + // The macOS system-tree guard deliberately refuses canonical /private. + let temp = tempfile::tempdir_in(std::env::current_dir().expect("cwd")) + .expect("tempdir in the checkout"); + let root = std::fs::canonicalize(temp.path()).expect("canonical root"); + std::fs::create_dir(root.join("copies")).expect("mkdir"); + let original = root.join("original"); + std::fs::write(&original, vec![0x5a; 8 * 1024 * 1024]).expect("write"); + assert!( + std::process::Command::new("cp") + .arg("-c") + .arg(&original) + .arg(root.join("copies/clone")) + .status() + .expect("cp -c") + .success() + ); + let options = ScanOptions::default(); + let tree = scan(&root, options.clone()).expect("scan"); + let root_for_view = root.clone(); + let (view, cx) = cx.add_window_view(move |_, cx| { + Disktree::with_tree(root_for_view.clone(), tree, options, 3, cx) + }); + draw(cx); + update(&view, cx, |app, cx| { + let index = app + .tree() + .expect("tree") + .children + .iter() + .position(|node| node.name.as_ref() == "copies") + .expect("copies"); + app.toggle_mark(&[index], cx); + app.screen = Screen::Review; + }); + draw(cx); + read(&view, cx, |app| { + assert_eq!(app.plan().bytes(), 8 * 1024 * 1024); + assert_eq!(app.plan().reclaimable_bytes(), 0); + }); + update(&view, cx, |app, _| { + app.removal_mode = RemovalMode::Permanent; + app.confirm_open = true; + }); + draw(cx); + update(&view, cx, |app, _| app.confirm_open = false); + // Break sharing and refresh the same absolute-path mark. + std::fs::remove_file(root.join("copies/clone")).expect("remove clone"); + std::fs::write(root.join("copies/clone"), vec![0x31; 8 * 1024 * 1024]) + .expect("replace"); + let refreshed = scan(&root, ScanOptions::default()).expect("rescan"); + update(&view, cx, |app, _| { + app.marks.refresh( + &root, + &refreshed, + disktree_core::tree::Metric::Bytes, + ); + app.tree = Some(std::sync::Arc::new(refreshed.clone())); + }); + assert_eq!( + read(&view, cx, |app| app.plan().reclaimable_bytes()), + 8 * 1024 * 1024 + ); + draw(cx); +} + #[gpui_kit::test] fn power_efficiency_menu_saves_without_discarding_the_tree( cx: &mut TestAppContext, diff --git a/crates/disktree-app/src/views.rs b/crates/disktree-app/src/views.rs index c23e91b3..be84d04e 100644 --- a/crates/disktree-app/src/views.rs +++ b/crates/disktree-app/src/views.rs @@ -266,8 +266,8 @@ fn delete_dialog( targets => format!("Delete {} items permanently?", targets.len()), }; let body = format!( - "This frees {}. Deleted files can\u{2019}t be recovered; move them to the trash if you might need them again.", - human_bytes(plan.bytes()) + "Estimated to free ≈ {}. Shared blocks and snapshots may retain space. Deleted files can\u{2019}t be recovered; move them to the trash if you might need them again.", + human_bytes(plan.reclaimable_bytes()) ); let confirm = cx.entity().downgrade(); let cancel = confirm.clone(); @@ -1228,6 +1228,19 @@ fn scan_totals(app: &Disktree, theme: &Theme) -> Div { widgets::human_count(files), widgets::human_count(dirs) )) + .when(tree.is_some_and(|node| node.sharing().files > 0), |this| { + let sharing = tree.expect("checked above").sharing(); + this.child(format!( + "· APFS clones: {} · ≈ {} counted again{}", + human_bytes(sharing.bytes), + human_bytes(sharing.duplicate_bytes), + if sharing.groups_truncated { + " (partial)" + } else { + "" + } + )) + }) .when(errors > 0, |this| { this.child(div().text_color(theme.warning).child(format!( "· {} unreadable", @@ -1440,7 +1453,21 @@ fn selection_section( text::TITLE, cx, )) - .child(widgets::bar(share, highlight, cx)); + .child(widgets::bar(share, highlight, cx)) + .when(node.sharing().files > 0, |this| { + this.child( + div() + .text_size(text::CAPTION) + .text_color(theme.secondary) + .child(format!( + "APFS clones · frees ≈ {} excluding shared blocks", + human_bytes(node.sharing().reclaimable(node.bytes)) + )), + ) + .when(node.sharing().unknown_files > 0, |this| { + this.child("Some clone private sizes are unavailable") + }) + }); let fourth = if node.is_dir() && let Some(path) = &path @@ -2075,7 +2102,7 @@ fn disk_section( )) .text_color(highlight) .child(format!( - "→ {} free", + "→ ≈ {} free", human_bytes(after.available) )), ) @@ -2197,7 +2224,7 @@ fn review_button( .overflow_hidden() .text_ellipsis() .child(format!( - "Review {count} marked · frees {}…", + "Review {count} marked · frees ≈ {}…", human_bytes(reclaiming) )), ) @@ -2583,9 +2610,9 @@ fn review( .child(screen_header( "Review", &format!( - "{} marked \u{00b7} {} to free", + "{} marked \u{00b7} ≈ {} to free", app.marks.len(), - human_bytes(plan.bytes()) + human_bytes(plan.reclaimable_bytes()) ), &theme, cx, @@ -2800,13 +2827,22 @@ fn review_summary( cx, )) .child(widgets::row( - "Space freed", - human_bytes(plan.bytes()), + if app.removal_mode == RemovalMode::Trash { + "After emptying trash (est.)" + } else { + "Estimated space freed" + }, + human_bytes(plan.reclaimable_bytes()), cx, )) .children(left_out), ); + if cfg!(target_os = "macos") { + panel = panel.child(div().text_size(text::CAPTION) + .text_color(theme.secondary) + .child("Estimate excludes shared clone blocks. Removing all copies together can free more; snapshots can retain space. Rescan to update estimates.")); + } if let Some(volume) = app.space { panel = panel.child( div() diff --git a/crates/disktree-core/Cargo.toml b/crates/disktree-core/Cargo.toml index 8f121ebf..12ba2740 100644 --- a/crates/disktree-core/Cargo.toml +++ b/crates/disktree-core/Cargo.toml @@ -34,6 +34,7 @@ windows-sys = { version = "0.61", features = [ # without asking to control Finder. Only this backend is used; the crate's # Linux and Windows code is not built. [target.'cfg(target_os = "macos")'.dependencies] +libc = "0.2" trash = { version = "5.2.9", default-features = false } [dev-dependencies] diff --git a/crates/disktree-core/examples/scan_accounting.rs b/crates/disktree-core/examples/scan_accounting.rs new file mode 100644 index 00000000..5608c205 --- /dev/null +++ b/crates/disktree-core/examples/scan_accounting.rs @@ -0,0 +1,36 @@ +//! Compare clone accounting with the ordinary walk on the same tree: +//! `cargo run --release -p disktree-core --example scan_accounting -- PATH` +//! Append --no-clones to measure the baseline without private-size queries. + +use std::path::PathBuf; +use std::time::Instant; + +use disktree_core::scan::{ScanOptions, scan}; + +fn main() -> std::io::Result<()> { + let mut args = std::env::args_os().skip(1); + let root = args + .next() + .map_or_else(|| PathBuf::from("."), PathBuf::from); + let clones = args.next().is_none_or(|arg| arg != "--no-clones"); + let started = Instant::now(); + let tree = scan( + &root, + ScanOptions { + apfs_clone_metadata: clones, + ..ScanOptions::default() + }, + )?; + println!( + "seconds={:.3} files={} listed={} clone_files={} clone_bytes={} duplicate_bytes={} private_bytes={} unknown_clones={}", + started.elapsed().as_secs_f64(), + tree.files, + tree.bytes, + tree.sharing().files, + tree.sharing().bytes, + tree.sharing().duplicate_bytes, + tree.sharing().private_bytes, + tree.sharing().unknown_files + ); + Ok(()) +} diff --git a/crates/disktree-core/src/lib.rs b/crates/disktree-core/src/lib.rs index a620d802..41b82362 100644 --- a/crates/disktree-core/src/lib.rs +++ b/crates/disktree-core/src/lib.rs @@ -17,6 +17,7 @@ mod mft; pub mod removal; pub mod scan; pub mod scan_threads; +pub mod sharing; pub mod size; pub mod space; pub mod tree; diff --git a/crates/disktree-core/src/macos.rs b/crates/disktree-core/src/macos.rs index 6cdac06d..b98ed6d1 100644 --- a/crates/disktree-core/src/macos.rs +++ b/crates/disktree-core/src/macos.rs @@ -8,12 +8,15 @@ #![allow( unsafe_code, - reason = "getattrlistbulk, which neither std nor rustix wraps; each block \ + reason = "Darwin attribute calls, which neither std nor rustix wraps; each block \ says why it is sound" )] use std::cell::RefCell; -use std::ffi::{OsStr, c_int, c_void}; +use std::ffi::{CString, OsStr, c_int, c_void}; +use std::fs::Metadata; +use std::os::unix::fs::MetadataExt as _; + use std::io; use std::os::fd::{AsRawFd as _, OwnedFd}; use std::os::unix::ffi::OsStrExt as _; @@ -21,9 +24,17 @@ use std::path::{Path, PathBuf}; use rustix::fs::{AtFlags, FileType, Mode, OFlags}; +use crate::sharing::CloneInfo; + /// Bytes the kernel fills per call: some hundreds of entries. const BUFFER_BYTES: usize = 64 * 1024; +const REFCNT: u32 = 0x1000; +const MAY_SHARE: u64 = 0x1 | 0x40; +const EXTENDED: u32 = libc::FSOPT_ATTR_CMN_EXTENDED; +const RETURNED: u32 = libc::ATTR_CMN_RETURNED_ATTRS; +const HINTS: u32 = libc::ATTR_CMNEXT_CLONEID | libc::ATTR_CMNEXT_EXT_FLAGS; + // From and , which the libc crate does not cover. const ATTR_BIT_MAP_COUNT: u16 = 5; const ATTR_CMN_NAME: u32 = 0x0000_0001; @@ -116,6 +127,7 @@ pub struct Entry { inode: u64, links: u32, flags: u32, + hints: Option<(u64, u64)>, } impl Entry { @@ -181,6 +193,21 @@ impl Entry { matches!(self.kind, Kind::Directory) && self.flags & SF_DATALESS != 0 } + /// Only clone candidates pay for private-size reads; ordinary metadata + /// and identity remain the values returned by the bulk listing. + pub fn clone_info(&self, dir: &Path) -> Option> { + let (id, flags) = self.hints?; + if self.kind != Kind::File || flags & MAY_SHARE == 0 { + return None; + } + Some(Box::new(private_info( + &self.path(dir), + self.device, + self.inode, + id, + ))) + } + fn named(name: &[u8]) -> (Box, Option>) { match std::str::from_utf8(name) { Ok(text) => (text.into(), None), @@ -210,6 +237,7 @@ impl Entry { inode: stat.st_ino, links: u32::from(stat.st_nlink), flags: stat.st_flags, + hints: None, } } } @@ -226,11 +254,12 @@ pub struct ReadDir { /// Where the next of them starts. offset: usize, done: bool, + clones: bool, } /// List `dir`, with each entry's size, identity and age. A link at `dir` /// itself is followed, as `std::fs::read_dir` follows it. -pub fn read_dir(dir: &Path) -> io::Result { +pub fn read_dir(dir: &Path, clones: bool) -> io::Result { let fd = rustix::fs::open( dir, OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, @@ -245,6 +274,7 @@ pub fn read_dir(dir: &Path) -> io::Result { remaining: 0, offset: 0, done: false, + clones, }) } @@ -305,7 +335,7 @@ impl ReadDir { volattr: 0, dirattr: 0, fileattr: FILE_ATTRIBUTES, - forkattr: 0, + forkattr: if self.clones { HINTS } else { 0 }, }; loop { // SAFETY: the descriptor is an open directory owned by `self`; @@ -318,13 +348,25 @@ impl ReadDir { &raw mut request, buffer.as_mut_ptr().cast(), BUFFER_BYTES, - 0, + if self.clones { u64::from(EXTENDED) } else { 0 }, ) }; if let Ok(count) = u32::try_from(count) { return Ok(count); } let error = io::Error::last_os_error(); + // Optional clone hints must not disable ordinary enumeration on + // volumes or kernels that reject extended attributes. + if self.clones + && matches!( + error.raw_os_error(), + Some(libc::EINVAL | libc::ENOTSUP) + ) + { + self.clones = false; + request.forkattr = 0; + continue; + } if error.kind() != io::ErrorKind::Interrupted { return Err(error); } @@ -362,7 +404,7 @@ impl ReadDir { let common = cursor.u32()?; cursor.at += 8; let file = cursor.u32()?; - cursor.at += 4; + let extended = cursor.u32()?; let error = if common & ATTR_CMN_ERROR == 0 { 0 } else { @@ -374,7 +416,7 @@ impl ReadDir { return Ok((Err(error), length)); } let entry = if common & MEASURED == MEASURED { - cursor.entry(file, name)? + cursor.entry(file, extended, name)? } else { None }; @@ -437,18 +479,28 @@ impl<'a> Cursor<'a> { let offset = isize::try_from(self.u32()?.cast_signed()) .map_err(|_| malformed())?; let length = self.u32()? as usize; - reference + let name = reference .checked_add_signed(offset) .zip(length.checked_sub(1)) .and_then(|(start, length)| { self.record.get(start..start.checked_add(length)?) }) - .ok_or_else(malformed) + .ok_or_else(malformed)?; + if name.is_empty() || name == b"." || name == b".." || name.contains(&0) + { + return Err(malformed()); + } + Ok(name) } /// The attributes after the name, every one of them asked for; `None` /// for a file whose sizes the file system left out. - fn entry(&mut self, file: u32, name: &[u8]) -> io::Result> { + fn entry( + &mut self, + file: u32, + extended: u32, + name: &[u8], + ) -> io::Result> { let device = self.u32()?.cast_signed(); let kind = match self.u32()? { VDIR => Kind::Directory, @@ -472,6 +524,17 @@ impl<'a> Cursor<'a> { } else { return Ok(None); }; + let clone_id = if extended & libc::ATTR_CMNEXT_CLONEID != 0 { + Some(self.u64()?) + } else { + None + }; + let clone_flags = if extended & libc::ATTR_CMNEXT_EXT_FLAGS != 0 { + Some(self.u64()?) + } else { + None + }; + let hints = clone_id.zip(clone_flags); let (name, exact) = Entry::named(name); Ok(Some(Entry { name, @@ -484,6 +547,7 @@ impl<'a> Cursor<'a> { inode, links, flags, + hints, })) } } @@ -496,7 +560,7 @@ mod tests { use tempfile::TempDir; fn listed(dir: &Path) -> Vec { - let mut entries: Vec = read_dir(dir) + let mut entries: Vec = read_dir(dir, false) .expect("list") .collect::>() .expect("entries"); @@ -578,6 +642,16 @@ mod tests { let listed: Vec = listed(root).iter().map(fields).collect(); let stated: Vec = stated(root).iter().map(fields).collect(); assert_eq!(listed, stated); + // Adding clone hints must preserve ordinary metadata, including + // resource forks and hardlink identity in the same bulk record. + let mut clone_entries: Vec = read_dir(root, true) + .expect("list with clone hints") + .collect::>() + .expect("entries"); + clone_entries.sort_by(|left, right| left.name.cmp(&right.name)); + let clone_fields: Vec = + clone_entries.iter().map(fields).collect(); + assert_eq!(clone_fields, stated); let forked = listed.iter().find(|leaf| leaf.0 == "forked").unwrap(); assert_eq!(forked.2, 4, "the data fork's length"); assert!(forked.3 >= 20_000, "the fork's blocks: {}", forked.3); @@ -617,7 +691,274 @@ mod tests { #[test] fn a_missing_directory_is_an_error() { let temp = TempDir::new().expect("tempdir"); - let error = read_dir(&temp.path().join("gone")).expect_err("missing"); + let error = + read_dir(&temp.path().join("gone"), false).expect_err("missing"); assert_eq!(error.kind(), io::ErrorKind::NotFound); } } + +const fn clone_attrs(common: u32, file: u32, extended: u32) -> libc::attrlist { + libc::attrlist { + bitmapcount: libc::ATTR_BIT_MAP_COUNT, + reserved: 0, + commonattr: RETURNED | common, + volattr: 0, + dirattr: 0, + fileattr: file, + forkattr: extended, + } +} + +fn private_info(path: &Path, device: u64, inode: u64, id: u64) -> CloneInfo { + let mut info = CloneInfo { + device, + id, + ..CloneInfo::default() + }; + let Ok(path) = CString::new(path.as_os_str().as_bytes()) else { + return info; + }; + // Re-read identity alongside private size: a replacement between the + // listing/stat and this call must not receive the old file's estimate. + let mut request = clone_attrs( + libc::ATTR_CMN_FILEID, + libc::ATTR_FILE_DATAALLOCSIZE, + libc::ATTR_CMNEXT_PRIVATESIZE | HINTS | REFCNT, + ); + let mut buffer = [0_u8; 96]; + loop { + // SAFETY: valid NUL-terminated path, initialized ABI request and + // writable buffer. NOFOLLOW never asks a symlink target for data. + let result = unsafe { + libc::getattrlist( + path.as_ptr(), + (&raw mut request).cast(), + buffer.as_mut_ptr().cast(), + buffer.len(), + EXTENDED | libc::FSOPT_NOFOLLOW | libc::FSOPT_PACK_INVAL_ATTRS, + ) + }; + if result == 0 { + break; + } + if io::Error::last_os_error().raw_os_error() == Some(libc::EINVAL) + && request.forkattr & REFCNT != 0 + { + request.forkattr &= !REFCNT; + continue; + } + return info; + } + let Some(length) = clone_read_u32(&buffer, 0) else { + return info; + }; + let Some(buffer) = buffer.get(..length as usize) else { + return info; + }; + let common = clone_read_u32(buffer, 4).unwrap_or(0); + let files = clone_read_u32(buffer, 16).unwrap_or(0); + let extended = clone_read_u32(buffer, 20).unwrap_or(0); + if common & libc::ATTR_CMN_FILEID == 0 + || clone_read_u64(buffer, 24) != Some(inode) + { + return info; + } + // Packed order: fileid, dataallocsize, privatesize, cloneid, flags, refcnt. + if extended & libc::ATTR_CMNEXT_CLONEID == 0 + || clone_read_u64(buffer, 48) != Some(id) + { + return info; + } + if extended & libc::ATTR_CMNEXT_PRIVATESIZE != 0 { + info.private_bytes = clone_read_u64(buffer, 40); + } + if files & libc::ATTR_FILE_DATAALLOCSIZE != 0 { + info.data_bytes = clone_read_u64(buffer, 32).unwrap_or(0); + } + if extended & REFCNT != 0 { + info.references = clone_read_u32(buffer, 64).unwrap_or(0); + } + info +} + +fn clone_read_u32(bytes: &[u8], at: usize) -> Option { + Some(u32::from_ne_bytes(bytes.get(at..at + 4)?.try_into().ok()?)) +} +fn clone_read_u64(bytes: &[u8], at: usize) -> Option { + Some(u64::from_ne_bytes(bytes.get(at..at + 8)?.try_into().ok()?)) +} + +/// Exceptional path for followed symlinks; ordinary entries use bulk hints. +pub fn clone_info(path: &Path, meta: &Metadata) -> Option> { + let path_c = CString::new(path.as_os_str().as_bytes()).ok()?; + let mut request = clone_attrs(0, 0, HINTS); + let mut buffer = [0_u8; 40]; + // SAFETY: valid path, ABI request and writable buffer of the stated size. + let result = unsafe { + libc::getattrlist( + path_c.as_ptr(), + (&raw mut request).cast(), + buffer.as_mut_ptr().cast(), + buffer.len(), + EXTENDED | libc::FSOPT_NOFOLLOW | libc::FSOPT_PACK_INVAL_ATTRS, + ) + }; + if result != 0 + || clone_read_u32(&buffer, 20)? & HINTS != HINTS + || clone_read_u64(&buffer, 32)? & MAY_SHARE == 0 + { + return None; + } + Some(Box::new(private_info( + path, + meta.dev(), + meta.ino(), + clone_read_u64(&buffer, 24)?, + ))) +} + +#[cfg(test)] +mod clone_tests { + use super::*; + use crate::scan::{ScanOptions, scan}; + use std::fs; + use std::io::{Seek, SeekFrom, Write}; + use std::process::Command; + + fn fixture() -> tempfile::TempDir { + let temp = tempfile::tempdir().expect("tempdir"); + let source = temp.path().join("original"); + fs::write(&source, vec![0x5a; 8 * 1024 * 1024]).expect("write"); + assert!( + Command::new("cp") + .arg("-c") + .arg(&source) + .arg(temp.path().join("clone")) + .status() + .expect("cp -c") + .success() + ); + temp + } + + #[test] + fn real_clones_keep_listed_bytes_and_report_sharing() { + let temp = fixture(); + let tree = scan(temp.path(), ScanOptions::default()).expect("scan"); + assert_eq!(tree.bytes, 16 * 1024 * 1024); + assert_eq!(tree.sharing().files, 2, "{tree:#?}"); + assert_eq!(tree.sharing().bytes, tree.bytes); + assert_eq!(tree.sharing().duplicate_bytes, 8 * 1024 * 1024); + assert_eq!(tree.sharing().private_bytes, 0); + assert_eq!(tree.sharing().unknown_files, 0); + let baseline = scan( + temp.path(), + ScanOptions { + apfs_clone_metadata: false, + ..ScanOptions::default() + }, + ) + .expect("scan without clone metadata"); + assert_eq!(baseline.bytes, tree.bytes); + assert_eq!(baseline.sharing().files, 0); + let apparent = scan( + temp.path(), + ScanOptions { + apparent_size: true, + ..ScanOptions::default() + }, + ) + .expect("apparent scan"); + assert_eq!(apparent.sharing().files, 0); + } + + #[test] + fn modified_clones_keep_private_bytes_and_hardlinks_are_not_clones() { + let temp = fixture(); + let clone = temp.path().join("clone"); + let mut file = fs::OpenOptions::new() + .write(true) + .open(&clone) + .expect("open"); + file.seek(SeekFrom::Start(1024 * 1024)).expect("seek"); + file.write_all(&[1; 4096]).expect("write"); + file.sync_all().expect("sync"); + fs::hard_link(&clone, temp.path().join("linked")).expect("hardlink"); + std::os::unix::fs::symlink(&clone, temp.path().join("symlink")) + .expect("symlink"); + for follow_links in [false, true] { + let tree = scan( + temp.path(), + ScanOptions { + follow_links, + ..ScanOptions::default() + }, + ) + .expect("scan"); + assert_eq!(tree.sharing().files, 2, "{tree:#?}"); + assert!(tree.sharing().private_bytes >= 4096); + assert!(tree.sharing().private_bytes < tree.sharing().bytes); + assert_eq!(tree.sharing().duplicate_bytes, 0); + } + } + + #[test] + fn unreadable_private_metadata_is_unknown_not_full_allocation() { + let temp = fixture(); + let source = temp.path().join("original"); + let meta = fs::metadata(&source).expect("metadata"); + let info = private_info( + &temp.path().join("missing"), + meta.dev(), + meta.ino(), + 42, + ); + assert_eq!(info.private_bytes, None); + assert_eq!(info.references, 0); + } + + #[test] + fn malformed_bulk_names_and_records_are_rejected() { + assert!( + Cursor { + record: &[], + at: 24 + } + .name() + .is_err() + ); + let mut record = vec![0_u8; 64]; + record[4..8] + .copy_from_slice(&(RETURNED | libc::ATTR_CMN_NAME).to_ne_bytes()); + record[24..28].copy_from_slice(&8_u32.to_ne_bytes()); + record[28..32].copy_from_slice(&3_u32.to_ne_bytes()); + record[32..35].copy_from_slice(b"ok\0"); + assert_eq!( + Cursor { + record: &record, + at: 24 + } + .name() + .expect("name"), + b"ok" + ); + record[32..35].copy_from_slice(b"..\0"); + assert!( + Cursor { + record: &record, + at: 24 + } + .name() + .is_err() + ); + record[24..28].copy_from_slice(&u32::MAX.to_ne_bytes()); + assert!( + Cursor { + record: &record, + at: 24 + } + .name() + .is_err() + ); + } +} diff --git a/crates/disktree-core/src/removal.rs b/crates/disktree-core/src/removal.rs index 60277529..1dd4ab24 100644 --- a/crates/disktree-core/src/removal.rs +++ b/crates/disktree-core/src/removal.rs @@ -82,6 +82,11 @@ pub struct Plan { reclaim: u64, /// Bytes the targets free on other volumes; see [`Self::foreign`]. foreign: u64, + /// Attribution from the same mount-table read that judged the guards. + attribution: Vec, + /// Clone estimates are separate from listed allocation and its volumes. + reclaimable_bytes: Option, + reclaim_estimate: Option, } impl Plan { @@ -97,7 +102,35 @@ impl Plan { /// disk mounted under the root can be marked — gives its space back /// there, so projecting it here would lend one disk another's bytes. pub const fn reclaim(&self) -> u64 { - self.reclaim + match self.reclaim_estimate { + Some(bytes) => bytes, + None => self.reclaim, + } + } + + /// Estimated savings across all targets, independent of the volume meter. + pub fn reclaimable_bytes(&self) -> u64 { + self.reclaimable_bytes.unwrap_or_else(|| self.bytes()) + } + + /// Apply scan-derived savings without lending another volume's space to + /// this one's meter. Missing nodes contribute no unmeasured saving. + pub fn estimate_reclaim( + &mut self, + estimate: impl Fn(&Target) -> Option, + ) { + let mut total = 0_u64; + let mut scanned = 0_u64; + for (target, attribution) in self.targets.iter().zip(&self.attribution) + { + let bytes = estimate(target).unwrap_or(0).min(target.bytes); + total = total.saturating_add(bytes); + if *attribution == crate::space::Attribution::Scanned { + scanned = scanned.saturating_add(bytes); + } + } + self.reclaimable_bytes = Some(total); + self.reclaim_estimate = Some(scanned); } /// Bytes the targets free on other volumes, which the meter leaves out. @@ -185,11 +218,13 @@ fn plan_against(targets: &[Target], root: &Path, mounts: &MountTable) -> Plan { // the scanned volume may be shown against its free space. for target in &plan.targets { let bytes = target.bytes; - match crate::space::attribution( + let attribution = crate::space::attribution( mounts.mounts.as_deref(), &plan.root, &target.path, - ) { + ); + plan.attribution.push(attribution); + match attribution { crate::space::Attribution::Scanned => plan.reclaim += bytes, crate::space::Attribution::Other => plan.foreign += bytes, crate::space::Attribution::Unknown => {} @@ -1802,6 +1837,16 @@ mod tests { assert_eq!(plan.reclaim(), 100, "the Btrfs subvolume is this disk"); assert_eq!(plan.foreign(), 600, "another disk, and a snapshot"); assert_eq!(plan.unattributed(), 0); + let mut estimated = plan; + estimated.estimate_reclaim(|target| Some(target.bytes / 2)); + assert_eq!(estimated.reclaimable_bytes(), 350); + assert_eq!(estimated.reclaim(), 50, "only this volume's estimate"); + assert_eq!(estimated.foreign(), 600, "listed foreign allocation"); + assert_eq!( + estimated.unattributed(), + 0, + "sharing is not an unknown volume" + ); // A mark inside another is removed with it, so its bytes are counted // once here as they are in the total. @@ -1829,6 +1874,15 @@ mod tests { assert_eq!(plan.reclaim(), 0); assert_eq!(plan.foreign(), 0); assert_eq!(plan.unattributed(), 50, "no saving may be claimed"); + let mut estimated = plan; + estimated.estimate_reclaim(|_| Some(25)); + assert_eq!(estimated.reclaimable_bytes(), 25); + assert_eq!( + estimated.reclaim(), + 0, + "unplaced estimates stay off the meter" + ); + assert_eq!(estimated.unattributed(), 50); } #[test] diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rs index 693253a7..88d326a4 100644 --- a/crates/disktree-core/src/scan.rs +++ b/crates/disktree-core/src/scan.rs @@ -42,6 +42,8 @@ pub struct ScanOptions { /// what `ls -l` shows; blocks are what the volume actually spends, which is /// what a disk-space tool normally wants. pub apparent_size: bool, + /// Read APFS clone hints and private sizes on macOS. + pub apfs_clone_metadata: bool, /// Follow symlinks. Off by default: a home directory is full of links, and /// following them double-counts. pub follow_links: bool, @@ -69,6 +71,7 @@ impl Default for ScanOptions { fn default() -> Self { Self { apparent_size: false, + apfs_clone_metadata: true, follow_links: false, include_hidden: true, one_filesystem: true, @@ -353,7 +356,7 @@ impl WalkContext { } Listing::Leaf(kind) => kind, }; - match entry.facts(self.options.apparent_size) { + match entry.facts(dir, self.options.apparent_size) { Ok(facts) => self.leaf(entry.take_name(), kind, &facts), Err(error) => { self.progress.record_error(&entry.entry_path(dir), &error); @@ -477,6 +480,12 @@ impl WalkContext { // What the link leads to, so a file that is also reached directly is // charged once. facts.identity = identity_of(path, &meta); + #[cfg(target_os = "macos")] + if self.options.apfs_clone_metadata && !self.options.apparent_size { + facts.clone_info = fs::canonicalize(path) + .ok() + .and_then(|target| crate::macos::clone_info(&target, &meta)); + } self.leaf(name, kind_of(&meta, meta.file_type()), &facts) } @@ -512,7 +521,7 @@ trait Listed { fn take_name(&mut self) -> Box; fn listing(&self) -> io::Result; /// Size, identity and age of a leaf. - fn facts(&self, apparent_size: bool) -> io::Result; + fn facts(&self, _dir: &Path, apparent_size: bool) -> io::Result; /// What a directory is, read before the walk enters it. fn directory(&self) -> io::Result; /// Hidden by an attribute rather than by a leading dot: on Windows, @@ -555,6 +564,7 @@ struct Facts { /// The file may have another name the walk could meet: its identity is /// worth keeping for hardlink de-duplication. shared: bool, + clone_info: Option>, } impl Facts { @@ -564,6 +574,7 @@ impl Facts { identity: file_identity(meta), modified: modified_seconds(meta), shared: shares_inode(meta), + clone_info: None, } } } @@ -602,10 +613,9 @@ impl Listed for Named { }) } - fn facts(&self, apparent_size: bool) -> io::Result { - self.entry - .metadata() - .map(|meta| Facts::of(&meta, apparent_size)) + fn facts(&self, _dir: &Path, apparent_size: bool) -> io::Result { + let meta = self.entry.metadata()?; + Ok(Facts::of(&meta, apparent_size)) } fn directory(&self) -> io::Result { @@ -639,7 +649,7 @@ impl Listed for crate::macos::Entry { }) } - fn facts(&self, apparent_size: bool) -> io::Result { + fn facts(&self, dir: &Path, apparent_size: bool) -> io::Result { Ok(Facts { size: if apparent_size { self.apparent() @@ -649,6 +659,11 @@ impl Listed for crate::macos::Entry { identity: Some(self.identity()), modified: self.modified(), shared: self.shared(), + clone_info: if apparent_size { + None + } else { + self.clone_info(dir) + }, }) } @@ -682,7 +697,7 @@ impl Listed for crate::windows::Entry { }) } - fn facts(&self, apparent_size: bool) -> io::Result { + fn facts(&self, _dir: &Path, apparent_size: bool) -> io::Result { Ok(Facts { size: if apparent_size { self.apparent() @@ -694,6 +709,7 @@ impl Listed for crate::windows::Entry { // The listing has no link count; a file id is free here, so // every file keeps one. shared: true, + clone_info: None, }) } @@ -717,8 +733,10 @@ impl Listed for crate::windows::Entry { fn list( path: &Path, _volume: Option, + _clones: bool, ) -> io::Result>> { - Ok(fs::read_dir(path)?.map(|entry| { + let entries = fs::read_dir(path)?; + Ok(entries.map(|entry| { entry.map(|entry| Named { name: entry.file_name().into_string().map_or_else( |raw| raw.to_string_lossy().into(), @@ -733,14 +751,16 @@ fn list( fn list( path: &Path, _volume: Option, + clones: bool, ) -> io::Result { - crate::macos::read_dir(path) + crate::macos::read_dir(path, clones) } #[cfg(windows)] fn list( path: &Path, volume: Option, + _clones: bool, ) -> io::Result { crate::windows::read_dir(path, volume) } @@ -813,6 +833,8 @@ impl PendingDir { files: 0, dirs: 1, inode: None, + clone_info: None, + sharing: None, read_error: self.read_error.load(Ordering::Relaxed), modified: 0, category: crate::classify::Category::Other, @@ -1076,7 +1098,11 @@ fn walk(dir: &Arc, context: &WalkContext) -> Vec> { let mut leaves: Vec = Vec::new(); let mut tally = Tally::default(); - match list(&dir.path, context.volume.get().copied()) { + match list( + &dir.path, + context.volume.get().copied(), + context.options.apfs_clone_metadata && !context.options.apparent_size, + ) { Ok(entries) => { for entry in entries { if context.cancelled() { @@ -1205,6 +1231,7 @@ fn leaf_node( .and_then(|(device, file)| Some((device, NonZeroU64::new(file)?))); } node.modified = facts.modified; + node.clone_info.clone_from(&facts.clone_info); node } diff --git a/crates/disktree-core/src/sharing.rs b/crates/disktree-core/src/sharing.rs new file mode 100644 index 00000000..ebded10c --- /dev/null +++ b/crates/disktree-core/src/sharing.rs @@ -0,0 +1,213 @@ +//! APFS sharing, reported separately from the treemap's allocated bytes. +//! +//! Ported from huntharo/diskhound's clone accounting (PR #4). Clone IDs +//! describe full data-fork clones, not every shared extent of modified files. +//! The source license is retained in `licenses/diskhound-MIT.txt`. + +use rustc_hash::FxHashMap; + +use crate::tree::Node; + +/// Metadata retained only for files that may share allocation. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CloneInfo { + pub device: u64, + pub id: u64, + pub references: u32, + pub private_bytes: Option, + /// Clone identity describes the data fork, not resource forks or xattrs. + pub data_bytes: u64, +} + +/// Derived figures; none changes `Node::bytes` or a tile's weight. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Sharing { + pub files: u64, + pub bytes: u64, + pub private_bytes: u64, + pub duplicate_bytes: u64, + pub unknown_files: u64, + /// More than one million distinct groups: repeated-byte reporting is partial. + pub groups_truncated: bool, +} + +impl Sharing { + /// Conservative estimate: shared extents are excluded even if all their + /// references are selected together. Snapshots can retain other blocks. + pub const fn reclaimable(self, listed: u64) -> u64 { + listed + .saturating_sub(self.bytes) + .saturating_add(self.private_bytes) + } +} + +#[derive(Debug)] +struct Group { + bytes: u64, + count: u64, +} + +impl Group { + const fn duplicates(&self) -> u64 { + self.bytes.saturating_mul(self.count - 1) + } +} + +type Groups = FxHashMap<(u64, u64), Group>; +// Bound temporary grouping memory on volumes with millions of clone IDs. +const MAX_CLONE_GROUPS: usize = 1_000_000; + +/// Small-to-large merging avoids walking every subtree for every ancestor. +/// Hardlink duplicates already have zero weight when this pass runs. +pub(crate) fn refresh(node: &mut Node) { + collect(node, MAX_CLONE_GROUPS); +} + +fn collect(node: &mut Node, limit: usize) -> Groups { + let mut groups = Groups::default(); + let mut summary = Sharing::default(); + if let Some(info) = node.clone_info.as_deref().filter(|_| node.bytes > 0) { + summary.files = 1; + summary.bytes = node.bytes; + summary.private_bytes = info.private_bytes.unwrap_or(0).min(node.bytes); + summary.unknown_files = u64::from(info.private_bytes.is_none()); + if info.id != 0 && info.references >= 2 { + groups.insert( + (info.device, info.id), + Group { + bytes: info.data_bytes.min(node.bytes), + count: 1, + }, + ); + } + } + for child in &mut node.children { + let mut child_groups = collect(child, limit); + summary.duplicate_bytes = summary + .duplicate_bytes + .saturating_add(child.sharing().duplicate_bytes); + summary.groups_truncated |= child.sharing().groups_truncated; + summary.files += child.sharing().files; + summary.bytes = summary.bytes.saturating_add(child.sharing().bytes); + summary.private_bytes = summary + .private_bytes + .saturating_add(child.sharing().private_bytes); + summary.unknown_files += child.sharing().unknown_files; + if groups.len() < child_groups.len() { + std::mem::swap(&mut groups, &mut child_groups); + } + for (key, group) in child_groups { + if groups.len() >= limit && !groups.contains_key(&key) { + summary.groups_truncated = true; + continue; + } + groups + .entry(key) + .and_modify(|old| { + // A scan is not atomic. Use the smaller observation when + // a file changes while its clone group is being measured. + let before = + old.duplicates().saturating_add(group.duplicates()); + old.bytes = old.bytes.min(group.bytes); + old.count += group.count; + summary.duplicate_bytes = summary + .duplicate_bytes + .saturating_sub(before) + .saturating_add(old.duplicates()); + }) + .or_insert(group); + } + } + node.sharing = (summary.files > 0).then(|| Box::new(summary)); + groups +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tree::{Metric, NodeKind, Seen, aggregate, aggregate_deduped}; + + fn clone(name: &str, device: u64, id: u64, private: Option) -> Node { + let mut node = Node::entry(name, NodeKind::File, 100); + node.clone_info = Some(Box::new(CloneInfo { + device, + id, + references: 2, + private_bytes: private, + data_bytes: 80, + })); + node + } + + #[test] + fn group_limit_marks_the_summary_partial_without_losing_private_bytes() { + let mut root = Node::directory("root"); + root.children = + vec![clone("a", 1, 1, Some(20)), clone("b", 1, 2, Some(20))]; + aggregate(&mut root, Metric::Bytes); + collect(&mut root, 1); + assert!(root.sharing().groups_truncated); + assert_eq!(root.sharing().private_bytes, 40); + } + + #[test] + fn groups_are_volume_scoped_and_only_data_forks_are_duplicated() { + let mut root = Node::directory("root"); + root.children = vec![ + clone("a", 1, 42, Some(20)), + clone("b", 1, 42, Some(20)), + clone("c", 2, 42, Some(20)), + ]; + aggregate(&mut root, Metric::Bytes); + assert_eq!(root.bytes, 300); + assert_eq!(root.sharing().duplicate_bytes, 80); + assert_eq!(root.sharing().reclaimable(root.bytes), 60); + aggregate(&mut root, Metric::Files); + assert_eq!(root.sharing().duplicate_bytes, 80); + } + + #[test] + fn unknown_and_oversized_private_values_stay_conservative() { + let mut root = Node::directory("root"); + root.children = vec![ + clone("a", 1, 1, None), + clone("b", 1, 2, Some(200)), + Node::entry("ordinary", NodeKind::File, 30), + ]; + aggregate(&mut root, Metric::Bytes); + assert_eq!(root.sharing().unknown_files, 1); + assert_eq!(root.sharing().reclaimable(root.bytes), 130); + } + + #[test] + fn deduplicated_hardlinks_do_not_inflate_sharing() { + let mut root = Node::directory("root"); + root.children = + vec![clone("a", 1, 1, Some(0)), clone("b", 1, 1, Some(0))]; + for node in &mut root.children { + node.inode = + Some((1, std::num::NonZeroU64::new(123).expect("inode"))); + } + aggregate_deduped(&mut root, Metric::Bytes, &Seen::new()); + assert_eq!(root.sharing().files, 1); + assert_eq!(root.sharing().bytes, 100); + assert_eq!(root.sharing().duplicate_bytes, 0); + } + + #[test] + fn clone_groups_merge_across_directories_and_refresh_after_removal() { + let mut root = Node::directory("root"); + for name in ["a", "b"] { + let mut dir = Node::directory(name); + dir.children.push(clone("file", 1, 1, Some(0))); + root.children.push(dir); + } + aggregate(&mut root, Metric::Bytes); + assert_eq!(root.sharing().duplicate_bytes, 80); + assert_eq!(root.children[0].sharing().duplicate_bytes, 0); + root.children.pop(); + aggregate(&mut root, Metric::Bytes); + assert_eq!(root.sharing().duplicate_bytes, 0); + assert_eq!(root.sharing().files, 1); + } +} diff --git a/crates/disktree-core/src/tree.rs b/crates/disktree-core/src/tree.rs index 6991a86a..298f53ea 100644 --- a/crates/disktree-core/src/tree.rs +++ b/crates/disktree-core/src/tree.rs @@ -75,6 +75,8 @@ pub struct Node { /// `(device, inode)` for files, used to de-duplicate hardlinks. No file /// has inode zero, which lets `None` cost nothing. pub inode: Option<(u64, NonZeroU64)>, + pub clone_info: Option>, + pub sharing: Option>, /// The directory could not be read; its contents are unknown. pub read_error: bool, /// Newest write time at or beneath this node, in Unix seconds; `0` when @@ -103,6 +105,8 @@ impl Node { files: 0, dirs: 1, inode: None, + clone_info: None, + sharing: None, read_error: false, modified: 0, category: Category::Other, @@ -124,6 +128,8 @@ impl Node { files: u64::from(kind == NodeKind::File), dirs: 0, inode: None, + clone_info: None, + sharing: None, read_error: false, modified: 0, category: Category::Other, @@ -132,6 +138,11 @@ impl Node { } } + /// Sparse: ordinary files do not pay for five additional byte counters. + pub fn sharing(&self) -> crate::sharing::Sharing { + self.sharing.as_deref().copied().unwrap_or_default() + } + pub const fn is_dir(&self) -> bool { self.kind.is_dir() } @@ -265,6 +276,7 @@ impl Node { /// it follows without a second pass. pub fn aggregate(node: &mut Node, metric: Metric) { aggregate_at(node, metric, 0, None); + crate::sharing::refresh(node); } /// [`aggregate`], charging a hardlinked file once: a leaf whose identity @@ -273,6 +285,7 @@ pub fn aggregate(node: &mut Node, metric: Metric) { /// can split it differently between folders; the totals are the same. pub(crate) fn aggregate_deduped(node: &mut Node, metric: Metric, seen: &Seen) { aggregate_at(node, metric, 0, Some(seen)); + crate::sharing::refresh(node); } /// Identities a finish pass has met. @@ -432,7 +445,8 @@ mod tests { #[test] #[cfg(target_pointer_width = "64")] fn a_node_stays_small() { - assert_eq!(size_of::(), 88); + // Two optional pointers keep clone data sparse on ordinary nodes. + assert_eq!(size_of::(), 104); } #[test] diff --git a/licenses/diskhound-MIT.txt b/licenses/diskhound-MIT.txt new file mode 100644 index 00000000..def762cc --- /dev/null +++ b/licenses/diskhound-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Thomas Zarebczan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 4ed80d5abc3910a49596ea25946a2b48a831207c Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 13:57:05 -0400 Subject: [PATCH 02/19] test: expose quadratic mark resolution and removal budgets --- crates/disktree-app/src/marks.rs | 68 +++++++++++++++++-- crates/disktree-app/src/state.rs | 13 ++-- crates/disktree-app/src/tests.rs | 66 ++++++++++++++++++ crates/disktree-core/src/removal.rs | 51 ++++++++++++-- docs/performance/mark-resolution/red-core.log | 33 +++++++++ docs/performance/mark-resolution/red.log | 32 +++++++++ 6 files changed, 250 insertions(+), 13 deletions(-) create mode 100644 docs/performance/mark-resolution/red-core.log create mode 100644 docs/performance/mark-resolution/red.log diff --git a/crates/disktree-app/src/marks.rs b/crates/disktree-app/src/marks.rs index 89141ae3..6dc2844a 100644 --- a/crates/disktree-app/src/marks.rs +++ b/crates/disktree-app/src/marks.rs @@ -10,6 +10,25 @@ use disktree_core::removal::Target; use disktree_core::tree::{Metric, Node}; use rustc_hash::FxHashSet; +#[cfg(test)] +pub(crate) mod work { + use std::cell::Cell; + + thread_local! { + pub static RESOLUTION: Cell = const { Cell::new(0) }; + pub static FRAMES: Cell = const { Cell::new(0) }; + } + + pub fn step() { + RESOLUTION.set(RESOLUTION.get() + 1); + } + + pub fn reset() { + RESOLUTION.set(0); + FRAMES.set(0); + } +} + /// Marked paths, in the order they were marked. #[derive(Debug, Default)] pub struct Marks { @@ -64,6 +83,8 @@ impl Marks { pub fn refresh(&mut self, root_path: &Path, root: &Node, metric: Metric) { let mut resolved = Vec::with_capacity(self.items.len()); for item in &self.items { + #[cfg(test)] + work::step(); match find(root_path, root, &item.path) { Some(node) => resolved.push(Target { bytes: node.value(metric), @@ -92,10 +113,11 @@ pub fn find<'a>( let mut node = root; for component in relative.components() { let name = component.as_os_str().to_string_lossy(); - node = node - .children - .iter() - .find(|child| child.name.as_ref() == name)?; + node = node.children.iter().find(|child| { + #[cfg(test)] + work::step(); + child.name.as_ref() == name + })?; } Some(node) } @@ -149,6 +171,44 @@ mod tests { } } + #[test] + fn refresh_resolution_budget() { + let measure = |count| { + let root_path = std::env::temp_dir().join("marks-budget"); + let mut root = Node::directory("root"); + let mut marks = Marks::default(); + for index in 0..count { + let name = format!("folder-{index:05}"); + let mut dir = Node::directory(name.as_str()); + dir.children.push(file("payload", 1)); + root.children.push(dir); + marks.toggle(Target { + path: root_path.join(name).join("payload"), + bytes: 0, + is_dir: false, + hidden: false, + }); + } + work::reset(); + let start = std::time::Instant::now(); + for _ in 0..4 { + marks.refresh(&root_path, &root, Metric::Bytes); + } + let elapsed = start.elapsed(); + let operations = work::RESOLUTION.get(); + assert!(marks.items().iter().all(|item| item.bytes == 1)); + eprintln!( + "refresh M={count} N={} R=4 work={operations} wall={elapsed:?}", + count * 2 + ); + operations + }; + let small = measure(128); + let large = measure(1024); + assert!(large <= small * 16, "8x marks/folders: {small} -> {large}"); + assert!(large <= 1024 * 4 * 16, "absolute work cap: {large}"); + } + #[test] fn toggling_marks_and_unmarks() { let mut marks = Marks::default(); diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs index 02ba8b49..4667adf9 100644 --- a/crates/disktree-app/src/state.rs +++ b/crates/disktree-app/src/state.rs @@ -1785,6 +1785,8 @@ impl Disktree { /// to draw, and the marked/hidden/selected state of every tile is decided /// here, where the tree and the marks are both at hand. pub fn prepare(&mut self) -> Mosaic { + #[cfg(test)] + crate::marks::work::FRAMES.set(crate::marks::work::FRAMES.get() + 1); let metric = self.options.metric; let view = self.view; let hovered = self.hovered.clone(); @@ -1794,6 +1796,8 @@ impl Disktree { // rather than building a path for every tile. let mut marked: FxHashSet> = FxHashSet::default(); for item in self.marks.items() { + #[cfg(test)] + crate::marks::work::step(); if let Some(crumbs) = self.crumbs_for_path(&item.path) { marked.insert(crumbs); } @@ -1920,10 +1924,11 @@ impl Disktree { let mut crumbs = Vec::new(); for component in relative.components() { let name = component.as_os_str().to_string_lossy(); - let index = node - .children - .iter() - .position(|child| child.name.as_ref() == name)?; + let index = node.children.iter().position(|child| { + #[cfg(test)] + crate::marks::work::step(); + child.name.as_ref() == name + })?; crumbs.push(index); node = node.child(index)?; } diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index 142c33e4..f8b499ae 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -1915,3 +1915,69 @@ fn the_power_menu_offers_only_what_the_cpus_can_tell_apart( draw(cx); assert!(read(&view, cx, |app| app.power_menu.is_none())); } + +/// Exercise prepare through actual window draws, including repeated renders. +#[gpui_kit::test] +fn marked_window_resolution_budget(cx: &mut TestAppContext) { + use crate::marks::work; + use disktree_core::removal::Target; + use disktree_core::tree::{Metric, Node, NodeKind, aggregate}; + + cx.update(gpui_omarchy::init); + let mut measure = |count| { + let temp = tempfile::tempdir().expect("tempdir"); + let root_path = temp.path().to_path_buf(); + let mut tree = Node::directory("root"); + for index in 0..count { + let mut folder = Node::directory(format!("folder-{index:05}")); + folder + .children + .push(Node::entry("payload", NodeKind::File, 1)); + tree.children.push(folder); + } + aggregate(&mut tree, Metric::Bytes); + let (view, window) = cx.add_window_view(move |_, cx| { + Disktree::with_tree(root_path.clone(), tree, options(), 3, cx) + }); + update(&view, window, |app, _| { + for index in 0..count { + app.marks.toggle(Target { + path: temp + .path() + .join(format!("folder-{index:05}")) + .join("payload"), + bytes: 1, + is_dir: false, + hidden: false, + }); + } + }); + work::reset(); + let start = std::time::Instant::now(); + for _ in 0..4 { + update(&view, window, |_, cx| cx.notify()); + draw(window); + } + let elapsed = start.elapsed(); + let frames = work::FRAMES.get(); + let operations = work::RESOLUTION.get(); + window.update(|window, _| window.remove_window()); + assert!(frames >= 4, "budget must cover actual renders: {frames}"); + assert!(operations > 0, "resolution instrumentation must run"); + eprintln!( + "window M={count} N={} draws=4 prepares={frames} work={operations} wall={elapsed:?}", + count * 2 + ); + (operations, frames) + }; + let (small, small_frames) = measure(128); + let (large, large_frames) = measure(1024); + assert!( + large * small_frames <= small * large_frames * 16, + "8x marks/folders per frame: {small}/{small_frames} -> {large}/{large_frames}" + ); + assert!( + large <= 1024 * large_frames * 16, + "absolute work cap: {large}" + ); +} diff --git a/crates/disktree-core/src/removal.rs b/crates/disktree-core/src/removal.rs index 1dd4ab24..fe2675e8 100644 --- a/crates/disktree-core/src/removal.rs +++ b/crates/disktree-core/src/removal.rs @@ -24,6 +24,13 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{self, Receiver}; use std::thread; +#[cfg(test)] +thread_local! { + static NESTING_WORK: std::cell::Cell = const { + std::cell::Cell::new(0) + }; +} + /// One path the user asked to remove. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Target { @@ -200,13 +207,18 @@ fn plan_against(targets: &[Target], root: &Path, mounts: &MountTable) -> Plan { // A path inside another target is removed with it. Keep the outer one and // report the inner one so the review screen can explain the nesting. - accepted.sort_by(|left, right| left.path.cmp(&right.path)); + accepted.sort_by(|left, right| { + #[cfg(test)] + NESTING_WORK.set(NESTING_WORK.get() + 1); + left.path.cmp(&right.path) + }); let mut outer: Vec = Vec::new(); for target in accepted { - if outer - .iter() - .any(|candidate| within(&target.path, &candidate.path)) - { + if outer.iter().any(|candidate| { + #[cfg(test)] + NESTING_WORK.set(NESTING_WORK.get() + 1); + within(&target.path, &candidate.path) + }) { plan.covered.push(target); } else { outer.push(target); @@ -1784,6 +1796,35 @@ mod tests { ); } + #[test] + fn removal_nesting_budget() { + let temp = TempDir::new().expect("tempdir"); + let measure = |count| { + let targets: Vec<_> = (0..count) + .map(|index| (index * 73) % count) + .map(|index| Target { + path: temp.path().join(format!("item-{index:05}")), + bytes: 1, + is_dir: false, + hidden: false, + }) + .collect(); + NESTING_WORK.set(0); + let start = std::time::Instant::now(); + let plan = + plan_against(&targets, temp.path(), &MountTable::default()); + let elapsed = start.elapsed(); + let operations = NESTING_WORK.get(); + assert_eq!(plan.targets.len(), count); + eprintln!("plan M={count} work={operations} wall={elapsed:?}"); + operations + }; + let small = measure(128); + let large = measure(1024); + assert!(large <= small * 16, "8x marks: {small} -> {large}"); + assert!(large <= 1024 * 32, "absolute work cap: {large}"); + } + #[test] fn a_plan_keeps_only_the_outermost_targets() { let temp = tree(); diff --git a/docs/performance/mark-resolution/red-core.log b/docs/performance/mark-resolution/red-core.log new file mode 100644 index 00000000..2227d991 --- /dev/null +++ b/docs/performance/mark-resolution/red-core.log @@ -0,0 +1,33 @@ + Compiling fastrand v2.5.0 + Compiling either v1.18.0 + Compiling log v0.4.34 + Compiling num-traits v0.2.19 + Compiling rustix v1.1.4 + Compiling once_cell v1.21.4 + Compiling errno v0.3.14 + Compiling objc2-foundation v0.3.2 + Compiling rayon v1.12.0 + Compiling trash v5.2.9 + Compiling chrono v0.4.45 + Compiling tempfile v3.27.0 + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Finished `test` profile [optimized + debuginfo] target(s) in 3.82s + Running unittests src/lib.rs (target/debug/deps/disktree_core-c3cb863045931ea5) + +running 1 test +plan M=128 work=9187 wall=9.931833ms +plan M=1024 work=537027 wall=390.640459ms + +thread 'removal::tests::removal_nesting_budget' (403531013) panicked at crates/disktree-core/src/removal.rs:1789:9: +8x marks: 9187 -> 537027 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +test removal::tests::removal_nesting_budget ... FAILED + +failures: + +failures: + removal::tests::removal_nesting_budget + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 149 filtered out; finished in 0.40s + +error: test failed, to rerun pass `-p disktree-core --lib` diff --git a/docs/performance/mark-resolution/red.log b/docs/performance/mark-resolution/red.log new file mode 100644 index 00000000..75475620 --- /dev/null +++ b/docs/performance/mark-resolution/red.log @@ -0,0 +1,32 @@ + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 5.18s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 4 tests +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +refresh M=128 N=256 R=4 work=34048 wall=364µs +refresh M=1024 N=2048 R=4 work=2107392 wall=8.818542ms + +thread 'marks::tests::refresh_resolution_budget' (403532227) panicked at crates/disktree-app/src/marks.rs:208:9: +8x marks/folders: 34048 -> 2107392 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +test marks::tests::refresh_resolution_budget ... FAILED +window M=128 N=256 draws=4 prepares=8 work=68312 wall=171.271792ms +window M=1024 N=2048 draws=4 prepares=8 work=4215000 wall=6.734163791s + +thread 'tests::marked_window_resolution_budget' (403532228) panicked at crates/disktree-app/src/tests.rs:1904:5: +8x marks/folders per frame: 68312/8 -> 4215000/8 +test tests::marked_window_resolution_budget ... FAILED + +failures: + +failures: + marks::tests::refresh_resolution_budget + tests::marked_window_resolution_budget + +test result: FAILED. 2 passed; 2 failed; 0 ignored; 0 measured; 69 filtered out; finished in 6.96s + +error: test failed, to rerun pass `-p disktree-app --bin disktree` From f95d36890aea80d0a7104c3cf38836cf2c6db6a7 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 14:04:44 -0400 Subject: [PATCH 03/19] fix: batch marked path resolution and bound removal nesting --- crates/disktree-app/src/marks.rs | 328 ++++++++++++++++++++++++---- crates/disktree-app/src/state.rs | 16 +- crates/disktree-app/src/tests.rs | 82 +++++++ crates/disktree-core/src/removal.rs | 96 ++++++-- 4 files changed, 441 insertions(+), 81 deletions(-) diff --git a/crates/disktree-app/src/marks.rs b/crates/disktree-app/src/marks.rs index 6dc2844a..76348437 100644 --- a/crates/disktree-app/src/marks.rs +++ b/crates/disktree-app/src/marks.rs @@ -5,18 +5,20 @@ //! gone) rather than silently pointing at a different node. use std::path::{Path, PathBuf}; +use std::sync::{Arc, Weak}; use disktree_core::removal::Target; use disktree_core::tree::{Metric, Node}; -use rustc_hash::FxHashSet; +use rustc_hash::{FxHashMap, FxHashSet}; #[cfg(test)] -pub(crate) mod work { +pub mod work { use std::cell::Cell; thread_local! { pub static RESOLUTION: Cell = const { Cell::new(0) }; pub static FRAMES: Cell = const { Cell::new(0) }; + pub static TILES: Cell = const { Cell::new(0) }; } pub fn step() { @@ -26,6 +28,7 @@ pub(crate) mod work { pub fn reset() { RESOLUTION.set(0); FRAMES.set(0); + TILES.set(0); } } @@ -34,6 +37,17 @@ pub(crate) mod work { pub struct Marks { items: Vec, index: FxHashSet, + resolved: Option, +} + +/// Retain only the allocation identity: a rescan must not keep the old tree +/// alive while a new one is being displayed. +#[derive(Debug)] +struct ResolvedMarks { + tree: Weak, + root_path: PathBuf, + metric: Metric, + crumbs: Arc>>, } impl Marks { @@ -57,6 +71,7 @@ impl Marks { /// /// Returns `true` when the path ended up marked. pub fn toggle(&mut self, target: Target) -> bool { + self.resolved = None; if self.index.remove(&target.path) { self.items.retain(|item| item.path != target.path); false @@ -69,24 +84,28 @@ impl Marks { pub fn remove(&mut self, path: &Path) { if self.index.remove(path) { + self.resolved = None; self.items.retain(|item| item.path != path); } } pub fn clear(&mut self) { + self.resolved = None; self.items.clear(); self.index.clear(); } - /// Re-read sizes from a freshly scanned tree and drop marks whose path no - /// longer exists, so the tally never claims space that is already gone. + /// Re-read sizes from a freshly scanned tree. Missing paths stay marked + /// with zero bytes, so they cannot claim space that is already gone. pub fn refresh(&mut self, root_path: &Path, root: &Node, metric: Metric) { + self.resolved = None; + let mut resolver = Resolver::new(root_path, root); let mut resolved = Vec::with_capacity(self.items.len()); for item in &self.items { #[cfg(test)] work::step(); - match find(root_path, root, &item.path) { - Some(node) => resolved.push(Target { + match resolver.resolve(&item.path) { + Some((_, node)) => resolved.push(Target { bytes: node.value(metric), is_dir: node.is_dir(), hidden: is_hidden(&item.path), @@ -100,26 +119,90 @@ impl Marks { } self.items = resolved; } + + /// Crumbs are display state, never removal authorization. Rebuild on every + /// mark mutation, refresh, tree replacement, root change or metric change. + pub fn crumbs( + &mut self, + root_path: &Path, + tree: &Arc, + metric: Metric, + ) -> Arc>> { + if let Some(resolved) = &self.resolved + && resolved.tree.as_ptr() == Arc::as_ptr(tree) + && resolved.root_path == root_path + && resolved.metric == metric + { + return Arc::clone(&resolved.crumbs); + } + let mut resolver = Resolver::new(root_path, tree); + let mut crumbs = FxHashSet::default(); + for item in &self.items { + #[cfg(test)] + work::step(); + if let Some((path, _)) = resolver.resolve(&item.path) { + crumbs.insert(path); + } + } + let crumbs = Arc::new(crumbs); + self.resolved = Some(ResolvedMarks { + tree: Arc::downgrade(tree), + root_path: root_path.to_path_buf(), + metric, + crumbs: Arc::clone(&crumbs), + }); + crumbs + } } -/// Walk the tree to the node at an absolute path. Path components are compared -/// one at a time, so a name containing a path separator cannot confuse it. -pub fn find<'a>( - root_path: &Path, +/// A batch of path lookups indexes each visited directory's children once. +/// Work is O(S + M × D) expected hash operations: S sibling entries across +/// visited directories, M paths and D components per path. Unvisited subtrees +/// cost nothing, and no index is stored on the millions of scan nodes. +#[derive(Debug)] +pub struct Resolver<'a> { + root_path: &'a Path, root: &'a Node, - path: &Path, -) -> Option<&'a Node> { - let relative = path.strip_prefix(root_path).ok()?; - let mut node = root; - for component in relative.components() { - let name = component.as_os_str().to_string_lossy(); - node = node.children.iter().find(|child| { + indexes: FxHashMap<*const Node, FxHashMap<&'a str, usize>>, +} + +impl<'a> Resolver<'a> { + pub fn new(root_path: &'a Path, root: &'a Node) -> Self { + Self { + root_path, + root, + indexes: FxHashMap::default(), + } + } + + /// Compare whole components with the scan's existing case-sensitive, + /// lossy name semantics. The first equal child wins, as in a linear walk. + pub fn resolve(&mut self, path: &Path) -> Option<(Vec, &'a Node)> { + let relative = path.strip_prefix(self.root_path).ok()?; + let mut node = self.root; + let mut crumbs = Vec::new(); + for component in relative.components() { #[cfg(test)] work::step(); - child.name.as_ref() == name - })?; + let name = component.as_os_str().to_string_lossy(); + let children = self + .indexes + .entry(std::ptr::from_ref(node)) + .or_insert_with(|| { + let mut children = FxHashMap::default(); + for (index, child) in node.children.iter().enumerate() { + #[cfg(test)] + work::step(); + children.entry(child.name.as_ref()).or_insert(index); + } + children + }); + let index = *children.get(name.as_ref())?; + crumbs.push(index); + node = &node.children[index]; + } + Some((crumbs, node)) } - Some(node) } /// A dotfile or dot-directory by name. @@ -209,6 +292,115 @@ mod tests { assert!(large <= 1024 * 4 * 16, "absolute work cap: {large}"); } + #[test] + fn cached_crumbs_follow_every_invalidation_boundary() { + let root_path = Path::new("/home/tobi"); + let mut root = Arc::new(tree()); + let mut marks = Marks::default(); + marks.toggle(target("/home/tobi/.cache", 0)); + let first = marks.crumbs(root_path, &root, Metric::Bytes); + assert!(first.contains(&vec![0])); + work::reset(); + assert!(Arc::ptr_eq( + &first, + &marks.crumbs(root_path, &root, Metric::Bytes) + )); + assert_eq!( + work::RESOLUTION.get(), + 0, + "unchanged frames need no lookup" + ); + assert_eq!( + Arc::strong_count(&root), + 1, + "the cache must not retain a scan" + ); + + marks.toggle(target("/home/tobi/notes.bin", 0)); + assert_eq!(marks.crumbs(root_path, &root, Metric::Bytes).len(), 2); + marks.remove(Path::new("/home/tobi/.cache")); + assert!( + marks + .crumbs(root_path, &root, Metric::Bytes) + .contains(&vec![1]) + ); + marks.clear(); + assert!(marks.crumbs(root_path, &root, Metric::Bytes).is_empty()); + marks.toggle(target("/home/tobi/notes.bin", 0)); + marks.toggle(target("/home/tobi/notes.bin", 0)); + assert!(marks.crumbs(root_path, &root, Metric::Bytes).is_empty()); + marks.toggle(target("/home/tobi/notes.bin", 0)); + + let before = marks.crumbs(root_path, &root, Metric::Bytes); + let files = marks.crumbs(root_path, &root, Metric::Files); + assert!(!Arc::ptr_eq(&before, &files), "metric changes invalidate"); + marks.refresh(root_path, &root, Metric::Files); + let refreshed = marks.crumbs(root_path, &root, Metric::Files); + assert!(!Arc::ptr_eq(&files, &refreshed), "refresh invalidates"); + assert!( + marks + .crumbs(Path::new("/elsewhere"), &root, Metric::Files) + .is_empty() + ); + assert!( + marks + .crumbs(root_path, &root, Metric::Files) + .contains(&vec![1]) + ); + + let old_tree = Arc::downgrade(&root); + let mut replacement = (*root).clone(); + replacement.children.reverse(); + root = Arc::new(replacement); + assert!(old_tree.upgrade().is_none(), "old scan is released"); + assert!( + marks + .crumbs(root_path, &root, Metric::Files) + .contains(&vec![0]) + ); + Arc::make_mut(&mut root).children.clear(); + assert!(marks.crumbs(root_path, &root, Metric::Files).is_empty()); + } + + #[test] + fn resolver_preserves_scan_name_semantics() { + let root_path = Path::new("/home/tobi"); + let mut root = tree(); + root.children.push(file("notes.bin", 200)); + let mut resolver = Resolver::new(root_path, &root); + let (crumbs, node) = resolver + .resolve(&root_path.join("notes.bin")) + .expect("file"); + assert_eq!(crumbs, [1]); + assert_eq!(node.bytes, 100, "first equal scan name wins"); + assert!(resolver.resolve(&root_path.join("NOTES.BIN")).is_none()); + assert!(resolver.resolve(&root_path.join("notes")).is_none()); + assert!( + resolver + .resolve(&root_path.join(".cache/missing")) + .is_none() + ); + assert!(resolver.resolve(root_path).expect("root").0.is_empty()); + } + + #[cfg(unix)] + #[test] + fn resolver_preserves_lossy_non_utf8_components() { + use std::os::unix::ffi::OsStrExt as _; + let root_path = Path::new("/synthetic"); + let name = std::ffi::OsStr::from_bytes(b"file-\xff"); + let mut root = Node::directory("root"); + root.children.push(file(&name.to_string_lossy(), 1)); + let mut resolver = Resolver::new(root_path, &root); + assert_eq!( + resolver + .resolve(&root_path.join(name)) + .expect("lossy scan name") + .0, + [0] + ); + } + #[test] fn toggling_marks_and_unmarks() { let mut marks = Marks::default(); @@ -246,37 +438,18 @@ mod tests { #[test] fn find_matches_whole_components_only() { let root = tree(); + let mut resolver = Resolver::new(Path::new("/home/tobi"), &root); + assert!(resolver.resolve(Path::new("/home/tobi/.cache")).is_some()); assert!( - find( - Path::new("/home/tobi"), - &root, - Path::new("/home/tobi/.cache") - ) - .is_some() - ); - assert!( - find( - Path::new("/home/tobi"), - &root, - Path::new("/home/tobi/.cache/blob.bin") - ) - .is_some() - ); - assert!( - find( - Path::new("/home/tobi"), - &root, - Path::new("/home/tobi/cache") - ) - .is_none() + resolver + .resolve(Path::new("/home/tobi/.cache/blob.bin")) + .is_some() ); + assert!(resolver.resolve(Path::new("/home/tobi/cache")).is_none()); assert!( - find( - Path::new("/elsewhere"), - &root, - Path::new("/home/tobi/notes.bin") - ) - .is_none() + Resolver::new(Path::new("/elsewhere"), &root) + .resolve(Path::new("/home/tobi/notes.bin")) + .is_none() ); } @@ -292,4 +465,63 @@ mod tests { assert_eq!(display_path(Path::new("/var/log"), Some(home)), "/var/log"); assert_eq!(display_path(Path::new("/var/log"), None), "/var/log"); } + /// Vary each input separately: files, marks, path depth and refreshes + /// must not be conflated with growing siblings and marks together. + #[test] + fn refresh_dimension_budget() { + let measure = |folders: usize, + marked: usize, + depth: usize, + repeats: usize| { + let root_path = std::env::temp_dir().join("marks-dimensions"); + let mut root = Node::directory("root"); + let mut marks = Marks::default(); + for index in 0..folders { + let name = format!("folder-{index:05}"); + let mut child = file("payload", 1); + let mut relative = PathBuf::from(&name); + for _ in 1..depth { + let mut parent = Node::directory("nested"); + parent.children.push(child); + child = parent; + relative.push("nested"); + } + let mut folder = Node::directory(name); + folder.children.push(child); + root.children.push(folder); + relative.push("payload"); + if index >= folders - marked { + marks.toggle(Target { + path: root_path.join(relative), + bytes: 0, + is_dir: false, + hidden: false, + }); + } + } + work::reset(); + let start = std::time::Instant::now(); + for _ in 0..repeats { + marks.refresh(&root_path, &root, Metric::Bytes); + } + let operations = work::RESOLUTION.get(); + assert!(marks.items().iter().all(|item| item.bytes == 1)); + eprintln!( + "dimensions folders={folders} M={marked} D={} R={repeats} work={operations} wall={:?}", + depth + 1, + start.elapsed() + ); + operations + }; + let base = measure(128, 8, 1, 1); + let more_files = measure(1024, 8, 1, 1); + assert!(more_files <= base * 16); + let more_marks = measure(1024, 64, 1, 1); + assert!(more_marks <= more_files * 16); + let more_depth = measure(128, 8, 15, 1); + assert!(more_depth <= base * 16); + let more_refreshes = measure(128, 8, 1, 8); + assert_eq!(more_refreshes, base * 8); + assert!(more_files <= 1024 * 8 * 4); + } } diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs index 4667adf9..9a398562 100644 --- a/crates/disktree-app/src/state.rs +++ b/crates/disktree-app/src/state.rs @@ -1792,16 +1792,10 @@ impl Disktree { let hovered = self.hovered.clone(); let selected = self.selected.clone(); - // Marks are paths; the mosaic thinks in crumbs. Resolve once per frame - // rather than building a path for every tile. - let mut marked: FxHashSet> = FxHashSet::default(); - for item in self.marks.items() { - #[cfg(test)] - crate::marks::work::step(); - if let Some(crumbs) = self.crumbs_for_path(&item.path) { - marked.insert(crumbs); - } - } + let marked = self.tree.as_ref().map_or_else( + || Arc::new(FxHashSet::default()), + |tree| self.marks.crumbs(&self.root_path, tree, metric), + ); let Some(tiles) = self.layout().map(<[Tile]>::to_vec) else { return Mosaic { @@ -1817,6 +1811,8 @@ impl Disktree { let mut decorations = Vec::with_capacity(tiles.len()); let mut labels = Vec::new(); for tile in &tiles { + #[cfg(test)] + crate::marks::work::TILES.set(crate::marks::work::TILES.get() + 1); let crumbs = tile.crumbs(); let node = match &tile.kind { TileKind::Node { crumbs } => self.node_at(crumbs), diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index f8b499ae..9c199af4 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -1961,6 +1961,31 @@ fn marked_window_resolution_budget(cx: &mut TestAppContext) { let elapsed = start.elapsed(); let frames = work::FRAMES.get(); let operations = work::RESOLUTION.get(); + let tiles = work::TILES.get(); + assert!(tiles > 0 && tiles <= count * 4 * frames); + work::reset(); + let warm_start = std::time::Instant::now(); + for _ in 0..4 { + update(&view, window, |_, cx| cx.notify()); + draw(window); + } + let warm_elapsed = warm_start.elapsed(); + let warm_frames = work::FRAMES.get(); + let warm_work = work::RESOLUTION.get(); + assert!(warm_frames >= 4); + assert!( + warm_work <= 64 * warm_frames, + "unchanged renders must not resolve all marks: {warm_work}" + ); + assert_eq!( + work::TILES.get() * frames, + tiles * warm_frames, + "visible tile work remains linear in frame count" + ); + eprintln!( + "warm window M={count} prepares={warm_frames} resolution={warm_work} tiles={} wall={warm_elapsed:?}", + work::TILES.get() + ); window.update(|window, _| window.remove_window()); assert!(frames >= 4, "budget must cover actual renders: {frames}"); assert!(operations > 0, "resolution instrumentation must run"); @@ -1981,3 +2006,60 @@ fn marked_window_resolution_budget(cx: &mut TestAppContext) { "absolute work cap: {large}" ); } + +#[gpui_kit::test] +fn marked_tiles_follow_metric_rescan_and_mark_changes(cx: &mut TestAppContext) { + use disktree_core::removal::Target; + use std::sync::Arc; + + fn marked_paths(app: &mut Disktree) -> Vec { + let mosaic = app.prepare(); + let tiles = app.layout().expect("layout").to_vec(); + tiles + .iter() + .zip(mosaic.tiles) + .filter_map(|(tile, deco)| { + deco.marked.then(|| app.path_at(tile.crumbs())).flatten() + }) + .collect() + } + + cx.update(gpui_omarchy::init); + let temp = fixture(); + let (view, window) = view_over(temp.path(), cx); + let junk = temp.path().join("junk"); + update(&view, window, |app, cx| { + app.marks.toggle(Target { + path: junk.clone(), + bytes: 300_000, + is_dir: true, + hidden: false, + }); + cx.notify(); + }); + draw(window); + assert_eq!( + update(&view, window, |app, _| marked_paths(app)), + std::slice::from_ref(&junk) + ); + update(&view, window, Disktree::toggle_metric); + draw(window); + assert_eq!( + update(&view, window, |app, _| marked_paths(app)), + std::slice::from_ref(&junk) + ); + update(&view, window, |app, cx| { + let mut tree = (**app.tree.as_ref().expect("tree")).clone(); + tree.children.reverse(); + app.tree = Some(Arc::new(tree)); + cx.notify(); + }); + draw(window); + assert_eq!( + update(&view, window, |app, _| marked_paths(app)), + std::slice::from_ref(&junk) + ); + update(&view, window, |app, cx| app.unmark(&junk, cx)); + draw(window); + assert!(update(&view, window, |app, _| marked_paths(app)).is_empty()); +} diff --git a/crates/disktree-core/src/removal.rs b/crates/disktree-core/src/removal.rs index fe2675e8..df02e984 100644 --- a/crates/disktree-core/src/removal.rs +++ b/crates/disktree-core/src/removal.rs @@ -176,7 +176,7 @@ fn plan_against(targets: &[Target], root: &Path, mounts: &MountTable) -> Plan { root: root.clone(), ..Plan::default() }; - let mut accepted: Vec = Vec::new(); + let mut accepted: Vec<(PathBuf, Target)> = Vec::new(); if targets.is_empty() { return plan; @@ -199,32 +199,45 @@ fn plan_against(targets: &[Target], root: &Path, mounts: &MountTable) -> Plan { }); continue; } - accepted.push(Target { - path, - ..target.clone() - }); + accepted.push(( + guard_key(&path), + Target { + path, + ..target.clone() + }, + )); } - // A path inside another target is removed with it. Keep the outer one and - // report the inner one so the review screen can explain the nesting. + // Descendants are contiguous only when sorted by the same keys the + // guards compare (including case and macOS Data-volume aliases). Once + // sorted, only the last retained outer path can cover the next one. accepted.sort_by(|left, right| { #[cfg(test)] NESTING_WORK.set(NESTING_WORK.get() + 1); - left.path.cmp(&right.path) + left.0 + .cmp(&right.0) + .then_with(|| left.1.path.cmp(&right.1.path)) }); - let mut outer: Vec = Vec::new(); - for target in accepted { - if outer.iter().any(|candidate| { - #[cfg(test)] - NESTING_WORK.set(NESTING_WORK.get() + 1); - within(&target.path, &candidate.path) - }) { + let mut outer: Option = None; + for (key, target) in accepted { + #[cfg(test)] + NESTING_WORK.set(NESTING_WORK.get() + 1); + if outer.as_ref().is_some_and(|parent| key.starts_with(parent)) { plan.covered.push(target); } else { - outer.push(target); + outer = Some(key); + plan.targets.push(target); } } - plan.targets = outer; + // Preserve the review list's original path ordering, independently of + // the guard-key ordering used to establish nesting. + for targets in [&mut plan.targets, &mut plan.covered] { + targets.sort_by(|left, right| { + #[cfg(test)] + NESTING_WORK.set(NESTING_WORK.get() + 1); + left.path.cmp(&right.path) + }); + } // Which volume each target's bytes come back to. Only what comes back to // the scanned volume may be shown against its free space. @@ -326,12 +339,6 @@ fn windows_key(path: &Path) -> PathBuf { path.to_path_buf() } -/// `path` is `base` or inside it, by whole components, as the guards -/// compare them. -fn within(path: &Path, base: &Path) -> bool { - guard_key(path).starts_with(guard_key(base)) -} - /// The system trees with their guard keys, computed once: the review screen /// plans on every frame, for every mark. fn system_tree_keys() -> &'static [(String, PathBuf)] { @@ -1825,6 +1832,49 @@ mod tests { assert!(large <= 1024 * 32, "absolute work cap: {large}"); } + #[test] + fn nesting_keeps_component_boundaries_duplicates_and_output_order() { + let temp = TempDir::new().expect("tempdir"); + let targets: Vec<_> = + ["z", "ab", "a/deep", "a", "a", "ab/inner", "a/deeper/file"] + .map(|name| target(&temp.path().join(name), 1)) + .into(); + let result = + plan_against(&targets, temp.path(), &MountTable::default()); + let paths = |targets: &[Target]| -> Vec<_> { + targets + .iter() + .map(|item| { + item.path + .strip_prefix(temp.path()) + .expect("under root") + .to_path_buf() + }) + .collect() + }; + assert_eq!(paths(&result.targets), ["a", "ab", "z"].map(PathBuf::from)); + assert_eq!( + paths(&result.covered), + ["a", "a/deep", "a/deeper/file", "ab/inner"].map(PathBuf::from) + ); + assert!(result.blocked.is_empty()); + } + + #[cfg(any(windows, target_os = "macos"))] + #[test] + fn nesting_uses_guard_case_order_even_when_the_child_sorts_first() { + let temp = TempDir::new().expect("tempdir"); + let parent = target(&temp.path().join("alpha"), 3); + let child = target(&temp.path().join("ALPHA/child"), 1); + let result = plan_against( + &[child.clone(), parent.clone()], + temp.path(), + &MountTable::default(), + ); + assert_eq!(result.targets, [parent]); + assert_eq!(result.covered, [child]); + } + #[test] fn a_plan_keeps_only_the_outermost_targets() { let temp = tree(); From 92e54198ee8c12b5b06e3aaffff8309dd8f55609 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 14:06:02 -0400 Subject: [PATCH 04/19] docs: record red-green mark performance evidence --- docs/performance/mark-resolution/README.md | 180 ++++++++++++ docs/performance/mark-resolution/green.log | 40 +++ docs/performance/mark-resolution/lint.log | 7 + .../mark-resolution/red-dimensions.log | 17 ++ docs/performance/mark-resolution/tests.log | 264 ++++++++++++++++++ .../mark-resolution/wall-green-repeats.log | 126 +++++++++ .../mark-resolution/wall-red-repeats.log | 174 ++++++++++++ 7 files changed, 808 insertions(+) create mode 100644 docs/performance/mark-resolution/README.md create mode 100644 docs/performance/mark-resolution/green.log create mode 100644 docs/performance/mark-resolution/lint.log create mode 100644 docs/performance/mark-resolution/red-dimensions.log create mode 100644 docs/performance/mark-resolution/tests.log create mode 100644 docs/performance/mark-resolution/wall-green-repeats.log create mode 100644 docs/performance/mark-resolution/wall-red-repeats.log diff --git a/docs/performance/mark-resolution/README.md b/docs/performance/mark-resolution/README.md new file mode 100644 index 00000000..ca493ab5 --- /dev/null +++ b/docs/performance/mark-resolution/README.md @@ -0,0 +1,180 @@ +# Mark resolution and removal-plan performance + +Measured on 2026-10-14, Darwin arm64, Rust 1.97.1, repository development/test +profile (application opt-level 1, dependency opt-level 2). All trees and paths +are synthetic or inside private temporary directories. No home/volume scan, +real Trash operation, APFS accounting change or parent-branch edit was used. + +## Red before green + +Base: `6c8d4ce6211bf135ff4e42890faebff1040e1846` (upstream/main and fork main). +Commit `5a3c92e` adds probes and budgets without changing the algorithms. +Both application budgets and the core removal budget fail there. Their exact +output is in [red.log](red.log) and [red-core.log](red-core.log). + +Fix: `1206fda1437c288a53c4625e9ed2f0df05a8997e`. The same budgets pass; +[green.log](green.log) also includes the additional dimension and warm-frame +checks. The baseline dimension test was temporarily applied to the red +snapshot before restoring the fixed sources; see +[red-dimensions.log](red-dimensions.log). + +The three red budgets grow marks from 128 to 1,024. Refresh and window fixtures +also grow sibling folders from 128 to 1,024, each containing one marked file: +N=256/2,048 non-root nodes, M=128/1,024 marks, D=2 path components. The removal +fixture uses disjoint paths in a deterministic permutation, so the sort does +real work and no early ancestor match hides the quadratic loop. + +| Production path | Main work at 128 / 1,024 marks | Growth | Fixed work at 128 / 1,024 marks | Growth | +| --- | ---: | ---: | ---: | ---: | +| `plan_against`, one plan | 9,187 / 537,027 | 58.5x | 1,314 / 15,298 | 11.6x | +| `Marks::refresh`, four refreshes | 34,048 / 2,107,392 | 61.9x | 2,560 / 20,480 | 8.0x | +| Real window, four forced draws | 68,312 / 4,215,000 | 61.7x | 856 / 5,336 | 6.2x | + +Every budget requires at most 16x work for 8x input, plus an absolute cap: +32 work units per mark for removal; 16 per mark per refresh for refresh; 16 per +mark per observed `prepare` for the window. No elapsed-time assertion is used. + +Removal units count sort comparisons and nesting-candidate checks. Resolution +units count mark visits and sibling-name comparisons before the fix; after the +fix they count mark visits, indexed sibling entries and component lookups. +These are logical work budgets, not CPU instruction counts. Hashing/comparing +long names and paths still costs their length. All probes are test-only. + +The GPUI harness forces notifications, draws the actual window and asserts +that `Disktree::prepare` ran at least once per draw: eight calls were observed +for four draws. This includes path lookups elsewhere in the screen, not just +the new resolver. It catches reintroducing a linear sibling search per mark +into `prepare`, even if a resolver microbenchmark still passes. + +## Wall measurements + +These are medians of three serial repetitions with `--test-threads=1` on the +same machine/profile. Fixture construction, compilation and mark insertion +are outside the timers. The window timer includes the real render path, +layout/painting work and freshly evaluated removal projections. Backing roots +are empty temporary directories, with the displayed trees constructed in +memory; these are scaling workloads, not a filesystem latency benchmark. + +| Workload at 1,024 marks | Main median | Fixed median | Speedup | +| --- | ---: | ---: | ---: | +| One removal plan | 398.13 ms | 23.24 ms | 17.1x | +| Four mark refreshes | 9.07 ms | 1.53 ms | 5.9x | +| Four window draws, eight prepares | 6,602.00 ms | 358.68 ms | 18.4x | + +Raw samples: [wall-red-repeats.log](wall-red-repeats.log) and +[wall-green-repeats.log](wall-green-repeats.log). Fixed warm draws have their +own timer and are excluded from the cold-window median above. Timings are +observations, not portable thresholds or a frame-rate guarantee. + +## Which variables actually cause the growth + +N is all scanned nodes, M is marked paths, D is path depth, R is the number +of refreshes/renders. F below is the number of first-level sibling folders. +The remaining single-child chains make depth independently controllable. + +| Folders F | Marks M | Depth D | Refreshes R | Main work | Fixed work | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 128 | 8 | 2 | 1 | 1,012 | 160 | +| 1,024 | 8 | 2 | 1 | 8,180 | 1,056 | +| 1,024 | 64 | 2 | 1 | 63,648 | 1,280 | +| 128 | 8 | 16 | 1 | 1,124 | 384 | +| 128 | 8 | 2 | 8 | 8,096 | 1,280 | + +Main is linear in each of F, M, D and R when the others are held fixed. +It becomes quadratic when siblings and marks grow together. Repeating an +unchanged linear pass R times is O(items * R); calling it quadratic solely +because R can grow would misidentify the problem. Unmarked file counts do +not affect removal nesting at all, but sibling counts affect path lookup. + +On main: + +- Disjoint accepted removal marks cause M(M-1)/2 ancestor checks after sorting. +- Refresh resolves each marked path separately. At a broad directory, each + lookup scans siblings again: O(M*N) worst-case per refresh. +- `prepare` repeats analogous path-to-crumb searches for every mark on every + render, even when the tree and marks are unchanged. + +After the fix: + +- Nesting costs O(M log M) path comparisons plus M prefix checks. Guard keys + are computed once per accepted target, sorted using the guards' case and + macOS Data-volume semantics, and compared with the last retained outer + target. Display output retains the original path ordering. Including path + length L, this stage costs O(M log M * L) time and O(M * L) storage. +- One `Resolver` batch indexes each visited directory's siblings once. Let S + be those indexed entries (S <= N). Expected hash work is O(S + M*D) per + batch, with O(S) temporary index storage; untouched subtrees cost nothing. + `Marks::refresh` creates one batch per refresh, so R refreshes still cost + O(R * (S + M*D)). Path/name lengths are additional hashing/copying costs. +- `Marks::crumbs` stores O(M*D) crumbs, shared by `Arc`. An unchanged frame + reuses that set without resolving marked paths. The first frame after an + invalidation costs one batch. Visible tiles still require processing each + frame: O(T*R) at bounded display depth, where T is the laid-out tile count. + Ancestor-mark checks depend on depth; this does not claim depth-independent + work for arbitrarily deep crumb vectors. + +The real-window warm budget allows only 64 resolution units per observed +prepare, independent of M. Both mark counts measured 216 units over eight +prepares (other screen lookups); marked-path resolution was reused. It also +counts actual tile visits and verifies proportional work across repeated +frames. Merging tiny tiles means T need not grow with N. + +Fresh removal plans still evaluate guards, resolve filesystem paths and +attribute volumes for marks on every render. The sort remains O(M log M) +per plan, and mount-table/system-tree checks retain their existing costs. +These costs are visible in the remaining ~359 ms window workload. No removal +authorization is cached. The worker's fresh mount/link checks immediately +before deletion and descriptor-based deletion protections are unchanged. + +## Invalidation and semantics + +The display cache is invalidated by toggle, remove, clear and refresh, and +checks tree allocation identity, root path and metric. It stores a `Weak` +reference to the tree, so it cannot retain an old scan; the weak reference also +prevents address reuse from masquerading as the same tree. `Arc::make_mut` +dissociates that identity before in-place changes can affect a cached tree. + +Tests cover every boundary, metric reordering and replacement through the +window, missing paths, component boundaries, exact case-sensitive scan names, +first-match behavior for equal lossy names, and non-UTF-8 Unix paths. Removal +nesting uses the platform guard keys, with coverage for case variants, +duplicates, component prefixes and stable output ordering. Existing safety +and synthetic permanent-deletion tests remain green. + +Integration API: + +```rust +let mut resolver = marks::Resolver::new(root_path, tree); +let resolved = resolver.resolve(path); // Option<(Vec, &Node)> +``` + +Reuse a resolver for the whole batch. The old standalone `marks::find` was +removed when its production caller moved to this API. Display callers use +`Marks::crumbs(&Path, &Arc, Metric) -> Arc>>`; +callers have no revision-counter or manual invalidation obligation. + +## Verification and reproduction + +- `cargo xtask lint`: pass; [lint.log](lint.log). +- `cargo xtask test`: 78 app, 150 core and 3 xtask tests pass; + [tests.log](tests.log). Two pre-existing ignored core tests remain ignored. +- `cargo test --workspace budget -- --nocapture`: all budgets pass. +- Three serial red repetitions fail the intended budgets (exit 101), followed + by three serial green repetitions passing them (exit 0). +- Local validation is macOS arm64. Other operating systems/architectures are + left to the repository's existing CI; no cross-platform result is claimed. + +From a clean checkout at `5a3c92e`, reproduce the failures with: + +```sh +cargo test -p disktree-app budget -- --nocapture --test-threads=1 +cargo test -p disktree-core removal_nesting_budget -- --nocapture +``` + +At `1206fda`, run the green budgets and gates: + +```sh +cargo test --workspace budget -- --nocapture --test-threads=1 +cargo xtask lint +cargo xtask test +``` diff --git a/docs/performance/mark-resolution/green.log b/docs/performance/mark-resolution/green.log new file mode 100644 index 00000000..64b445ee --- /dev/null +++ b/docs/performance/mark-resolution/green.log @@ -0,0 +1,40 @@ + Finished `test` profile [optimized + debuginfo] target(s) in 0.26s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 5 tests +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +dimensions folders=128 M=8 D=2 R=1 work=160 wall=30.334µs +refresh M=128 N=256 R=4 work=2560 wall=241.75µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=46.917µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=71.5µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=45µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=88.459µs +test marks::tests::refresh_dimension_budget ... ok +refresh M=1024 N=2048 R=4 work=20480 wall=1.580625ms +test marks::tests::refresh_resolution_budget ... ok +warm window M=128 prepares=8 resolution=216 tiles=1544 wall=61.302208ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=61.50175ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=354.14925ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=341.058584ms +test tests::marked_window_resolution_budget ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 73 filtered out; finished in 0.87s + + Running unittests src/lib.rs (target/debug/deps/disktree_core-9300e5ba20d6b25f) + +running 1 test +plan M=128 work=1314 wall=2.802916ms +plan M=1024 work=15298 wall=21.911541ms +test removal::tests::removal_nesting_budget ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 151 filtered out; finished in 0.03s + + Running unittests src/main.rs (target/debug/deps/xtask-0dc2df263f51d89e) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 3 filtered out; finished in 0.00s + diff --git a/docs/performance/mark-resolution/lint.log b/docs/performance/mark-resolution/lint.log new file mode 100644 index 00000000..363cf891 --- /dev/null +++ b/docs/performance/mark-resolution/lint.log @@ -0,0 +1,7 @@ + Finished `dev` profile [optimized + debuginfo] target(s) in 0.09s + Running `target/debug/xtask lint` + Checking disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `dev` profile [optimized + debuginfo] target(s) in 1.77s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` diff --git a/docs/performance/mark-resolution/red-dimensions.log b/docs/performance/mark-resolution/red-dimensions.log new file mode 100644 index 00000000..4d93cf97 --- /dev/null +++ b/docs/performance/mark-resolution/red-dimensions.log @@ -0,0 +1,17 @@ + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Finished `test` profile [optimized + debuginfo] target(s) in 5.10s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 1 test +dimensions folders=128 M=8 D=2 R=1 work=1012 wall=61.958µs +dimensions folders=1024 M=8 D=2 R=1 work=8180 wall=34.958µs +dimensions folders=1024 M=64 D=2 R=1 work=63648 wall=284.042µs +dimensions folders=128 M=8 D=16 R=1 work=1124 wall=10.666µs +dimensions folders=128 M=8 D=2 R=8 work=8096 wall=46.792µs +test marks::tests::refresh_dimension_budget ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 73 filtered out; finished in 0.00s + diff --git a/docs/performance/mark-resolution/tests.log b/docs/performance/mark-resolution/tests.log new file mode 100644 index 00000000..158bcd31 --- /dev/null +++ b/docs/performance/mark-resolution/tests.log @@ -0,0 +1,264 @@ + Finished `dev` profile [optimized + debuginfo] target(s) in 0.13s + Running `target/debug/xtask test` + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 3.58s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 78 tests +test appearance::tests::a_linux_desktop_is_followed_only_if_it_states_a_preference ... ok +test marks::tests::display_path_shortens_the_home_prefix ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::marking_the_same_path_twice_keeps_one_entry ... ok +test git::tests::summaries_say_what_would_be_lost ... ok +test marks::tests::toggling_marks_and_unmarks ... ok +test marks::tests::resolver_preserves_lossy_non_utf8_components ... ok +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test palette::tests::colourful_categories_share_one_level ... ok +test palette::tests::age_buckets_cover_every_age_in_order ... ok +test palette::tests::deeper_tiles_lift_away_from_the_background ... ok +test palette::tests::mix_clamps_its_parameter ... ok +test palette::tests::mixing_toward_a_grey_keeps_the_hue ... ok +test palette::tests::every_legend_category_has_its_own_hue ... ok +test palette::tests::the_highlight_is_not_a_category_colour ... ok +test power::tests::presets_are_fixed_and_show_the_effective_cpu_limit ... ok +test power::tests::presets_that_add_no_workers_are_not_offered ... ok +test marks::tests::find_matches_whole_components_only ... ok +test marks::tests::resolver_preserves_scan_name_semantics ... ok +test marks::tests::refresh_re_reads_sizes_from_a_new_tree ... ok +test state::tests::a_transition_grows_a_tile_that_is_being_entered ... ok +test marks::tests::cached_crumbs_follow_every_invalidation_boundary ... ok +test state::tests::a_transition_starts_where_the_region_was_and_ends_where_it_is ... ok +test state::tests::clamping_never_leaves_a_margin ... ok +test state::tests::directions_only_see_gaps_on_their_own_side ... ok +test state::tests::projecting_and_unprojecting_are_inverses ... ok +test state::tests::zooming_keeps_the_point_under_the_cursor_still ... ok +test argument_tests::saved_power_is_loaded_before_cli_overrides ... ok +test marks::tests::refresh_dimension_budget ... ok +test power::tests::saved_choice_survives_replacement_and_bad_data_is_reported ... ok +test marks::tests::refresh_resolution_budget ... ok +test git::tests::reads_a_real_checkout ... ok +test git::tests::a_checkout_cannot_make_git_run_its_programs ... ok +test git::tests::a_checkout_cannot_make_git_verify_a_signature_with_its_program ... ok +test git::tests::a_checkouts_own_filter_is_never_run ... ok +test tests::enter_opens_the_selected_directory_at_any_depth ... ok +test tests::escape_cancels_the_first_scan_and_r_starts_it_again ... ok +test tests::escape_in_the_delete_dialog_cancels ... ok +test tests::a_permanent_deletion_asks_in_an_alert_dialog_then_removes ... ok +test tests::a_crumb_lists_its_siblings_and_jumps_sideways ... ok +test tests::hovering_reports_the_tile_under_the_pointer ... ok +test tests::restart_arguments_parse_back_to_the_same_scan ... ok +test tests::a_parents_name_gets_its_own_band_above_its_children ... ok +test tests::enter_before_the_search_lands_applies_it_when_it_does ... ok +test tests::escape_cancels_widening_and_keeps_the_tree_on_screen ... ok +test tests::a_view_without_flags_or_marks_still_draws ... ok +test tests::after_descending_every_tile_is_inside_the_directory_drawn ... ok +test tests::enter_in_the_focused_volume_picker_scans_the_selected_root ... ok +test tests::command_chords_are_not_read_as_plain_letters ... ok +test tests::both_appearances_draw ... ok +test tests::keys_walk_the_tree_and_mark_what_is_selected ... ok +test tests::back_and_forward_retrace_where_you_have_been ... ok +test tests::dragging_the_panel_edge_resizes_it_within_its_limits ... ok +test tests::interface_zoom_scales_the_rem_and_the_header_band ... ok +test tests::marked_tiles_follow_metric_rescan_and_mark_changes ... ok +test tests::the_first_scan_shows_what_it_is_doing_then_the_treemap ... ok +test tests::marking_a_directory_marks_everything_inside_it ... ok +test tests::mouse_side_buttons_go_back_and_forward ... ok +test tests::showing_a_tile_that_is_gone_says_so_instead ... ok +test widgets::tests::ages_read_in_the_unit_a_person_would_use ... ok +test widgets::tests::sizes_split_into_number_and_unit ... ok +test tests::power_efficiency_menu_saves_without_discarding_the_tree ... ok +test tests::the_mark_key_follows_the_pointer_until_the_keyboard_moves ... ok +test tests::the_help_overlay_opens_and_closes ... ok +test tests::the_review_screen_copies_the_list_as_an_agent_prompt ... ok +test tests::the_treemap_zooms_with_the_wheel_and_resets ... ok +test tests::the_review_screen_switches_removal_mode ... ok +test tests::the_trash_is_the_default_when_there_is_one ... ok +test tests::the_power_menu_offers_only_what_the_cpus_can_tell_apart ... ok +test tests::right_click_selects_the_tile_and_reveals_it ... ok +test tests::the_volume_picker_is_centred_and_still_dismisses_from_outside ... ok +test tests::the_volume_picker_opens_moves_and_closes ... ok +test tests::the_window_draws_a_treemap_with_tiles ... ok +test tests::switching_the_metric_keeps_the_directory_and_the_selection ... ok +test tests::the_wheel_goes_into_the_directory_it_zoomed_into ... ok +test tests::widening_reuses_the_tree_it_has_and_reads_only_the_rest ... ok +test tests::typing_filters_live_and_enter_shows_only_the_matches ... ok +test tests::marked_window_resolution_budget ... ok + +test result: ok. 78 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 2.02s + + Running unittests src/lib.rs (target/debug/deps/disktree_core-9300e5ba20d6b25f) + +running 152 tests +test classify::tests::macos_developer_leftovers_are_reclaimable_and_its_risks_are_not ... ok +test classify::tests::the_legend_lists_every_named_category_once ... ok +test classify::tests::steam_libraries_and_emulation_are_media_not_disposable_caches ... ok +test classify::tests::windows_caches_and_sync_folders_are_recognized ... ok +test filter::tests::case_is_ignored_without_allocating ... ok +test export::tests::the_list_is_one_path_per_line ... ok +test export::tests::a_name_that_is_not_unicode_is_flagged_not_listed ... ok +test export::tests::a_name_with_a_newline_cannot_smuggle_in_a_path ... ok +test export::tests::the_prompt_lists_every_path_with_its_size_and_the_rules ... ok +test filter::tests::a_search_below_the_root_leaves_the_rest_alone ... ok +test filter::tests::matches_are_kept_whole_and_their_ancestors_by_what_matched ... ok +test classify::tests::a_steam_library_on_another_drive_is_media_not_its_temp_folder ... ok +test filter::tests::an_empty_needle_filters_nothing ... ok +test insights::tests::the_limit_keeps_the_largest ... ok +test insights::tests::documents_are_never_suggested ... ok +test insights::tests::ranks_findings_largest_first_and_skips_the_tiny ... ok +test access::tests::only_macos_has_an_answer_and_a_missing_database_is_not_a_refusal ... ok +test classify::tests::target_is_build_output_only_beside_a_cargo_manifest ... ok +test classify::tests::caches_are_reclaimable_all_the_way_down ... ok +test insights::tests::crumbs_address_the_finding_from_the_root ... ok +test classify::tests::an_unknown_top_level_directory_takes_its_largest_known_child ... ok +test classify::tests::names_announce_their_kind_and_children_inherit_it ... ok +test classify::tests::sandbox_layers_and_snapshots_are_reclaimable_only_in_sandbox_state ... ok +test removal::tests::a_directory_holding_a_system_tree_is_refused ... ok +test removal::tests::a_directory_holding_the_home_directory_is_refused ... ok +test macos::tests::a_missing_directory_is_an_error ... ok +test removal::tests::macos_always_uses_the_system_trash ... ok +test removal::tests::macos_guard_keys_fold_the_data_volume_and_case ... ok +test removal::tests::macos_guards_see_through_other_spellings ... ok +test removal::tests::a_name_that_is_not_utf8_is_refused_before_the_trash_sees_it ... ok +test removal::tests::macos_system_trees_are_refused_but_apps_are_not ... ok +test removal::tests::names_win32_would_misread_are_recognized ... ok +test macos::tests::a_listing_has_names_kinds_lengths_and_no_dot_entries ... ok +test macos::tests::names_are_text_and_open ... ok +test removal::tests::nesting_uses_guard_case_order_even_when_the_child_sorts_first ... ok +test removal::tests::normalize_resolves_dots_without_the_filesystem ... ok +test removal::tests::nesting_keeps_component_boundaries_duplicates_and_output_order ... ok +test macos::tests::every_leaf_reads_as_lstat_reads_it ... ok +test removal::tests::percent_encoding_escapes_what_the_spec_requires ... ok +test removal::tests::a_target_with_a_mount_inside_is_refused ... ok +test removal::tests::a_mount_below_a_target_is_caught_again_at_removal_time ... ok +test removal::tests::a_plan_keeps_only_the_outermost_targets ... ok +test removal::tests::a_missing_target_reports_not_found ... ok +test removal::tests::a_volume_is_judged_by_its_mount_where_the_kernel_names_one ... ok +test removal::tests::mount_points_are_recognized ... ok +test removal::tests::a_trash_info_path_is_restorable ... ok +test removal::tests::paths_outside_the_root_are_refused ... ok +test removal::tests::a_target_reached_through_a_link_is_refused ... ok +test removal::tests::home_is_found_by_its_real_path_too ... ok +test removal::tests::permanent_removal_takes_directories_and_leaves_siblings ... ok +test removal::tests::the_macos_trash_takes_a_file_and_leaves_a_linked_target ... ignored, puts a file in this Mac's real Trash +test removal::tests::a_removal_run_reports_every_item_and_a_total ... ok +test removal::tests::nothing_is_mounted_below_a_temp_dir ... ok +test removal::tests::permanent_removal_unlinks_nested_symlinks_without_following_them ... ok +test removal::tests::a_link_put_on_the_way_down_after_planning_stops_the_worker ... ok +test removal::tests::permanent_removal_follows_links_above_the_root ... ok +test removal::tests::the_mount_table_is_read_off_the_calling_thread_and_then_kept ... ok +test removal::tests::read_only_files_are_removed_too ... ok +test removal::tests::permanent_removal_does_not_follow_a_link_put_on_the_way_down ... ok +test removal::tests::permanent_removal_unlinks_a_symlink_instead_of_following_it ... ok +test removal::tests::permanent_removal_takes_nested_trees_and_hidden_files ... ok +test removal::tests::the_root_and_home_are_refused ... ok +test removal::tests::trash_names_avoid_collisions ... ok +test removal::tests::sibling_prefixes_are_not_treated_as_containment ... ok +test scan::tests::a_file_root_is_rejected ... ok +test removal::tests::this_macs_mount_table_is_read_without_proc ... ok +test removal::tests::the_xdg_trash_moves_a_file_and_records_where_it_came_from ... ok +test scan::tests::fixed_worker_pools_change_size_without_reinitializing_rayon ... ok +test scan::tests::an_unreadable_directory_is_recorded_not_fatal ... ok +test scan::tests::disk_usage_is_whole_allocation_units ... ok +test scan::tests::a_directory_reports_its_own_bytes_separately ... ok +test scan::tests::children_are_ranked_largest_first ... ok +test scan::tests::a_spawned_scan_reports_progress_and_a_result ... ok +test scan::tests::whole_disk_smoke ... ignored, walks the whole disk +test scan::tests::followed_symlink_loops_do_not_hang_the_scan ... ok +test scan_threads::tests::governor_waits_for_retirement_and_recovers_without_exceeding_target ... ok +test scan::tests::hardlinks_are_charged_once_by_default ... ok +test scan_threads::tests::inconclusive_windows_cannot_accept_a_reduction ... ok +test scan_threads::tests::increased_limit_is_not_supplied_until_jobs_are_dispatched ... ok +test scan_threads::tests::long_directory_never_becomes_a_fully_supplied_window ... ok +test scan_threads::tests::search_refines_one_reference_without_compounding_loss ... ok +test size::tests::a_value_that_rounds_up_to_the_next_unit_is_shown_in_it ... ok +test size::tests::bytes_use_binary_units_with_one_decimal_below_ten ... ok +test scan::tests::a_wider_scan_reuses_the_subtree_it_already_knows ... ok +test size::tests::counts_switch_to_metric_suffixes ... ok +test size::tests::share_bar_is_always_the_requested_width ... ok +test size::tests::share_survives_a_zero_total ... ok +test size::tests::short_form_drops_the_space ... ok +test space::tests::a_bind_mount_inside_the_scan_is_left_out ... ok +test space::tests::a_home_scan_has_no_foreign_mounts_here ... ok +test space::tests::a_missing_path_reports_the_io_error ... ok +test space::tests::a_real_volume_reports_plausible_numbers ... ok +test scan::tests::hidden_entries_are_included_by_default_and_excluded_on_request ... ok +test space::tests::a_root_booted_from_a_snapshot_keeps_its_subvolumes ... ok +test space::tests::a_table_that_does_not_place_a_path_claims_nothing ... ok +test scan_threads::tests::real_pool_retires_at_job_boundaries_and_drains_queued_descendants ... ok +test space::tests::a_volume_includes_its_subvolumes_and_nothing_else ... ok +test space::tests::a_volume_keeps_its_subvolumes_and_loses_everything_else ... ok +test space::tests::btrfs_subvolume_devices_can_differ ... ok +test space::tests::excluded_views_cannot_replace_visible_files ... ok +test space::tests::hidden_original_and_nested_mounts_are_preserved ... ok +test space::tests::macos_mount_points_keep_spaces_and_brackets ... ok +test space::tests::mountinfo_escapes_are_decoded_once ... ok +test space::tests::only_macos_skips_the_data_volume_and_in_the_roots_spelling ... ok +test space::tests::projecting_removal_cannot_exceed_the_volume ... ok +test space::tests::separate_home_disk_is_its_own_volume ... ok +test space::tests::the_device_is_the_longest_matching_mount ... ok +test space::tests::the_home_disk_on_macos_is_the_root_and_has_a_device ... ok +test space::tests::subvolumes_are_not_repeats_but_a_second_mount_of_a_disk_is ... ok +test space::tests::the_top_of_a_btrfs_disk_beside_its_subvolumes_is_counted_once ... ok +test space::tests::the_whole_disk_is_the_top_of_the_home_volume ... ok +test space::tests::unrelated_tmpfs_mounts_are_not_duplicates ... ok +test space::tests::used_fraction_handles_an_empty_volume_report ... ok +test scan::tests::symlinks_are_not_followed_by_default ... ok +test space::tests::volume_candidates_are_real_disks_not_pseudo_filesystems ... ok +test space::tests::volumes_with_least_space_come_first_and_unknowns_last ... ok +test tree::tests::a_node_stays_small ... ok +test tree::tests::aggregate_derives_totals_and_orders_children ... ok +test tree::tests::find_reports_crumbs_and_is_case_insensitive ... ok +test tree::tests::depth_counts_edges ... ok +test tree::tests::identities_that_differ_only_in_their_top_bits_are_two_files ... ok +test tree::tests::path_of_joins_names_beneath_the_root ... ok +test tree::tests::largest_child_is_the_first_child_because_children_are_sorted ... ok +test tree::tests::aggregate_can_rank_by_file_count ... ok +test tree::tests::resolve_walks_child_indices ... ok +test scan::tests::file_count_metric_ranks_by_entries ... ok +test treemap::tests::a_leaf_never_claims_a_header ... ok +test treemap::tests::a_tile_without_room_for_a_header_stays_whole ... ok +test treemap::tests::hit_returns_the_deepest_tile ... ok +test treemap::tests::layout_depth_one_stops_at_direct_children ... ok +test treemap::tests::a_filtered_layout_shows_only_the_matches_at_their_size ... ok +test treemap::tests::layout_merges_the_child_tail_into_one_tile ... ok +test treemap::tests::a_subdivided_directory_keeps_a_header_for_its_own_name ... ok +test treemap::tests::squarify_fills_the_area ... ok +test treemap::tests::layout_nests_children_inside_their_parent ... ok +test treemap::tests::squarify_keeps_aspect_ratios_reasonable ... ok +test treemap::tests::squarify_survives_degenerate_input ... ok +test treemap::tests::tile_crumbs_address_the_scanned_tree_not_the_drawn_node ... ok +test tree::tests::seen_charges_an_identity_once_in_either_store ... ok +test scan::tests::max_depth_stops_descending_but_keeps_direct_files ... ok +test space::tests::without_a_table_the_same_place_is_one_volume ... ok +test scan::tests::totals_are_summed_bottom_up ... ok +test removal::tests::system_trees_are_refused_in_a_whole_disk_scan ... ok +test removal::tests::the_directories_above_home_are_refused ... ok +test removal::tests::removal_nesting_budget ... ok +test scan::tests::cancelled_queued_jobs_release_parent_completion_tokens ... ok +test removal::tests::a_failing_trash_tool_reports_its_stderr ... ok +test scan::tests::fixed_worker_pools_preserve_a_wide_deep_tree ... ok +test removal::tests::only_the_scanned_volumes_marks_are_projected ... ok +test scan_threads::tests::controller_retires_recovers_after_search_and_cancels ... ok +test macos::tests::a_directory_larger_than_the_buffer_is_listed_whole ... ok +test removal::tests::a_trash_tool_is_called_with_the_path_after_a_separator ... ok + +test result: ok. 150 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.27s + + Running unittests src/main.rs (target/debug/deps/xtask-0dc2df263f51d89e) + +running 3 tests +test bundle::tests::notarizing_needs_a_real_signature ... ok +test bundle::tests::the_plist_template_names_the_executable_and_takes_the_version ... ok +test bundle::tests::signing_is_ad_hoc_unless_an_identity_is_given ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests disktree_core + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + diff --git a/docs/performance/mark-resolution/wall-green-repeats.log b/docs/performance/mark-resolution/wall-green-repeats.log new file mode 100644 index 00000000..ece27470 --- /dev/null +++ b/docs/performance/mark-resolution/wall-green-repeats.log @@ -0,0 +1,126 @@ + +REPEAT 1: disktree-app fixed + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 5.26s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 5 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=92.541µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=47.167µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=50.167µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=28.084µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=57.541µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=204.417µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.528584ms +ok +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=64.363667ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=66.748208ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=347.242958ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=358.680958ms +ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 73 filtered out; finished in 0.91s + +EXIT 0 + +REPEAT 1: disktree-core fixed + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Finished `test` profile [optimized + debuginfo] target(s) in 1.68s + Running unittests src/lib.rs (target/debug/deps/disktree_core-c3cb863045931ea5) + +running 1 test +test removal::tests::removal_nesting_budget ... plan M=128 work=1314 wall=3.372625ms +plan M=1024 work=15298 wall=23.837875ms +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 151 filtered out; finished in 0.03s + +EXIT 0 + +REPEAT 2: disktree-app fixed + Finished `test` profile [optimized + debuginfo] target(s) in 0.29s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 5 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=23.458µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=44.542µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=56.5µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=26µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=55.125µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=208.667µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.52875ms +ok +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=65.17375ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=67.732166ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=350.850083ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=356.051625ms +ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 73 filtered out; finished in 0.89s + +EXIT 0 + +REPEAT 2: disktree-core fixed + Finished `test` profile [optimized + debuginfo] target(s) in 0.10s + Running unittests src/lib.rs (target/debug/deps/disktree_core-c3cb863045931ea5) + +running 1 test +test removal::tests::removal_nesting_budget ... plan M=128 work=1314 wall=2.852625ms +plan M=1024 work=15298 wall=21.89725ms +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 151 filtered out; finished in 0.03s + +EXIT 0 + +REPEAT 3: disktree-app fixed + Finished `test` profile [optimized + debuginfo] target(s) in 0.30s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 5 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=23µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=51.5µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=59µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=33.667µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=60.167µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=198.958µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.648333ms +ok +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=65.083791ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=66.946292ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=340.564ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=359.560417ms +ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 73 filtered out; finished in 0.89s + +EXIT 0 + +REPEAT 3: disktree-core fixed + Finished `test` profile [optimized + debuginfo] target(s) in 0.11s + Running unittests src/lib.rs (target/debug/deps/disktree_core-c3cb863045931ea5) + +running 1 test +test removal::tests::removal_nesting_budget ... plan M=128 work=1314 wall=3.403792ms +plan M=1024 work=15298 wall=23.242542ms +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 151 filtered out; finished in 0.03s + +EXIT 0 diff --git a/docs/performance/mark-resolution/wall-red-repeats.log b/docs/performance/mark-resolution/wall-red-repeats.log new file mode 100644 index 00000000..55f720af --- /dev/null +++ b/docs/performance/mark-resolution/wall-red-repeats.log @@ -0,0 +1,174 @@ + +REPEAT 1: disktree-app baseline 5a3c92e + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Finished `test` profile [optimized + debuginfo] target(s) in 4.81s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 4 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=34048 wall=457.125µs +refresh M=1024 N=2048 R=4 work=2107392 wall=11.105667ms + +thread 'marks::tests::refresh_resolution_budget' (403559694) panicked at crates/disktree-app/src/marks.rs:208:9: +8x marks/folders: 34048 -> 2107392 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED +test tests::marked_window_resolution_budget ... window M=128 N=256 draws=4 prepares=8 work=68312 wall=159.687834ms +window M=1024 N=2048 draws=4 prepares=8 work=4215000 wall=6.601997625s + +thread 'tests::marked_window_resolution_budget' (403559695) panicked at crates/disktree-app/src/tests.rs:1904:5: +8x marks/folders per frame: 68312/8 -> 4215000/8 +FAILED + +failures: + +failures: + marks::tests::refresh_resolution_budget + tests::marked_window_resolution_budget + +test result: FAILED. 2 passed; 2 failed; 0 ignored; 0 measured; 69 filtered out; finished in 6.83s + +error: test failed, to rerun pass `-p disktree-app --bin disktree` +EXIT 101 + +REPEAT 1: disktree-core baseline 5a3c92e + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Finished `test` profile [optimized + debuginfo] target(s) in 0.68s + Running unittests src/lib.rs (target/debug/deps/disktree_core-c3cb863045931ea5) + +running 1 test +test removal::tests::removal_nesting_budget ... plan M=128 work=9187 wall=11.07275ms +plan M=1024 work=537027 wall=398.130709ms + +thread 'removal::tests::removal_nesting_budget' (403560362) panicked at crates/disktree-core/src/removal.rs:1789:9: +8x marks: 9187 -> 537027 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED + +failures: + +failures: + removal::tests::removal_nesting_budget + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 149 filtered out; finished in 0.41s + +error: test failed, to rerun pass `-p disktree-core --lib` +EXIT 101 + +REPEAT 2: disktree-app baseline 5a3c92e + Finished `test` profile [optimized + debuginfo] target(s) in 0.28s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 4 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=34048 wall=278.333µs +refresh M=1024 N=2048 R=4 work=2107392 wall=9.065291ms + +thread 'marks::tests::refresh_resolution_budget' (403560424) panicked at crates/disktree-app/src/marks.rs:208:9: +8x marks/folders: 34048 -> 2107392 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED +test tests::marked_window_resolution_budget ... window M=128 N=256 draws=4 prepares=8 work=68312 wall=161.706417ms +window M=1024 N=2048 draws=4 prepares=8 work=4215000 wall=6.607910459s + +thread 'tests::marked_window_resolution_budget' (403560426) panicked at crates/disktree-app/src/tests.rs:1904:5: +8x marks/folders per frame: 68312/8 -> 4215000/8 +FAILED + +failures: + +failures: + marks::tests::refresh_resolution_budget + tests::marked_window_resolution_budget + +test result: FAILED. 2 passed; 2 failed; 0 ignored; 0 measured; 69 filtered out; finished in 6.83s + +error: test failed, to rerun pass `-p disktree-app --bin disktree` +EXIT 101 + +REPEAT 2: disktree-core baseline 5a3c92e + Finished `test` profile [optimized + debuginfo] target(s) in 0.09s + Running unittests src/lib.rs (target/debug/deps/disktree_core-c3cb863045931ea5) + +running 1 test +test removal::tests::removal_nesting_budget ... plan M=128 work=9187 wall=8.688291ms +plan M=1024 work=537027 wall=399.84275ms + +thread 'removal::tests::removal_nesting_budget' (403560664) panicked at crates/disktree-core/src/removal.rs:1789:9: +8x marks: 9187 -> 537027 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED + +failures: + +failures: + removal::tests::removal_nesting_budget + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 149 filtered out; finished in 0.41s + +error: test failed, to rerun pass `-p disktree-core --lib` +EXIT 101 + +REPEAT 3: disktree-app baseline 5a3c92e + Finished `test` profile [optimized + debuginfo] target(s) in 0.26s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 4 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=34048 wall=263.917µs +refresh M=1024 N=2048 R=4 work=2107392 wall=7.863542ms + +thread 'marks::tests::refresh_resolution_budget' (403560714) panicked at crates/disktree-app/src/marks.rs:208:9: +8x marks/folders: 34048 -> 2107392 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED +test tests::marked_window_resolution_budget ... window M=128 N=256 draws=4 prepares=8 work=68312 wall=155.442209ms +window M=1024 N=2048 draws=4 prepares=8 work=4215000 wall=6.564761291s + +thread 'tests::marked_window_resolution_budget' (403560715) panicked at crates/disktree-app/src/tests.rs:1904:5: +8x marks/folders per frame: 68312/8 -> 4215000/8 +FAILED + +failures: + +failures: + marks::tests::refresh_resolution_budget + tests::marked_window_resolution_budget + +test result: FAILED. 2 passed; 2 failed; 0 ignored; 0 measured; 69 filtered out; finished in 6.77s + +error: test failed, to rerun pass `-p disktree-app --bin disktree` +EXIT 101 + +REPEAT 3: disktree-core baseline 5a3c92e + Finished `test` profile [optimized + debuginfo] target(s) in 0.10s + Running unittests src/lib.rs (target/debug/deps/disktree_core-c3cb863045931ea5) + +running 1 test +test removal::tests::removal_nesting_budget ... plan M=128 work=9187 wall=8.564791ms +plan M=1024 work=537027 wall=393.824917ms + +thread 'removal::tests::removal_nesting_budget' (403560958) panicked at crates/disktree-core/src/removal.rs:1789:9: +8x marks: 9187 -> 537027 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED + +failures: + +failures: + removal::tests::removal_nesting_budget + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 149 filtered out; finished in 0.40s + +error: test failed, to rerun pass `-p disktree-core --lib` +EXIT 101 From aa4f168ae665bcd3be23a5386643e4da3e4ad385 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 14:23:02 -0400 Subject: [PATCH 05/19] fix: account for selected clone groups and bound metadata reuse --- README.md | 13 +- crates/disktree-app/src/marks.rs | 4 +- crates/disktree-app/src/state.rs | 69 ++- crates/disktree-app/src/tests.rs | 34 ++ crates/disktree-app/src/views.rs | 26 +- .../disktree-core/examples/scan_accounting.rs | 6 +- crates/disktree-core/src/clone_cache.rs | 241 +++++++++ crates/disktree-core/src/lib.rs | 2 + crates/disktree-core/src/macos.rs | 154 +++++- crates/disktree-core/src/removal.rs | 66 ++- crates/disktree-core/src/scan.rs | 126 ++++- crates/disktree-core/src/sharing.rs | 143 +++++- crates/disktree-core/src/sharing/reclaim.rs | 480 ++++++++++++++++++ crates/disktree-core/src/tree.rs | 11 +- docs/performance/apfs-accounting/README.md | 105 ++++ .../apfs-accounting/budget-green.log | 10 + .../apfs-accounting/regression-red.log | 57 +++ 17 files changed, 1438 insertions(+), 109 deletions(-) create mode 100644 crates/disktree-core/src/clone_cache.rs create mode 100644 crates/disktree-core/src/sharing/reclaim.rs create mode 100644 docs/performance/apfs-accounting/README.md create mode 100644 docs/performance/apfs-accounting/budget-green.log create mode 100644 docs/performance/apfs-accounting/regression-red.log diff --git a/README.md b/README.md index c7135bce..2800c720 100644 --- a/README.md +++ b/README.md @@ -96,13 +96,18 @@ What is different from Linux: makes) remain counted in full in the treemap. The scan line separately reports bytes in cloned files and estimated repeated data-fork allocation. The selection, review and permanent-delete confirmation estimate savings - using private bytes for clones. Unknown private sizes contribute zero. - These estimates exclude shared extents even when all copies are marked, - so deleting copies together can free more. Snapshots can retain blocks, + from the accepted set of marks together. Complete clone groups credit + shared data once; a remaining hardlink retains its file data. Unknown or + modified sharing and allocation outside the data fork supply a range, with only its lower bound used by the + volume meter. Unknown private sizes contribute zero to that bound. + Snapshots and open files can retain blocks, and changing files can stale the estimate; rescan to refresh it. The final result still measures the volume's actual free-space change. Clone hints come from bulk directory reads; only potentially shared files - need a private-size query. Unsupported filesystems or kernels fall back to + need a private-size query. A bounded, sharded cache reuses confirmed full + clones within their observed reference count, separately on each real + APFS volume and each scan. Modified clones are measured individually. + Unsupported filesystems or kernels fall back to the ordinary listing without clone detection. Apparent-size scans do not collect clone metadata. - **Time Machine's local snapshots** are not files and do not appear; they diff --git a/crates/disktree-app/src/marks.rs b/crates/disktree-app/src/marks.rs index 76348437..0244f9f6 100644 --- a/crates/disktree-app/src/marks.rs +++ b/crates/disktree-app/src/marks.rs @@ -97,7 +97,7 @@ impl Marks { /// Re-read sizes from a freshly scanned tree. Missing paths stay marked /// with zero bytes, so they cannot claim space that is already gone. - pub fn refresh(&mut self, root_path: &Path, root: &Node, metric: Metric) { + pub fn refresh(&mut self, root_path: &Path, root: &Node, _metric: Metric) { self.resolved = None; let mut resolver = Resolver::new(root_path, root); let mut resolved = Vec::with_capacity(self.items.len()); @@ -106,7 +106,7 @@ impl Marks { work::step(); match resolver.resolve(&item.path) { Some((_, node)) => resolved.push(Target { - bytes: node.value(metric), + bytes: node.bytes, is_dir: node.is_dir(), hidden: is_hidden(&item.path), path: item.path.clone(), diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs index 9a398562..96bced96 100644 --- a/crates/disktree-app/src/state.rs +++ b/crates/disktree-app/src/state.rs @@ -4,10 +4,10 @@ //! decisions — what is selected, what a mark means, what a key does, when to //! re-scan — live here so they can be reasoned about in one place. -use std::cell::Cell; +use std::cell::{Cell, RefCell}; use std::path::{Path, PathBuf}; use std::rc::Rc; -use std::sync::Arc; +use std::sync::{Arc, Weak}; use std::time::{Duration, Instant}; use disktree_core::access::file_table_readable; @@ -321,6 +321,15 @@ pub struct RunSummary { pub failed: usize, } +/// Scan-derived display estimates; never cached removal authorization. +#[derive(Default)] +struct AccountingCache { + tree: Weak, + root: PathBuf, + // Total and scanned-volume subsets only: bounded by the current marks. + estimates: Vec<(Vec, disktree_core::sharing::Estimate)>, +} + /// Everything the app knows and everything it can do. pub struct Disktree { /// The directory the tree was scanned from. @@ -360,6 +369,7 @@ pub struct Disktree { pub transition: Option, pub layout_options: LayoutOptions, cache: Option, + accounting: RefCell, /// Mouse position in treemap-local pixels, for hit-testing and the tooltip. pub pointer: Option>, @@ -495,6 +505,7 @@ impl Disktree { ..LayoutOptions::default() }, cache: None, + accounting: RefCell::new(AccountingCache::default()), pointer: None, treemap_origin: Rc::new(Cell::new(Point::new(px(0.), px(0.)))), treemap_size: Rc::new(Cell::new(size(px(0.), px(0.)))), @@ -1749,7 +1760,7 @@ impl Disktree { Some(Target { hidden: is_hidden(&path), path, - bytes: node.value(self.options.metric), + bytes: node.bytes, is_dir: node.is_dir(), }) } @@ -1768,11 +1779,55 @@ impl Disktree { /// The plan the review screen shows and the removal runs. pub fn plan(&self) -> Plan { let mut plan = plan(self.marks.items(), &self.root_path); - if let Some(tree) = self.tree() { - plan.estimate_reclaim(|target| { - crate::marks::find(&self.root_path, tree, &target.path) - .map(|node| node.sharing().reclaimable(node.bytes)) + if let Some(tree) = &self.tree { + let mut cached = self.accounting.borrow_mut(); + if !cached.tree.ptr_eq(&Arc::downgrade(tree)) + || cached.root != self.root_path + { + *cached = AccountingCache { + tree: Arc::downgrade(tree), + root: self.root_path.clone(), + estimates: Vec::new(), + }; + } + let mut resolver = + crate::marks::Resolver::new(&self.root_path, tree); + let mut next = Vec::new(); + plan.estimate_selection(|targets| { + let paths: Vec<_> = + targets.iter().map(|target| target.path.clone()).collect(); + let value = cached + .estimates + .iter() + .find(|(old, _)| *old == paths) + .map_or_else( + || { + let nodes: Vec<_> = targets + .iter() + .filter_map(|target| { + resolver + .resolve(&target.path) + .map(|(_, node)| node) + }) + .collect(); + if let [node] = nodes.as_slice() { + node.sharing().estimate.unwrap_or( + disktree_core::sharing::Estimate { + lower: node.bytes, + upper: node.bytes, + partial: false, + }, + ) + } else { + disktree_core::sharing::estimate(nodes) + } + }, + |(_, value)| *value, + ); + next.push((paths, value)); + value }); + cached.estimates = next; } plan } diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index 9c199af4..db8f2937 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -1772,6 +1772,40 @@ fn apfs_clones_use_private_bytes_in_review_and_refresh( }); draw(cx); update(&view, cx, |app, _| app.confirm_open = false); + // Separate marks complete the group. Changing tile weights must never + // turn the review's byte values into file counts. + update(&view, cx, |app, cx| { + app.options.metric = disktree_core::tree::Metric::Files; + let index = app + .tree() + .expect("tree") + .children + .iter() + .position(|node| node.name.as_ref() == "original") + .expect("original"); + app.toggle_mark(&[index], cx); + let plan = app.plan(); + assert_eq!(plan.bytes(), 16 * 1024 * 1024); + assert_eq!(plan.reclaimable_bytes(), 8 * 1024 * 1024); + assert_eq!(plan.possible_bytes(), 8 * 1024 * 1024); + }); + draw(cx); + crate::marks::work::reset(); + read(&view, cx, |app| { + for _ in 0..4 { + assert_eq!(app.plan().reclaimable_bytes(), 8 * 1024 * 1024); + } + }); + assert_eq!( + crate::marks::work::RESOLUTION.get(), + 0, + "warm plans must not resolve paths again" + ); + update(&view, cx, |app, _| { + app.marks.remove(&root.join("original")); + assert_eq!(app.plan().reclaimable_bytes(), 0); + app.options.metric = disktree_core::tree::Metric::Bytes; + }); // Break sharing and refresh the same absolute-path mark. std::fs::remove_file(root.join("copies/clone")).expect("remove clone"); std::fs::write(root.join("copies/clone"), vec![0x31; 8 * 1024 * 1024]) diff --git a/crates/disktree-app/src/views.rs b/crates/disktree-app/src/views.rs index be84d04e..2041d91e 100644 --- a/crates/disktree-app/src/views.rs +++ b/crates/disktree-app/src/views.rs @@ -267,7 +267,7 @@ fn delete_dialog( }; let body = format!( "Estimated to free ≈ {}. Shared blocks and snapshots may retain space. Deleted files can\u{2019}t be recovered; move them to the trash if you might need them again.", - human_bytes(plan.reclaimable_bytes()) + savings(plan.reclaimable_bytes(), plan.possible_bytes()) ); let confirm = cx.entity().downgrade(); let cancel = confirm.clone(); @@ -1454,14 +1454,17 @@ fn selection_section( cx, )) .child(widgets::bar(share, highlight, cx)) - .when(node.sharing().files > 0, |this| { + .when(node.sharing().estimate.is_some(), |this| { this.child( div() .text_size(text::CAPTION) .text_color(theme.secondary) .child(format!( - "APFS clones · frees ≈ {} excluding shared blocks", - human_bytes(node.sharing().reclaimable(node.bytes)) + "Estimated space freed · ≈ {}", + savings( + node.sharing().reclaimable(node.bytes), + node.sharing().possible(node.bytes) + ) )), ) .when(node.sharing().unknown_files > 0, |this| { @@ -2612,7 +2615,7 @@ fn review( &format!( "{} marked \u{00b7} ≈ {} to free", app.marks.len(), - human_bytes(plan.reclaimable_bytes()) + savings(plan.reclaimable_bytes(), plan.possible_bytes()) ), &theme, cx, @@ -2832,7 +2835,7 @@ fn review_summary( } else { "Estimated space freed" }, - human_bytes(plan.reclaimable_bytes()), + savings(plan.reclaimable_bytes(), plan.possible_bytes()), cx, )) .children(left_out), @@ -2841,7 +2844,7 @@ fn review_summary( if cfg!(target_os = "macos") { panel = panel.child(div().text_size(text::CAPTION) .text_color(theme.secondary) - .child("Estimate excludes shared clone blocks. Removing all copies together can free more; snapshots can retain space. Rescan to update estimates.")); + .child("Complete clone groups count shared data once. Remaining hardlinks retain file data. Ranges show unmeasured sharing; snapshots and open files can retain space. Rescan to update estimates.")); } if let Some(volume) = app.space { panel = panel.child( @@ -3667,3 +3670,12 @@ fn help_overlay(app: &Disktree, cx: &gpui_kit::App) -> Div { ), ) } + +/// Both ends are scan-derived; the meter uses the lower bound only. +fn savings(lower: u64, upper: u64) -> String { + if lower == upper { + human_bytes(lower) + } else { + format!("{}–{}", human_bytes(lower), human_bytes(upper)) + } +} diff --git a/crates/disktree-core/examples/scan_accounting.rs b/crates/disktree-core/examples/scan_accounting.rs index 5608c205..8f59716b 100644 --- a/crates/disktree-core/examples/scan_accounting.rs +++ b/crates/disktree-core/examples/scan_accounting.rs @@ -22,7 +22,7 @@ fn main() -> std::io::Result<()> { }, )?; println!( - "seconds={:.3} files={} listed={} clone_files={} clone_bytes={} duplicate_bytes={} private_bytes={} unknown_clones={}", + "seconds={:.3} files={} listed={} clone_files={} clone_bytes={} duplicate_bytes={} private_bytes={} unknown_clones={} reclaim_lower={} reclaim_upper={}", started.elapsed().as_secs_f64(), tree.files, tree.bytes, @@ -30,7 +30,9 @@ fn main() -> std::io::Result<()> { tree.sharing().bytes, tree.sharing().duplicate_bytes, tree.sharing().private_bytes, - tree.sharing().unknown_files + tree.sharing().unknown_files, + tree.sharing().reclaimable(tree.bytes), + tree.sharing().possible(tree.bytes) ); Ok(()) } diff --git a/crates/disktree-core/src/clone_cache.rs b/crates/disktree-core/src/clone_cache.rs new file mode 100644 index 00000000..a9171872 --- /dev/null +++ b/crates/disktree-core/src/clone_cache.rs @@ -0,0 +1,241 @@ +//! Per-scan reuse of confirmed full-clone metadata, without a serial consumer. + +use std::hash::{Hash, Hasher as _}; +use std::sync::Mutex; + +use rustc_hash::{FxHashMap, FxHashSet, FxHasher}; + +use crate::sharing::CloneInfo; + +const SHARDS: usize = 64; +const GROUPS_PER_SHARD: usize = 1_000_000 / SHARDS; +const ALIASES_PER_SHARD: usize = GROUPS_PER_SHARD; + +struct Sample { + info: CloneInfo, + remaining: u32, + aliases: FxHashSet, +} + +#[derive(Default)] +struct Shard { + groups: FxHashMap<(u64, u64), Sample>, + aliases: usize, +} + +pub struct Cache { + shards: [Mutex; SHARDS], + group_limit: usize, + alias_limit: usize, +} + +impl Default for Cache { + fn default() -> Self { + Self { + shards: std::array::from_fn(|_| Mutex::new(Shard::default())), + group_limit: GROUPS_PER_SHARD, + alias_limit: ALIASES_PER_SHARD, + } + } +} + +impl Cache { + /// A reference count is a reuse budget, not a permanent fact. A later + /// member past that budget is measured again, catching clone growth. + /// Modified clones and unknown physical volumes are never reused. + pub fn get( + &self, + device: u64, + id: u64, + inode: u64, + links: u64, + allocated: u64, + measure: impl FnOnce() -> CloneInfo, + ) -> CloneInfo { + let key = (device, id); + let mut hash = FxHasher::default(); + key.hash(&mut hash); + let index = usize::try_from(hash.finish() % SHARDS as u64) + .expect("shard index fits"); + if device != 0 && id != 0 && inode != 0 { + let mut shard = self.shards[index].lock().expect("clone cache"); + let alias_room = shard.aliases < self.alias_limit; + if let Some(sample) = shard + .groups + .get_mut(&key) + .filter(|sample| sample.info.data_bytes == allocated) + { + if sample.aliases.contains(&inode) { + return sample.info; + } + if sample.remaining > 0 && (links == 1 || alias_room) { + sample.remaining -= 1; + let info = sample.info; + if links > 1 && sample.aliases.insert(inode) { + shard.aliases += 1; + } + return info; + } + } + drop(shard); + } + // Filesystem calls never hold a shard lock. Racing first samples + // may duplicate a query; they cannot funnel directory workers into + // one slow reader. Each scan owns its cache, including re-scans. + let info = measure(); + if device == 0 + || inode == 0 + || info.device != device + || info.id != id + || info.references < 2 + || info.private_bytes != Some(0) + { + return info; + } + let mut shard = self.shards[index].lock().expect("clone cache"); + if let Some(old) = shard.groups.remove(&key) { + shard.aliases -= old.aliases.len(); + } + if shard.groups.len() < self.group_limit + && (links == 1 || shard.aliases < self.alias_limit) + { + let mut aliases = FxHashSet::default(); + if links > 1 { + aliases.insert(inode); + shard.aliases += 1; + } + shard.groups.insert( + key, + Sample { + info, + remaining: info.references - 1, + aliases, + }, + ); + } + info + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + fn sample(device: u64, id: u64, references: u32) -> CloneInfo { + CloneInfo { + device, + id, + references, + private_bytes: Some(0), + data_bytes: 4096, + } + } + + #[test] + fn volume_scope_scan_lifetime_and_memory_caps_limit_reuse() { + let cache = Cache { + group_limit: 1, + alias_limit: 1, + ..Cache::default() + }; + let calls = Cell::new(0); + for device in 1..=128 { + for inode in 1..=3 { + let value = cache.get(device, 42, inode, 2, 4096, || { + calls.set(calls.get() + 1); + sample(device, 42, 3) + }); + assert_eq!(value.device, device); + } + } + for shard in &cache.shards { + let shard = shard.lock().expect("cache"); + assert!(shard.groups.len() <= 1); + assert!(shard.aliases <= 1); + drop(shard); + } + assert!(calls.get() > 128, "capped aliases must be measured"); + let fresh = Cache::default(); + let mut measured = false; + fresh.get(1, 42, 1, 1, 4096, || { + measured = true; + sample(1, 42, 3) + }); + assert!(measured, "new scans must not reuse previous observations"); + } + + #[test] + fn different_all_fork_allocation_requires_a_fresh_private_measurement() { + let cache = Cache::default(); + cache.get(1, 42, 1, 1, 4096, || sample(1, 42, 2)); + let mut measured = false; + let info = cache.get(1, 42, 2, 1, 8192, || { + measured = true; + let mut info = sample(1, 42, 2); + info.private_bytes = Some(4096); + info + }); + assert!( + measured, + "a resource fork must not inherit another member's zero private size" + ); + assert_eq!(info.private_bytes, Some(4096)); + } + + #[test] + fn reference_budget_refreshes_and_hardlink_aliases_do_not_spend_it() { + let cache = Cache::default(); + let calls = Cell::new(0); + for (inode, links, refs) in [(1, 2, 2), (1, 2, 2), (2, 1, 2), (3, 1, 3)] + { + let info = cache.get(1, 42, inode, links, 4096, || { + calls.set(calls.get() + 1); + sample(1, 42, refs) + }); + assert_eq!(info.references, refs); + } + assert_eq!(calls.get(), 2); + } + + #[test] + fn unknown_volumes_and_modified_members_are_measured_individually() { + for device in [0, 1] { + let cache = Cache::default(); + let calls = Cell::new(0); + for inode in 1..=8 { + cache.get(device, 42, inode, 1, 4096, || { + calls.set(calls.get() + 1); + let mut info = sample(device, 42, 8); + info.private_bytes = Some(4096); + info + }); + } + assert_eq!(calls.get(), 8); + } + } + + #[test] + fn clone_groups_bound_queries_as_both_groups_and_members_grow() { + for n in [128_u32, 1024] { + let cache = Cache::default(); + let calls = Cell::new(0); + for group in 1..=n { + for inode in 1..=n { + cache.get( + 1, + u64::from(group), + u64::from(inode), + 1, + 4096, + || { + calls.set(calls.get() + 1); + sample(1, u64::from(group), n) + }, + ); + } + } + assert_eq!(calls.get(), n, "one query per confirmed group"); + } + } +} diff --git a/crates/disktree-core/src/lib.rs b/crates/disktree-core/src/lib.rs index 41b82362..8ecd8a45 100644 --- a/crates/disktree-core/src/lib.rs +++ b/crates/disktree-core/src/lib.rs @@ -7,6 +7,8 @@ pub mod access; pub mod classify; +#[cfg(any(target_os = "macos", test))] +mod clone_cache; pub mod export; pub mod filter; pub mod insights; diff --git a/crates/disktree-core/src/macos.rs b/crates/disktree-core/src/macos.rs index b98ed6d1..c97ffed3 100644 --- a/crates/disktree-core/src/macos.rs +++ b/crates/disktree-core/src/macos.rs @@ -30,10 +30,13 @@ use crate::sharing::CloneInfo; const BUFFER_BYTES: usize = 64 * 1024; const REFCNT: u32 = 0x1000; -const MAY_SHARE: u64 = 0x1 | 0x40; +const FULL_SHARE: u64 = 0x40; +const MAY_SHARE: u64 = 0x1 | FULL_SHARE; const EXTENDED: u32 = libc::FSOPT_ATTR_CMN_EXTENDED; const RETURNED: u32 = libc::ATTR_CMN_RETURNED_ATTRS; -const HINTS: u32 = libc::ATTR_CMNEXT_CLONEID | libc::ATTR_CMNEXT_EXT_FLAGS; +const REAL_DEVICE: u32 = 0x40; +const HINTS: u32 = + REAL_DEVICE | libc::ATTR_CMNEXT_CLONEID | libc::ATTR_CMNEXT_EXT_FLAGS; // From and , which the libc crate does not cover. const ATTR_BIT_MAP_COUNT: u16 = 5; @@ -127,7 +130,7 @@ pub struct Entry { inode: u64, links: u32, flags: u32, - hints: Option<(u64, u64)>, + hints: Option<(u64, u64, u64)>, } impl Entry { @@ -174,8 +177,13 @@ impl Entry { } /// `(st_dev, st_ino)`: the same for two names of one file. - pub const fn identity(&self) -> (u64, u64) { - (self.device, self.inode) + pub fn identity(&self) -> (u64, u64) { + ( + self.hints.map_or(self.device, |(device, _, _)| { + if device == 0 { self.device } else { device } + }), + self.inode, + ) } pub const fn device(&self) -> u64 { @@ -183,6 +191,10 @@ impl Entry { } /// Whether the file has more than one name. + pub const fn links(&self) -> u64 { + self.links as u64 + } + pub const fn shared(&self) -> bool { self.links > 1 } @@ -195,17 +207,29 @@ impl Entry { /// Only clone candidates pay for private-size reads; ordinary metadata /// and identity remain the values returned by the bulk listing. - pub fn clone_info(&self, dir: &Path) -> Option> { - let (id, flags) = self.hints?; + pub fn clone_info( + &self, + dir: &Path, + cache: &crate::clone_cache::Cache, + ) -> Option { + let (device, id, flags) = self.hints?; if self.kind != Kind::File || flags & MAY_SHARE == 0 { return None; } - Some(Box::new(private_info( - &self.path(dir), - self.device, - self.inode, + let measure = || private_info(&self.path(dir), device, self.inode, id); + if flags & FULL_SHARE == 0 { + return Some(measure()); + } + // A resource fork can differ even when the data fork is cloned. + // Reuse only when the listing agrees with the cached data allocation. + Some(cache.get( + device, id, - ))) + self.inode, + u64::from(self.links), + self.allocated, + measure, + )) } fn named(name: &[u8]) -> (Box, Option>) { @@ -524,6 +548,11 @@ impl<'a> Cursor<'a> { } else { return Ok(None); }; + let real_device = if extended & REAL_DEVICE != 0 { + u64::from(self.u32()?) + } else { + 0 + }; let clone_id = if extended & libc::ATTR_CMNEXT_CLONEID != 0 { Some(self.u64()?) } else { @@ -534,7 +563,9 @@ impl<'a> Cursor<'a> { } else { None }; - let hints = clone_id.zip(clone_flags); + let hints = clone_id + .zip(clone_flags) + .map(|(id, flags)| (real_device, id, flags)); let (name, exact) = Entry::named(name); Ok(Some(Entry { name, @@ -763,9 +794,20 @@ fn private_info(path: &Path, device: u64, inode: u64, id: u64) -> CloneInfo { { return info; } - // Packed order: fileid, dataallocsize, privatesize, cloneid, flags, refcnt. + // Packed order includes realdevid before cloneid, even when unavailable. + // PACK_INVAL_ATTRS reserves its slot; ordinary st_dev can alias APFS + // System and Data volumes and must never substitute for this identity. + let real_device = if extended & REAL_DEVICE != 0 { + u64::from(clone_read_u32(buffer, 48).unwrap_or(0)) + } else { + 0 + }; + if device != 0 && device != real_device { + return info; + } + info.device = real_device; if extended & libc::ATTR_CMNEXT_CLONEID == 0 - || clone_read_u64(buffer, 48) != Some(id) + || clone_read_u64(buffer, 52) != Some(id) { return info; } @@ -776,7 +818,7 @@ fn private_info(path: &Path, device: u64, inode: u64, id: u64) -> CloneInfo { info.data_bytes = clone_read_u64(buffer, 32).unwrap_or(0); } if extended & REFCNT != 0 { - info.references = clone_read_u32(buffer, 64).unwrap_or(0); + info.references = clone_read_u32(buffer, 68).unwrap_or(0); } info } @@ -789,10 +831,10 @@ fn clone_read_u64(bytes: &[u8], at: usize) -> Option { } /// Exceptional path for followed symlinks; ordinary entries use bulk hints. -pub fn clone_info(path: &Path, meta: &Metadata) -> Option> { +pub fn clone_info(path: &Path, meta: &Metadata) -> Option { let path_c = CString::new(path.as_os_str().as_bytes()).ok()?; let mut request = clone_attrs(0, 0, HINTS); - let mut buffer = [0_u8; 40]; + let mut buffer = [0_u8; 48]; // SAFETY: valid path, ABI request and writable buffer of the stated size. let result = unsafe { libc::getattrlist( @@ -805,16 +847,16 @@ pub fn clone_info(path: &Path, meta: &Metadata) -> Option> { }; if result != 0 || clone_read_u32(&buffer, 20)? & HINTS != HINTS - || clone_read_u64(&buffer, 32)? & MAY_SHARE == 0 + || clone_read_u64(&buffer, 36)? & MAY_SHARE == 0 { return None; } - Some(Box::new(private_info( + Some(private_info( path, - meta.dev(), + u64::from(clone_read_u32(&buffer, 24)?), meta.ino(), - clone_read_u64(&buffer, 24)?, - ))) + clone_read_u64(&buffer, 28)?, + )) } #[cfg(test)] @@ -872,6 +914,72 @@ mod clone_tests { assert_eq!(apparent.sharing().files, 0); } + #[test] + fn real_clone_hardlinks_and_followed_names_have_set_level_savings() { + let temp = fixture(); + let original = temp.path().join("original"); + fs::hard_link(&original, temp.path().join("alias")).expect("hardlink"); + std::os::unix::fs::symlink(&original, temp.path().join("followed")) + .expect("symlink"); + let tree = scan( + temp.path(), + ScanOptions { + follow_links: true, + ..ScanOptions::default() + }, + ) + .expect("scan"); + let by_name = |name: &str| { + tree.children + .iter() + .find(|node| node.name.as_ref() == name) + .expect("entry") + }; + let original = by_name("original"); + let alias = by_name("alias"); + let clone = by_name("clone"); + let followed = by_name("followed"); + for node in [original, alias, clone, followed] { + assert_eq!(crate::sharing::estimate([node]).lower, 0); + } + assert_eq!(crate::sharing::estimate([original, clone]).lower, 0); + assert_eq!( + crate::sharing::estimate([original, alias, clone]).lower, + 8 * 1024 * 1024 + ); + assert_eq!(tree.sharing().reclaimable(tree.bytes), 8 * 1024 * 1024); + for node in [original, alias, clone] { + assert_ne!( + node.allocation + .as_deref() + .expect("allocation") + .clone + .expect("clone") + .device, + 0 + ); + } + } + + #[test] + fn resource_fork_allocation_remains_in_the_range_for_full_clones() { + let temp = fixture(); + let fork = temp.path().join("clone/..namedfork/rsrc"); + fs::write(fork, vec![0x31; 4096]).expect("resource fork"); + let tree = scan(temp.path(), ScanOptions::default()).expect("scan"); + assert_eq!(tree.sharing().private_bytes, 0); + assert_eq!(tree.sharing().reclaimable(tree.bytes), 8 * 1024 * 1024); + assert_eq!(tree.sharing().possible(tree.bytes), 8 * 1024 * 1024 + 4096); + assert_eq!(tree.sharing().duplicate_bytes, 8 * 1024 * 1024); + let clone = tree + .children + .iter() + .find(|node| node.name.as_ref() == "clone") + .expect("clone"); + assert_eq!(clone.sharing().reclaimable(clone.bytes), 0); + assert_eq!(clone.sharing().possible(clone.bytes), 4096); + } + #[test] fn modified_clones_keep_private_bytes_and_hardlinks_are_not_clones() { let temp = fixture(); diff --git a/crates/disktree-core/src/removal.rs b/crates/disktree-core/src/removal.rs index df02e984..f6434a34 100644 --- a/crates/disktree-core/src/removal.rs +++ b/crates/disktree-core/src/removal.rs @@ -94,6 +94,7 @@ pub struct Plan { /// Clone estimates are separate from listed allocation and its volumes. reclaimable_bytes: Option, reclaim_estimate: Option, + possible_bytes: Option, } impl Plan { @@ -120,24 +121,42 @@ impl Plan { self.reclaimable_bytes.unwrap_or_else(|| self.bytes()) } - /// Apply scan-derived savings without lending another volume's space to - /// this one's meter. Missing nodes contribute no unmeasured saving. - pub fn estimate_reclaim( + pub fn possible_bytes(&self) -> u64 { + self.possible_bytes + .unwrap_or_else(|| self.reclaimable_bytes()) + } + + /// Evaluate the accepted set together, preserving the guard's volume + /// attribution. An external volume can never inflate this volume's meter. + pub fn estimate_selection( &mut self, - estimate: impl Fn(&Target) -> Option, + mut estimate: impl FnMut(&[&Target]) -> crate::sharing::Estimate, ) { - let mut total = 0_u64; - let mut scanned = 0_u64; - for (target, attribution) in self.targets.iter().zip(&self.attribution) - { - let bytes = estimate(target).unwrap_or(0).min(target.bytes); - total = total.saturating_add(bytes); - if *attribution == crate::space::Attribution::Scanned { - scanned = scanned.saturating_add(bytes); - } - } - self.reclaimable_bytes = Some(total); - self.reclaim_estimate = Some(scanned); + let all: Vec<_> = self.targets.iter().collect(); + let total = if all.is_empty() { + crate::sharing::Estimate::default() + } else { + estimate(&all) + }; + let scanned: Vec<_> = self + .targets + .iter() + .zip(&self.attribution) + .filter_map(|(target, attribution)| { + (*attribution == crate::space::Attribution::Scanned) + .then_some(target) + }) + .collect(); + let local = if scanned.is_empty() { + crate::sharing::Estimate::default() + } else if scanned.len() == all.len() { + total + } else { + estimate(&scanned) + }; + self.reclaimable_bytes = Some(total.lower); + self.possible_bytes = Some(total.upper); + self.reclaim_estimate = Some(local.lower); } /// Bytes the targets free on other volumes, which the meter leaves out. @@ -1929,7 +1948,14 @@ mod tests { assert_eq!(plan.foreign(), 600, "another disk, and a snapshot"); assert_eq!(plan.unattributed(), 0); let mut estimated = plan; - estimated.estimate_reclaim(|target| Some(target.bytes / 2)); + estimated.estimate_selection(|targets| { + let bytes = targets.iter().map(|target| target.bytes / 2).sum(); + crate::sharing::Estimate { + lower: bytes, + upper: bytes, + partial: false, + } + }); assert_eq!(estimated.reclaimable_bytes(), 350); assert_eq!(estimated.reclaim(), 50, "only this volume's estimate"); assert_eq!(estimated.foreign(), 600, "listed foreign allocation"); @@ -1966,7 +1992,11 @@ mod tests { assert_eq!(plan.foreign(), 0); assert_eq!(plan.unattributed(), 50, "no saving may be claimed"); let mut estimated = plan; - estimated.estimate_reclaim(|_| Some(25)); + estimated.estimate_selection(|_| crate::sharing::Estimate { + lower: 25, + upper: 50, + partial: false, + }); assert_eq!(estimated.reclaimable_bytes(), 25); assert_eq!( estimated.reclaim(), diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rs index 88d326a4..c2e35511 100644 --- a/crates/disktree-core/src/scan.rs +++ b/crates/disktree-core/src/scan.rs @@ -230,6 +230,8 @@ impl ScanHandle { visited_dirs: Mutex::new(FxHashSet::default()), root: Mutex::new(None), volume: OnceLock::new(), + #[cfg(target_os = "macos")] + clone_cache: crate::clone_cache::Cache::default(), }); let (sender, result) = mpsc::channel(); @@ -280,6 +282,8 @@ pub fn scan(root: &Path, options: ScanOptions) -> io::Result { visited_dirs: Mutex::new(FxHashSet::default()), root: Mutex::new(None), volume: OnceLock::new(), + #[cfg(target_os = "macos")] + clone_cache: crate::clone_cache::Cache::default(), }); let node = scan_blocking(root, &context)?; progress.finish(); @@ -308,6 +312,8 @@ struct WalkContext { /// cannot leave it: every listing takes it instead of asking. See /// [`crate::windows::walk_volume`]. volume: OnceLock, + #[cfg(target_os = "macos")] + clone_cache: crate::clone_cache::Cache, } impl WalkContext { @@ -356,8 +362,8 @@ impl WalkContext { } Listing::Leaf(kind) => kind, }; - match entry.facts(dir, self.options.apparent_size) { - Ok(facts) => self.leaf(entry.take_name(), kind, &facts), + match entry.facts(dir, self.options.apparent_size, self) { + Ok(facts) => self.leaf(entry.take_name(), kind, facts), Err(error) => { self.progress.record_error(&entry.entry_path(dir), &error); Classified::Skipped @@ -424,6 +430,7 @@ impl WalkContext { Classified::Subdirectory { path, name: entry.take_name(), + followed: false, } } @@ -437,7 +444,7 @@ impl WalkContext { Classified::Entry(leaf_node( name, NodeKind::Symlink, - &facts, + facts, facts.shared, )) } @@ -473,6 +480,7 @@ impl WalkContext { return Classified::Subdirectory { path: path.to_path_buf(), name, + followed: true, }; } @@ -486,15 +494,11 @@ impl WalkContext { .ok() .and_then(|target| crate::macos::clone_info(&target, &meta)); } - self.leaf(name, kind_of(&meta, meta.file_type()), &facts) + facts.followed = true; + self.leaf(name, kind_of(&meta, meta.file_type()), facts) } - fn leaf( - &self, - name: Box, - kind: NodeKind, - facts: &Facts, - ) -> Classified { + fn leaf(&self, name: Box, kind: NodeKind, facts: Facts) -> Classified { // A followed link reaches a file a second way, whatever its link // count says. let track = self.options.follow_links || facts.shared; @@ -521,7 +525,12 @@ trait Listed { fn take_name(&mut self) -> Box; fn listing(&self) -> io::Result; /// Size, identity and age of a leaf. - fn facts(&self, _dir: &Path, apparent_size: bool) -> io::Result; + fn facts( + &self, + _dir: &Path, + apparent_size: bool, + _context: &WalkContext, + ) -> io::Result; /// What a directory is, read before the walk enters it. fn directory(&self) -> io::Result; /// Hidden by an attribute rather than by a leading dot: on Windows, @@ -557,6 +566,7 @@ enum Listing { } /// What a leaf contributes. +#[derive(Clone, Copy)] struct Facts { size: u64, identity: Option<(u64, u64)>, @@ -564,7 +574,9 @@ struct Facts { /// The file may have another name the walk could meet: its identity is /// worth keeping for hardlink de-duplication. shared: bool, - clone_info: Option>, + clone_info: Option, + links: u64, + followed: bool, } impl Facts { @@ -575,10 +587,23 @@ impl Facts { modified: modified_seconds(meta), shared: shares_inode(meta), clone_info: None, + links: link_count(meta), + followed: false, } } } +#[cfg(unix)] +fn link_count(meta: &Metadata) -> u64 { + use std::os::unix::fs::MetadataExt as _; + meta.nlink() +} + +#[cfg(not(unix))] +fn link_count(_meta: &Metadata) -> u64 { + 1 +} + /// A standard directory entry with its name decoded once, up front. #[cfg(not(any(windows, target_os = "macos")))] struct Named { @@ -613,7 +638,12 @@ impl Listed for Named { }) } - fn facts(&self, _dir: &Path, apparent_size: bool) -> io::Result { + fn facts( + &self, + _dir: &Path, + apparent_size: bool, + _context: &WalkContext, + ) -> io::Result { let meta = self.entry.metadata()?; Ok(Facts::of(&meta, apparent_size)) } @@ -649,7 +679,12 @@ impl Listed for crate::macos::Entry { }) } - fn facts(&self, dir: &Path, apparent_size: bool) -> io::Result { + fn facts( + &self, + dir: &Path, + apparent_size: bool, + context: &WalkContext, + ) -> io::Result { Ok(Facts { size: if apparent_size { self.apparent() @@ -662,8 +697,10 @@ impl Listed for crate::macos::Entry { clone_info: if apparent_size { None } else { - self.clone_info(dir) + self.clone_info(dir, &context.clone_cache) }, + links: self.links(), + followed: false, }) } @@ -697,7 +734,12 @@ impl Listed for crate::windows::Entry { }) } - fn facts(&self, _dir: &Path, apparent_size: bool) -> io::Result { + fn facts( + &self, + _dir: &Path, + apparent_size: bool, + _context: &WalkContext, + ) -> io::Result { Ok(Facts { size: if apparent_size { self.apparent() @@ -710,6 +752,8 @@ impl Listed for crate::windows::Entry { // every file keeps one. shared: true, clone_info: None, + links: 1, + followed: false, }) } @@ -768,7 +812,11 @@ fn list( /// What a directory entry turned out to be. enum Classified { /// Descend into this directory on a new task. - Subdirectory { path: PathBuf, name: Box }, + Subdirectory { + path: PathBuf, + name: Box, + followed: bool, + }, /// A leaf that contributes size. Entry(Node), /// Filtered out, unreadable, or a symlink we chose not to follow. @@ -787,6 +835,7 @@ struct PendingDir { partial: Mutex, read_error: AtomicBool, depth: usize, + followed: bool, } /// What a directory's node is made of before it is built: the name the @@ -815,6 +864,7 @@ impl PendingDir { }), read_error: AtomicBool::new(false), depth, + followed: false, } } @@ -833,7 +883,15 @@ impl PendingDir { files: 0, dirs: 1, inode: None, - clone_info: None, + allocation: self.followed.then(|| { + Box::new(crate::sharing::Allocation { + bytes: 0, + identity: None, + links: 1, + followed: true, + clone: None, + }) + }), sharing: None, read_error: self.read_error.load(Ordering::Relaxed), modified: 0, @@ -1111,14 +1169,20 @@ fn walk(dir: &Arc, context: &WalkContext) -> Vec> { match entry { Ok(mut entry) => { match context.classify(&dir.path, &mut entry) { - Classified::Subdirectory { path, name } => { + Classified::Subdirectory { + path, + name, + followed, + } => { tally.dirs += 1; - subdirs.push(Arc::new(PendingDir::new( + let mut child = PendingDir::new( path, name, Some(Arc::clone(dir)), dir.depth + 1, - ))); + ); + child.followed = followed; + subdirs.push(Arc::new(child)); } Classified::Entry(node) => { tally.add(&node); @@ -1221,7 +1285,7 @@ fn finish_tree( fn leaf_node( name: Box, kind: NodeKind, - facts: &Facts, + facts: Facts, track: bool, ) -> Node { let mut node = Node::entry(name, kind, facts.size); @@ -1231,7 +1295,19 @@ fn leaf_node( .and_then(|(device, file)| Some((device, NonZeroU64::new(file)?))); } node.modified = facts.modified; - node.clone_info.clone_from(&facts.clone_info); + if facts.clone_info.is_some() + || facts.links > 1 + || facts.followed + || (track && cfg!(unix)) + { + node.allocation = Some(Box::new(crate::sharing::Allocation { + bytes: facts.size, + identity: facts.identity, + links: facts.links, + followed: facts.followed, + clone: facts.clone_info, + })); + } node } @@ -1480,6 +1556,8 @@ mod tests { visited_dirs: Mutex::new(FxHashSet::default()), root: Mutex::new(None), volume: OnceLock::new(), + #[cfg(target_os = "macos")] + clone_cache: crate::clone_cache::Cache::default(), }); let root = Arc::new(PendingDir::new( temp.path().to_path_buf(), @@ -1760,6 +1838,8 @@ mod tests { visited_dirs: Mutex::new(FxHashSet::default()), root: Mutex::new(None), volume: OnceLock::new(), + #[cfg(target_os = "macos")] + clone_cache: crate::clone_cache::Cache::default(), }); let tree = scan_blocking(root, &context).expect("scan still succeeds"); let _ = fs::set_permissions(&locked, fs::Permissions::from_mode(0o700)); diff --git a/crates/disktree-core/src/sharing.rs b/crates/disktree-core/src/sharing.rs index ebded10c..b400344d 100644 --- a/crates/disktree-core/src/sharing.rs +++ b/crates/disktree-core/src/sharing.rs @@ -8,12 +8,26 @@ use rustc_hash::FxHashMap; use crate::tree::Node; +mod reclaim; +pub use reclaim::{Estimate, estimate}; + +/// Sparse allocation facts survive hardlink weight de-duplication. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Allocation { + pub bytes: u64, + pub identity: Option<(u64, u64)>, + pub links: u64, + pub followed: bool, + pub clone: Option, +} + /// Metadata retained only for files that may share allocation. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct CloneInfo { pub device: u64, pub id: u64, pub references: u32, + /// Private data-fork allocation; other forks can remain unmeasured. pub private_bytes: Option, /// Clone identity describes the data fork, not resource forks or xattrs. pub data_bytes: u64, @@ -29,15 +43,70 @@ pub struct Sharing { pub unknown_files: u64, /// More than one million distinct groups: repeated-byte reporting is partial. pub groups_truncated: bool, + pub estimate: Option, } impl Sharing { - /// Conservative estimate: shared extents are excluded even if all their - /// references are selected together. Snapshots can retain other blocks. - pub const fn reclaimable(self, listed: u64) -> u64 { - listed - .saturating_sub(self.bytes) - .saturating_add(self.private_bytes) + /// Lower bound before snapshots or changes since the scan. + pub fn reclaimable(self, listed: u64) -> u64 { + self.estimate.map_or(listed, |estimate| estimate.lower) + } + + pub fn possible(self, listed: u64) -> u64 { + self.estimate.map_or(listed, |estimate| estimate.upper) + } +} + +/// Leaf metadata is already retained; do not allocate another box per clone. +pub(crate) fn leaf_summary(node: &Node) -> Sharing { + let Some(allocation) = node.allocation.as_deref() else { + return Sharing::default(); + }; + let mut summary = Sharing { + estimate: Some(allocation.estimate()), + ..Sharing::default() + }; + if let Some(info) = allocation.clone.filter(|_| node.bytes > 0) { + summary.files = 1; + summary.bytes = node.bytes; + summary.private_bytes = info.private_bytes.unwrap_or(0).min(node.bytes); + summary.unknown_files = u64::from(info.private_bytes.is_none()); + } + summary +} + +impl Allocation { + fn estimate(self) -> Estimate { + if self.followed || (self.links > 1 && self.identity.is_some()) { + return Estimate::default(); + } + if self.links > 1 { + return Estimate { + upper: self.bytes, + partial: true, + ..Estimate::default() + }; + } + let lower = self.clone.map_or(self.bytes, |info| { + info.private_bytes.unwrap_or(0).min(self.bytes) + }); + let full = self.clone.is_some_and(|info| { + info.device != 0 + && info.id != 0 + && info.references >= 2 + && info.private_bytes == Some(0) + }); + Estimate { + lower, + upper: if full { + self.bytes.saturating_sub( + self.clone.map_or(0, |info| info.data_bytes), + ) + } else { + self.bytes + }, + partial: false, + } } } @@ -63,15 +132,25 @@ pub(crate) fn refresh(node: &mut Node) { collect(node, MAX_CLONE_GROUPS); } -fn collect(node: &mut Node, limit: usize) -> Groups { +fn collect(node: &mut Node, limit: usize) -> (Groups, reclaim::Accumulator) { + let mut allocation = if node.is_dir() { + reclaim::Accumulator::new(limit) + } else { + reclaim::Accumulator::leaf(node, limit) + }; let mut groups = Groups::default(); let mut summary = Sharing::default(); - if let Some(info) = node.clone_info.as_deref().filter(|_| node.bytes > 0) { + if let Some(info) = node + .allocation + .as_deref() + .and_then(|allocation| allocation.clone) + .filter(|_| node.bytes > 0) + { summary.files = 1; summary.bytes = node.bytes; summary.private_bytes = info.private_bytes.unwrap_or(0).min(node.bytes); summary.unknown_files = u64::from(info.private_bytes.is_none()); - if info.id != 0 && info.references >= 2 { + if info.device != 0 && info.id != 0 && info.references >= 2 { groups.insert( (info.device, info.id), Group { @@ -82,7 +161,8 @@ fn collect(node: &mut Node, limit: usize) -> Groups { } } for child in &mut node.children { - let mut child_groups = collect(child, limit); + let (mut child_groups, child_allocation) = collect(child, limit); + allocation.merge(child_allocation); summary.duplicate_bytes = summary .duplicate_bytes .saturating_add(child.sharing().duplicate_bytes); @@ -118,8 +198,18 @@ fn collect(node: &mut Node, limit: usize) -> Groups { .or_insert(group); } } - node.sharing = (summary.files > 0).then(|| Box::new(summary)); - groups + if node.allocation.as_deref().is_some_and(|info| info.followed) { + allocation = reclaim::Accumulator::new(limit); + } + summary.estimate = Some(allocation.estimate()); + node.sharing = (node.is_dir() + && (summary.files > 0 + || node.allocation.is_some() + || allocation.needs_selection() + || summary.reclaimable(node.bytes) != node.bytes + || summary.possible(node.bytes) != node.bytes)) + .then(|| Box::new(summary)); + (groups, allocation) } #[cfg(test)] @@ -129,16 +219,31 @@ mod tests { fn clone(name: &str, device: u64, id: u64, private: Option) -> Node { let mut node = Node::entry(name, NodeKind::File, 100); - node.clone_info = Some(Box::new(CloneInfo { - device, - id, - references: 2, - private_bytes: private, - data_bytes: 80, + node.allocation = Some(Box::new(Allocation { + bytes: node.bytes, + identity: None, + links: 1, + followed: false, + clone: Some(CloneInfo { + device, + id, + references: 2, + private_bytes: private, + data_bytes: 80, + }), })); node } + #[test] + fn selecting_a_complete_clone_group_reclaims_its_data_once() { + let mut root = Node::directory("root"); + root.children = + vec![clone("a", 1, 1, Some(0)), clone("b", 1, 1, Some(0))]; + aggregate(&mut root, Metric::Bytes); + assert_eq!(root.sharing().reclaimable(root.bytes), 80); + } + #[test] fn group_limit_marks_the_summary_partial_without_losing_private_bytes() { let mut root = Node::directory("root"); @@ -161,7 +266,7 @@ mod tests { aggregate(&mut root, Metric::Bytes); assert_eq!(root.bytes, 300); assert_eq!(root.sharing().duplicate_bytes, 80); - assert_eq!(root.sharing().reclaimable(root.bytes), 60); + assert_eq!(root.sharing().reclaimable(root.bytes), 140); aggregate(&mut root, Metric::Files); assert_eq!(root.sharing().duplicate_bytes, 80); } diff --git a/crates/disktree-core/src/sharing/reclaim.rs b/crates/disktree-core/src/sharing/reclaim.rs new file mode 100644 index 00000000..435f92f4 --- /dev/null +++ b/crates/disktree-core/src/sharing/reclaim.rs @@ -0,0 +1,480 @@ +//! Composable selected-set accounting. No pairwise mark or group comparisons. + +use rustc_hash::FxHashMap; + +use super::{Allocation, CloneInfo}; +use crate::tree::Node; + +/// Scan-derived bounds before snapshots, open files, compression and changes +/// after the scan. Only the lower bound is projected onto the volume meter. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Estimate { + pub lower: u64, + pub upper: u64, + pub partial: bool, +} + +#[derive(Default)] +struct Group { + members: u64, + references: u32, + shared: u64, + all_zero_private: bool, +} + +impl Group { + fn credit(&self) -> Estimate { + let complete = + self.references >= 2 && self.members == u64::from(self.references); + Estimate { + lower: if complete { self.shared } else { 0 }, + upper: if complete && self.all_zero_private { + self.shared + } else { + 0 + }, + partial: false, + } + } +} + +struct Links { + allocation: Allocation, + names: u64, +} + +impl Links { + const fn complete(&self) -> bool { + self.names == self.allocation.links + } +} + +#[derive(Default)] +pub(super) struct Accumulator { + base: Estimate, + credit: Estimate, + groups: FxHashMap<(u64, u64), Group>, + links: FxHashMap<(u64, u64), Links>, + limit: usize, +} + +impl Accumulator { + pub(super) fn new(limit: usize) -> Self { + Self { + limit, + ..Self::default() + } + } + + pub(super) fn needs_selection(&self) -> bool { + !self.groups.is_empty() || !self.links.is_empty() || self.base.partial + } + + pub(super) const fn estimate(&self) -> Estimate { + Estimate { + lower: self.base.lower.saturating_add(if self.base.partial { + 0 + } else { + self.credit.lower + }), + upper: self.base.upper.saturating_add(self.credit.upper), + partial: self.base.partial, + } + } + + pub(super) fn leaf(node: &Node, limit: usize) -> Self { + work(); + let mut acc = Self::new(limit); + if let Some(allocation) = node.allocation.as_deref() { + if allocation.followed { + return acc; + } + if allocation.links > 1 { + if let Some(key) = allocation.identity { + acc.links.insert( + key, + Links { + allocation: *allocation, + names: 1, + }, + ); + } else { + acc.base.partial = true; + acc.base.upper = allocation.bytes; + } + } else { + acc.file(allocation, true); + } + } else { + acc.base.lower = node.bytes; + acc.base.upper = node.bytes; + } + acc + } + + fn file(&mut self, allocation: &Allocation, add: bool) { + let info = allocation.clone; + let private = info.map_or(allocation.bytes, |info| { + info.private_bytes.unwrap_or(0).min(allocation.bytes) + }); + let grouped = info.filter(|info| { + info.device != 0 + && info.id != 0 + && info.references >= 2 + && info.private_bytes.is_some() + }); + // A confirmed full clone retains no blocks while an external member + // remains. Modified/unknown sharing only supplies a lower bound. + let upper = if let Some(info) = grouped.filter(|_| private == 0) { + // PRIVATESIZE describes the data fork. Other forks may still + // have allocation even when that fork is fully shared. + allocation.bytes.saturating_sub(info.data_bytes) + } else { + allocation.bytes + }; + adjust(&mut self.base.lower, private, add); + adjust(&mut self.base.upper, upper, add); + if let Some(info) = grouped { + self.member(info, allocation.bytes, add); + } + } + + fn member(&mut self, info: CloneInfo, allocated: u64, add: bool) { + work(); + let key = (info.device, info.id); + if !self.groups.contains_key(&key) && self.groups.len() >= self.limit { + self.base.partial = true; + // Group credit cannot be proved, but these blocks may be freed. + if add && info.private_bytes == Some(0) { + self.base.upper = self.base.upper.saturating_add(allocated); + } + return; + } + let group = self.groups.entry(key).or_insert_with(|| Group { + all_zero_private: true, + shared: u64::MAX, + ..Group::default() + }); + let before = group.credit(); + adjust(&mut group.members, 1, add); + group.references = group.references.max(info.references); + group.shared = group.shared.min(info.data_bytes.min( + allocated.saturating_sub(info.private_bytes.unwrap_or(allocated)), + )); + group.all_zero_private &= info.private_bytes == Some(0); + replace(&mut self.credit, before, group.credit()); + } + + pub(super) fn merge(&mut self, mut other: Self) { + // Every entry moves only into a map at least as large. This also + // bounds long chains and groups distributed across many directories. + if self.groups.len() + self.links.len() + < other.groups.len() + other.links.len() + { + std::mem::swap(self, &mut other); + } + // Re-evaluate completed hardlinks after names from both sets merge. + // Removing their old contribution first avoids counting a clone + // member twice when its names straddle selected directories. + let other_links = std::mem::take(&mut other.links); + for links in other_links.values().filter(|links| links.complete()) { + other.file(&links.allocation, false); + } + self.base.lower = self.base.lower.saturating_add(other.base.lower); + self.base.upper = self.base.upper.saturating_add(other.base.upper); + self.base.partial |= other.base.partial; + for (key, group) in other.groups { + work(); + if !self.groups.contains_key(&key) + && self.groups.len() >= self.limit + { + self.base.partial = true; + self.base.upper = self.base.upper.saturating_add(group.shared); + continue; + } + let old = self.groups.entry(key).or_insert_with(|| Group { + shared: u64::MAX, + all_zero_private: true, + ..Group::default() + }); + let before = old.credit(); + old.members += group.members; + old.references = old.references.max(group.references); + old.shared = old.shared.min(group.shared); + old.all_zero_private &= group.all_zero_private; + replace(&mut self.credit, before, old.credit()); + } + for (key, links) in other_links { + work(); + if let Some(old) = self.links.remove(&key) { + if old.complete() { + self.file(&old.allocation, false); + } + let mut allocation = old.allocation; + allocation.bytes = allocation.bytes.min(links.allocation.bytes); + allocation.links = allocation.links.max(links.allocation.links); + if allocation.clone != links.allocation.clone { + // Inconsistent live observations cannot authorize saving. + allocation.clone = Some(CloneInfo::default()); + } + let joined = Links { + allocation, + names: old.names + links.names, + }; + if joined.complete() { + self.file(&joined.allocation, true); + } + self.links.insert(key, joined); + } else if self.links.len() < self.limit { + if links.complete() { + self.file(&links.allocation, true); + } + self.links.insert(key, links); + } else { + self.base.partial = true; + self.base.upper = + self.base.upper.saturating_add(links.allocation.bytes); + } + } + } +} + +const fn adjust(total: &mut u64, value: u64, add: bool) { + *total = if add { + total.saturating_add(value) + } else { + total.saturating_sub(value) + }; +} + +const fn replace(total: &mut Estimate, before: Estimate, after: Estimate) { + total.lower = total + .lower + .saturating_sub(before.lower) + .saturating_add(after.lower); + total.upper = total + .upper + .saturating_sub(before.upper) + .saturating_add(after.upper); +} + +/// Accepted non-overlapping aggregated subtrees are evaluated together, so complete +/// groups spanning separate marks receive credit once. +/// +/// Callers cache this scan-derived answer; removal authorization must still be checked afresh. +pub fn estimate<'a>(nodes: impl IntoIterator) -> Estimate { + let mut total = Accumulator::new(super::MAX_CLONE_GROUPS); + for node in nodes { + total.merge(gather(node)); + } + total.estimate() +} + +fn gather(node: &Node) -> Accumulator { + if node.allocation.as_deref().is_some_and(|info| info.followed) { + return Accumulator::new(super::MAX_CLONE_GROUPS); + } + if !node.is_dir() { + return Accumulator::leaf(node, super::MAX_CLONE_GROUPS); + } + work(); + // Aggregation retains a summary on any subtree whose metadata can + // interact with another mark, including zero-weight hardlink aliases. + // Ordinary subtrees need no second walk after a mark changes. + if node.sharing.is_none() { + let mut total = Accumulator::new(super::MAX_CLONE_GROUPS); + total.base = Estimate { + lower: node.bytes, + upper: node.bytes, + partial: false, + }; + return total; + } + let mut total = Accumulator::new(super::MAX_CLONE_GROUPS); + for child in &node.children { + total.merge(gather(child)); + } + total +} + +#[cfg(test)] +thread_local! { + static WORK: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(not(test))] +const fn work() {} + +#[cfg(test)] +fn work() { + WORK.with(|work| work.set(work.get() + 1)); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tree::{Metric, NodeKind, Seen, aggregate_deduped}; + + fn file( + name: &str, + inode: u64, + links: u64, + clone: Option, + ) -> Node { + let mut node = Node::entry(name, NodeKind::File, 4096); + node.inode = + Some((1, std::num::NonZeroU64::new(inode).expect("inode"))); + node.allocation = Some(Box::new(Allocation { + bytes: 4096, + identity: Some((1, inode)), + links, + followed: false, + clone, + })); + node + } + + fn clone_info() -> CloneInfo { + CloneInfo { + device: 1, + id: 42, + references: 2, + private_bytes: Some(0), + data_bytes: 4096, + } + } + + #[test] + fn selected_aliases_free_nothing_until_the_last_real_name_is_selected() { + let mut root = Node::directory("root"); + for name in ["a", "b"] { + let mut dir = Node::directory(name); + dir.children.push(file("file", 1, 2, None)); + root.children.push(dir); + } + aggregate_deduped(&mut root, Metric::Bytes, &Seen::new()); + assert_eq!(root.bytes, 4096); + for child in &root.children { + assert_eq!(estimate([child]).lower, 0); + assert_eq!(child.sharing().reclaimable(child.bytes), 0); + } + assert_eq!(estimate(&root.children).lower, 4096); + assert_eq!(root.sharing().reclaimable(root.bytes), 4096); + } + + #[test] + fn hardlinks_are_one_clone_reference_even_across_marked_directories() { + let mut root = Node::directory("root"); + for (name, inode, links) in [("a", 1, 2), ("b", 1, 2), ("c", 2, 1)] { + let mut dir = Node::directory(name); + dir.children + .push(file("file", inode, links, Some(clone_info()))); + root.children.push(dir); + } + aggregate_deduped(&mut root, Metric::Bytes, &Seen::new()); + assert_eq!(root.bytes, 8192); + assert_eq!(root.sharing().duplicate_bytes, 4096); + assert_eq!(estimate(root.children.iter().take(2)).lower, 0); + assert_eq!(estimate(&root.children).lower, 4096); + assert_eq!(root.sharing().reclaimable(root.bytes), 4096); + assert_eq!(estimate(&root.children).upper, 4096); + } + + #[test] + fn missing_identity_and_private_metadata_never_prove_a_complete_group() { + for missing in ["device", "id", "private", "references"] { + let mut info = clone_info(); + match missing { + "device" => info.device = 0, + "id" => info.id = 0, + "private" => info.private_bytes = None, + _ => info.references = 0, + } + let a = file("a", 1, 1, Some(info)); + let b = file("b", 2, 1, Some(info)); + let estimate = estimate([&a, &b]); + assert_eq!(estimate.lower, 0, "{missing}"); + assert_eq!(estimate.upper, 8192, "{missing}"); + } + } + + #[test] + fn modified_groups_and_unknown_sharing_supply_bounds_not_exact_savings() { + let mut info = clone_info(); + info.private_bytes = Some(1024); + let a = file("a", 1, 1, Some(info)); + let b = file("b", 2, 1, Some(info)); + assert_eq!(estimate([&a]).lower, 1024); + assert_eq!(estimate([&a]).upper, 4096); + assert_eq!(estimate([&a, &b]).lower, 5120); + assert_eq!(estimate([&a, &b]).upper, 8192); + } + + #[test] + fn followed_links_cannot_credit_the_target_or_complete_a_clone_group() { + let mut link = file("link", 1, 1, Some(clone_info())); + link.allocation.as_mut().expect("allocation").followed = true; + let real = file("real", 2, 1, Some(clone_info())); + let mut directory_link = Node::directory("directory-link"); + directory_link.allocation = link.allocation.clone(); + directory_link.children.push(real.clone()); + assert_eq!(estimate([&link, &real]).lower, 0); + assert_eq!(estimate([&directory_link]).lower, 0); + } + + #[test] + fn bounded_grouping_drops_shared_credit_without_losing_private_bytes() { + let mut total = Accumulator::new(1); + for group in [1, 2] { + for inode in [1, 2] { + let mut info = clone_info(); + info.id = group; + info.private_bytes = Some(1024); + total.merge(Accumulator::leaf( + &file("f", inode, 1, Some(info)), + 1, + )); + } + } + let estimate = total.estimate(); + assert!(estimate.partial); + assert_eq!(estimate.lower, 4096); + assert!(estimate.upper >= estimate.lower); + assert!(total.groups.len() <= 1); + } + + #[test] + fn selected_set_work_scales_with_members_even_when_groups_and_marks_grow() { + let mut counts = Vec::new(); + for n in [128, 1024] { + let mut nodes = Vec::new(); + for group in 1..=n { + for alias in [0, 1] { + let mut dir = Node::directory(format!("g{group}-{alias}")); + let mut info = clone_info(); + info.id = group; + for member in [1, 2] { + dir.children.push(file( + "f", + group * 2 + member, + 2, + Some(info), + )); + } + aggregate_deduped(&mut dir, Metric::Bytes, &Seen::new()); + nodes.push(dir); + } + } + WORK.set(0); + let value = estimate(&nodes); + assert_eq!(value.lower, n * 4096); + let work = WORK.get(); + assert!(work < n * 40, "absolute work budget: {n}: {work}"); + counts.push(work); + } + assert!(counts[1] <= counts[0] * 16, "{counts:?}"); + eprintln!( + "selected-set accounting operations 128->1024 groups: {counts:?}" + ); + } +} diff --git a/crates/disktree-core/src/tree.rs b/crates/disktree-core/src/tree.rs index 298f53ea..9f2d0a91 100644 --- a/crates/disktree-core/src/tree.rs +++ b/crates/disktree-core/src/tree.rs @@ -75,7 +75,7 @@ pub struct Node { /// `(device, inode)` for files, used to de-duplicate hardlinks. No file /// has inode zero, which lets `None` cost nothing. pub inode: Option<(u64, NonZeroU64)>, - pub clone_info: Option>, + pub allocation: Option>, pub sharing: Option>, /// The directory could not be read; its contents are unknown. pub read_error: bool, @@ -105,7 +105,7 @@ impl Node { files: 0, dirs: 1, inode: None, - clone_info: None, + allocation: None, sharing: None, read_error: false, modified: 0, @@ -128,7 +128,7 @@ impl Node { files: u64::from(kind == NodeKind::File), dirs: 0, inode: None, - clone_info: None, + allocation: None, sharing: None, read_error: false, modified: 0, @@ -140,7 +140,10 @@ impl Node { /// Sparse: ordinary files do not pay for five additional byte counters. pub fn sharing(&self) -> crate::sharing::Sharing { - self.sharing.as_deref().copied().unwrap_or_default() + self.sharing + .as_deref() + .copied() + .unwrap_or_else(|| crate::sharing::leaf_summary(self)) } pub const fn is_dir(&self) -> bool { diff --git a/docs/performance/apfs-accounting/README.md b/docs/performance/apfs-accounting/README.md new file mode 100644 index 00000000..42c0f10d --- /dev/null +++ b/docs/performance/apfs-accounting/README.md @@ -0,0 +1,105 @@ +# APFS accounting and bounded selection work + +Validated on macOS 26.6.1, Apple silicon, Rust 1.97, 2026-10-14. +The baseline is PR #3 at `e9b4996`; the tests below exercise this branch's +accepted-set accounting. No new home-directory or whole-volume benchmark +was run. Earlier scan timings in the PR describe older implementations. + +## Accounting regressions + +The isolated baseline source snapshot failed both tests in +[regression-red.log](regression-red.log): + +| Selection | Baseline estimate | Required result | +| --- | ---: | ---: | +| Two complete synthetic clones, 80 shared bytes | 0 B | 80 B once | +| Charged name of a 4 KiB file with a retained hardlink | 4 KiB | 0 B | + +The checked-in sharing tests now cover complete groups across directories, +separate marks, hardlinks both within and across those marks, unknown real +volume identity, missing private metadata, changed metadata, followed files +and directories, grouping limits, and modified clones. Real APFS tests use +`cp -c`, hardlinks, followed symlinks and a resource fork. The window test +marks separate clone paths, changes tile metric, draws review/confirmation, +checks warm plan reuse, changes the selection and replaces the scan tree. + +`ATTR_CMNEXT_REALDEVID` scopes grouping and cache reuse. Ordinary `st_dev` +can alias APFS System and Data volumes on this host. Missing real identity +never permits grouping or cross-file metadata reuse. + +An 8 MiB original and two full clones retain 24 MiB of listed allocation. +Marking only the two copies estimates 0 B. Marking all three estimates +8 MiB, once, before snapshot/open-file effects. A retained hardlink prevents +crediting its inode, even if that inode owns the tree's nonzero weight. +Followed symlinks cannot authorize freeing their targets' file data. + +The private-size attribute measures the data fork. In the real resource-fork +fixture it remains zero while all-fork allocation grows by 4 KiB. The +complete clone group's range is therefore 8 MiB to 8 MiB + 4 KiB; the lone +copy's range is 0 to 4 KiB. That unmeasured fork allocation never inflates the +meter's lower estimate. Bounds describe scan observations, not guaranteed +changes to volume free space: snapshots, open files and filesystem changes +still matter. The final outcome uses measured free space. + +## Work and memory bounds + +The main-only red/green proof is separate in +[PR #9](https://github.com/huntharo/disktree/pull/9) and +[mark-resolution](../mark-resolution/README.md). This branch incorporates +that fix and reuses one lazy sibling resolver per accounting cache miss. +Removal authorization remains fresh on every plan and in the worker. + +Selected subtrees merge bounded maps small to large: expected +`O(K log K)` total map work for K visited entries, without mark-pair or +clone-group-pair comparisons. Unchanged accepted sets reuse estimates by +weak tree identity, root and accepted paths. Single marked subtrees use +precomputed scan summaries; ordinary aggregated subtrees need no second +walk. Tree replacement and changed accepted paths invalidate reuse. Tile +metric never changes removal byte units. Fresh guards and target sorting +still cost work per plan; the entire UI is not claimed to be constant time. + +[budget-green.log](budget-green.log) records operation counts with marks, +clone groups and hardlink names growing together: + +| Groups | Marked directories | Leaf entries | Accounting operations | +| ---: | ---: | ---: | ---: | +| 128 | 256 | 512 | 1,790 | +| 1,024 | 2,048 | 4,096 | 14,334 | + +Eight times the workload costs 8.01 times the operations. The test enforces +both a 16x ratio ceiling and an absolute ceiling of 40 operations per group. +This measures accounting operations, not wall-clock speed or kernel work. + +Metadata queries run in directory workers. A per-scan cache has 64 mutex +shards, with no lock held during filesystem queries and no clone-result +channel or dedicated consumer. Reuse requires a known real device, a clone +ID, a full-sharing bulk flag, a confirmed zero-private sample, and matching +all-fork/data-fork allocation. An observed reference count limits reuse; +hardlink aliases do not spend another clone-reference slot. Modified or +unknown observations are measured individually. Races may duplicate first +queries. Cache tests grow both groups and members: 128 groups x 128 members +need 128 measurement callbacks, and 1,024 x 1,024 need 1,024, instead of one +per member. These are deterministic callback counts, not measured speedups. + +The cache caps groups and exceptional hardlink identities independently at +at most one million entries, distributed across shards. Scan summaries cap +clone-group and hardlink maps separately at one million entries. Crossing a +limit keeps private lower estimates, drops unproved group credit, and marks +accounting partial. Leaf metadata uses one sparse box; it no longer copies +a boxed clone result or allocates a second sharing box per clone. The Node +size budget remains 104 bytes, versus upstream's 88-byte layout. + +## Reproduce + +```sh +cargo xtask lint +cargo xtask test +cargo test -p disktree-core selected_set_work -- --nocapture +cargo test -p disktree-core clone_cache -- --nocapture +cargo test -p disktree-core macos::clone_tests -- --nocapture +cargo run --release -p disktree-core --example scan_accounting -- PATH +``` + +Local validation: lint passed; 79 window/app, 174 core and 3 xtask tests passed; +two existing core tests remained ignored. No private file contents were +inspected and no user files were removed by this investigation. diff --git a/docs/performance/apfs-accounting/budget-green.log b/docs/performance/apfs-accounting/budget-green.log new file mode 100644 index 00000000..f63dce09 --- /dev/null +++ b/docs/performance/apfs-accounting/budget-green.log @@ -0,0 +1,10 @@ + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mujumhxf/disktree/crates/disktree-core) + Finished `test` profile [optimized + debuginfo] target(s) in 1.53s + Running unittests src/lib.rs (target/debug/deps/disktree_core-8aad222ba4e32dfe) + +running 1 test +selected-set accounting operations 128->1024 groups: [1790, 14334] +test sharing::reclaim::tests::selected_set_work_scales_with_members_even_when_groups_and_marks_grow ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 175 filtered out; finished in 0.19s + diff --git a/docs/performance/apfs-accounting/regression-red.log b/docs/performance/apfs-accounting/regression-red.log new file mode 100644 index 00000000..6abf3dc5 --- /dev/null +++ b/docs/performance/apfs-accounting/regression-red.log @@ -0,0 +1,57 @@ + Compiling libc v0.2.189 + Compiling crossbeam-utils v0.8.23 + Compiling crossbeam-epoch v0.9.21 + Compiling crossbeam-deque v0.8.8 + Compiling autocfg v1.5.1 + Compiling objc2 v0.6.4 + Compiling bitflags v2.13.2 + Compiling getrandom v0.4.3 + Compiling rustix v1.1.4 + Compiling core-foundation-sys v0.8.7 + Compiling objc2-encode v4.1.0 + Compiling rayon-core v1.13.0 + Compiling cfg-if v1.0.4 + Compiling percent-encoding v2.3.2 + Compiling memchr v2.8.3 + Compiling once_cell v1.21.4 + Compiling either v1.18.0 + Compiling fastrand v2.5.0 + Compiling log v0.4.34 + Compiling iana-time-zone v0.1.65 + Compiling rustc-hash v2.1.3 + Compiling num-traits v0.2.19 + Compiling objc2-foundation v0.3.2 + Compiling trash v5.2.9 + Compiling errno v0.3.14 + Compiling sysinfo v0.31.4 + Compiling rayon v1.12.0 + Compiling chrono v0.4.45 + Compiling tempfile v3.27.0 + Compiling disktree-core v0.11.0 (/private/var/folders/07/183kkj1d7l59l67s4tq0dmyr0000gn/T/disktree-accounting-red-e2t2vxnq/crates/disktree-core) + Finished `test` profile [optimized + debuginfo] target(s) in 7.74s + Running unittests src/lib.rs (target/debug/deps/disktree_core-8aad222ba4e32dfe) + +running 2 tests + +thread 'sharing::tests::regression_complete_clone_group' (403629209) panicked at crates/disktree-core/src/sharing.rs:218:9: +assertion `left == right` failed + left: 0 + right: 80 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +test sharing::tests::regression_complete_clone_group ... FAILED + +thread 'sharing::tests::regression_retained_hardlink' (403629210) panicked at crates/disktree-core/src/sharing.rs:227:9: +assertion `left == right` failed + left: 4096 + right: 0 +test sharing::tests::regression_retained_hardlink ... FAILED + +failures: + +failures: + sharing::tests::regression_complete_clone_group + sharing::tests::regression_retained_hardlink + +test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 158 filtered out; finished in 0.00s + +error: test failed, to rerun pass `-p disktree-core --lib` From 70f110b5d612c8e998321b27aea7590fa050b5c1 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 14:25:43 -0400 Subject: [PATCH 06/19] test: measure marking hover and review latency by queue size --- crates/disktree-app/src/marks.rs | 8 + crates/disktree-app/src/state.rs | 4 + crates/disktree-app/src/tests.rs | 126 ++++++++++++ docs/performance/mark-resolution/README.md | 20 +- .../mark-resolution/interactions-green.log | 65 +++++++ .../mark-resolution/interactions-red.log | 66 +++++++ .../mark-resolution/interactions.md | 92 +++++++++ docs/performance/mark-resolution/lint.log | 3 +- docs/performance/mark-resolution/tests.log | 181 +++++++++--------- 9 files changed, 472 insertions(+), 93 deletions(-) create mode 100644 docs/performance/mark-resolution/interactions-green.log create mode 100644 docs/performance/mark-resolution/interactions-red.log create mode 100644 docs/performance/mark-resolution/interactions.md diff --git a/crates/disktree-app/src/marks.rs b/crates/disktree-app/src/marks.rs index 0244f9f6..12fc5f01 100644 --- a/crates/disktree-app/src/marks.rs +++ b/crates/disktree-app/src/marks.rs @@ -18,6 +18,9 @@ pub mod work { thread_local! { pub static RESOLUTION: Cell = const { Cell::new(0) }; pub static FRAMES: Cell = const { Cell::new(0) }; + pub static RENDERS: Cell = const { Cell::new(0) }; + pub static PLANS: Cell = const { Cell::new(0) }; + pub static REFRESHES: Cell = const { Cell::new(0) }; pub static TILES: Cell = const { Cell::new(0) }; } @@ -28,6 +31,9 @@ pub mod work { pub fn reset() { RESOLUTION.set(0); FRAMES.set(0); + RENDERS.set(0); + PLANS.set(0); + REFRESHES.set(0); TILES.set(0); } } @@ -98,6 +104,8 @@ impl Marks { /// Re-read sizes from a freshly scanned tree. Missing paths stay marked /// with zero bytes, so they cannot claim space that is already gone. pub fn refresh(&mut self, root_path: &Path, root: &Node, _metric: Metric) { + #[cfg(test)] + work::REFRESHES.set(work::REFRESHES.get() + 1); self.resolved = None; let mut resolver = Resolver::new(root_path, root); let mut resolved = Vec::with_capacity(self.items.len()); diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs index 96bced96..4b750e6d 100644 --- a/crates/disktree-app/src/state.rs +++ b/crates/disktree-app/src/state.rs @@ -1778,6 +1778,8 @@ impl Disktree { /// The plan the review screen shows and the removal runs. pub fn plan(&self) -> Plan { + #[cfg(test)] + crate::marks::work::PLANS.set(crate::marks::work::PLANS.get() + 1); let mut plan = plan(self.marks.items(), &self.root_path); if let Some(tree) = &self.tree { let mut cached = self.accounting.borrow_mut(); @@ -3250,6 +3252,8 @@ impl Render for Disktree { window: &mut Window, cx: &mut Context<'_, Self>, ) -> impl gpui_kit::IntoElement { + #[cfg(test)] + crate::marks::work::RENDERS.set(crate::marks::work::RENDERS.get() + 1); self.rem = window.rem_size().as_f32(); self.tick_transition(window); // The titlebar names the directory on screen, however it got there: diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index db8f2937..a1236506 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -2097,3 +2097,129 @@ fn marked_tiles_follow_metric_rescan_and_mark_changes(cx: &mut TestAppContext) { draw(window); assert!(update(&view, window, |app, _| marked_paths(app)).is_empty()); } + +/// Opt-in latency evidence, separate from deterministic CI work budgets. +#[gpui_kit::test] +#[ignore = "manual interaction timing on synthetic temporary files"] +fn marked_interaction_timings(cx: &mut TestAppContext) { + use crate::marks::work; + use disktree_core::removal::Target; + use gpui_kit::Modifiers; + + fn measure( + view: &Entity, + window: &mut Window, + action: &str, + repeat: usize, + run: impl FnOnce(&mut Window), + ) { + let before = read(view, window, |app| app.marks.len()); + work::reset(); + let start = std::time::Instant::now(); + run(window); + let elapsed = start.elapsed(); + let after = read(view, window, |app| app.marks.len()); + assert!(work::RENDERS.get() > 0, "must render the real screen"); + assert!(work::PLANS.get() > 0, "must include removal projections"); + assert_eq!(work::REFRESHES.get(), 0, "interaction must not rescan"); + eprintln!( + "interaction action={action} repeat={repeat} marks={before}->{after} renders={} prepares={} plans={} refreshes={} wall={elapsed:?}", + work::RENDERS.get(), + work::FRAMES.get(), + work::PLANS.get(), + work::REFRESHES.get(), + ); + } + + cx.update(gpui_omarchy::init); + let temp = tempfile::tempdir().expect("tempdir"); + // One broad directory, kept fixed while only the number of marks grows. + // Each marked folder contains a file: descendants still make one mark. + for index in 0..1025 { + let folder = temp.path().join(format!("folder-{index:05}")); + std::fs::create_dir(&folder).expect("mkdir"); + std::fs::write(folder.join("payload"), [b'x'; 64]).expect("write"); + } + let tree = scan(temp.path(), options()).expect("scan private fixture"); + let extra_path = temp.path().join("folder-01024"); + for count in [0, 1, 8, 32, 128, 1024] { + let root_path = temp.path().to_path_buf(); + let snapshot = tree.clone(); + let (view, window) = cx.add_window_view(move |_, cx| { + Disktree::with_tree(root_path.clone(), snapshot, options(), 3, cx) + }); + window.simulate_resize(gpui_kit::size(px(1400.), px(900.))); + let focus = read(&view, window, |app| app.focus.clone()); + window.update(|window, cx| window.focus(&focus, cx)); + update(&view, window, |app, cx| { + assert!(app.space.is_some() && app.show_selection); + for index in 0..count { + app.marks.toggle(Target { + path: temp.path().join(format!("folder-{index:05}")), + bytes: 64, + is_dir: true, + hidden: false, + }); + } + cx.notify(); + }); + draw(window); + for repeat in 1..=3 { + update(&view, window, |app, cx| { + app.pointer_active = false; + app.hovered = None; + app.pointer = None; + let crumbs = + app.crumbs_for_path(&extra_path).expect("extra folder"); + app.select(Some(crumbs), cx); + }); + draw(window); + measure(&view, window, "mark-space", repeat, |window| { + press(window, "space"); + }); + assert_eq!(read(&view, window, |app| app.marks.len()), count + 1); + let at = update(&view, window, |app, _| { + let tile = app + .layout() + .expect("layout") + .iter() + .find(|tile| { + matches!( + tile.kind, + disktree_core::treemap::TileKind::Node { .. } + ) + }) + .expect("node tile") + .rect; + let rect = app.view.project(tile); + let origin = app.treemap_origin.get(); + Point::new( + origin.x + px(rect.x + rect.w / 2.), + origin.y + px(rect.y + rect.h / 2.), + ) + }); + measure(&view, window, "hover", repeat, |window| { + window.simulate_mouse_move(at, None, Modifiers::none()); + draw(window); + }); + assert!(read(&view, window, |app| app.hovered.is_some())); + measure(&view, window, "review-c", repeat, |window| { + press(window, "c"); + }); + assert_eq!(read(&view, window, |app| app.screen), Screen::Review); + press(window, "escape"); + update(&view, window, |app, cx| { + app.pointer_active = false; + app.hovered = None; + app.pointer = None; + let crumbs = + app.crumbs_for_path(&extra_path).expect("extra folder"); + app.select(Some(crumbs), cx); + }); + draw(window); + press(window, "space"); + assert_eq!(read(&view, window, |app| app.marks.len()), count); + } + window.update(|window, _| window.remove_window()); + } +} diff --git a/docs/performance/mark-resolution/README.md b/docs/performance/mark-resolution/README.md index ca493ab5..b8838306 100644 --- a/docs/performance/mark-resolution/README.md +++ b/docs/performance/mark-resolution/README.md @@ -5,6 +5,22 @@ profile (application opt-level 1, dependency opt-level 2). All trees and paths are synthetic or inside private temporary directories. No home/volume scan, real Trash operation, APFS accounting change or parent-branch edit was used. +## What the timing means + +A mark is one independently queued file or directory, not every file inside a +marked directory. The 1,024-mark case is a large-queue stress test, not an +established frequent manual workload. The 6.60-second result is **four forced +draws/eight prepares of Explore with that queue already present**. It is not +one scan or one click. + +[Individual-action measurements and call-path explanation](interactions.md) +now cover 0, 1, 8, 32, 128 and 1,024 queued paths on a fixed real temporary +tree, using Space, pointer movement and Review inputs. Main already completes +those inputs in a few milliseconds with a handful of marks; the demonstrated +large improvement applies to large queues. At 1,024 queued folders, adding +one more or hovering takes about 1.1 seconds on main in that fixture. These +costs can recur on renders while interacting, rather than once per scan. + ## Red before green Base: `6c8d4ce6211bf135ff4e42890faebff1040e1846` (upstream/main and fork main). @@ -157,7 +173,9 @@ callers have no revision-counter or manual invalidation obligation. - `cargo xtask lint`: pass; [lint.log](lint.log). - `cargo xtask test`: 78 app, 150 core and 3 xtask tests pass; - [tests.log](tests.log). Two pre-existing ignored core tests remain ignored. + [tests.log](tests.log). Two pre-existing ignored core tests remain ignored. The new opt-in + interaction timing test is also ignored by the normal suite; it passes + when explicitly run on both main and fixed snapshots. - `cargo test --workspace budget -- --nocapture`: all budgets pass. - Three serial red repetitions fail the intended budgets (exit 101), followed by three serial green repetitions passing them (exit 0). diff --git a/docs/performance/mark-resolution/interactions-green.log b/docs/performance/mark-resolution/interactions-green.log new file mode 100644 index 00000000..a17c207c --- /dev/null +++ b/docs/performance/mark-resolution/interactions-green.log @@ -0,0 +1,65 @@ + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 2.78s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 1 test +test tests::marked_interaction_timings ... interaction action=mark-space repeat=1 marks=0->1 renders=3 prepares=3 plans=6 refreshes=0 wall=3.678416ms +interaction action=hover repeat=1 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=2.546375ms +interaction action=review-c repeat=1 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=2.183334ms +interaction action=mark-space repeat=2 marks=0->1 renders=2 prepares=2 plans=4 refreshes=0 wall=2.440541ms +interaction action=hover repeat=2 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=2.752917ms +interaction action=review-c repeat=2 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=2.195ms +interaction action=mark-space repeat=3 marks=0->1 renders=2 prepares=2 plans=4 refreshes=0 wall=2.512208ms +interaction action=hover repeat=3 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=2.463708ms +interaction action=review-c repeat=3 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=2.125291ms +interaction action=mark-space repeat=1 marks=1->2 renders=3 prepares=3 plans=6 refreshes=0 wall=3.916458ms +interaction action=hover repeat=1 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=2.693916ms +interaction action=review-c repeat=1 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=2.467208ms +interaction action=mark-space repeat=2 marks=1->2 renders=2 prepares=2 plans=4 refreshes=0 wall=2.570084ms +interaction action=hover repeat=2 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=2.661209ms +interaction action=review-c repeat=2 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=2.573834ms +interaction action=mark-space repeat=3 marks=1->2 renders=2 prepares=2 plans=4 refreshes=0 wall=2.588833ms +interaction action=hover repeat=3 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=2.671125ms +interaction action=review-c repeat=3 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=2.675084ms +interaction action=mark-space repeat=1 marks=8->9 renders=3 prepares=3 plans=6 refreshes=0 wall=5.180041ms +interaction action=hover repeat=1 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=3.608791ms +interaction action=review-c repeat=1 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=4.043208ms +interaction action=mark-space repeat=2 marks=8->9 renders=2 prepares=2 plans=4 refreshes=0 wall=3.50525ms +interaction action=hover repeat=2 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=3.503375ms +interaction action=review-c repeat=2 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=4.018792ms +interaction action=mark-space repeat=3 marks=8->9 renders=2 prepares=2 plans=4 refreshes=0 wall=3.820917ms +interaction action=hover repeat=3 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=4.015208ms +interaction action=review-c repeat=3 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=4.169958ms +interaction action=mark-space repeat=1 marks=32->33 renders=3 prepares=3 plans=6 refreshes=0 wall=7.792416ms +interaction action=hover repeat=1 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=5.2135ms +interaction action=review-c repeat=1 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=8.745875ms +interaction action=mark-space repeat=2 marks=32->33 renders=2 prepares=2 plans=4 refreshes=0 wall=5.241834ms +interaction action=hover repeat=2 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=5.400709ms +interaction action=review-c repeat=2 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=8.7555ms +interaction action=mark-space repeat=3 marks=32->33 renders=2 prepares=2 plans=4 refreshes=0 wall=5.447334ms +interaction action=hover repeat=3 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=5.865541ms +interaction action=review-c repeat=3 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=9.197958ms +interaction action=mark-space repeat=1 marks=128->129 renders=3 prepares=3 plans=6 refreshes=0 wall=19.085708ms +interaction action=hover repeat=1 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=13.022875ms +interaction action=review-c repeat=1 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=30.691375ms +interaction action=mark-space repeat=2 marks=128->129 renders=2 prepares=2 plans=4 refreshes=0 wall=14.717709ms +interaction action=hover repeat=2 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=14.473875ms +interaction action=review-c repeat=2 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=30.634125ms +interaction action=mark-space repeat=3 marks=128->129 renders=2 prepares=2 plans=4 refreshes=0 wall=12.734667ms +interaction action=hover repeat=3 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=12.695834ms +interaction action=review-c repeat=3 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=30.7185ms +interaction action=mark-space repeat=1 marks=1024->1025 renders=3 prepares=3 plans=6 refreshes=0 wall=113.33325ms +interaction action=hover repeat=1 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=94.280958ms +interaction action=review-c repeat=1 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=247.047292ms +interaction action=mark-space repeat=2 marks=1024->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=76.356875ms +interaction action=hover repeat=2 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=74.409208ms +interaction action=review-c repeat=2 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=242.707958ms +interaction action=mark-space repeat=3 marks=1024->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=77.313917ms +interaction action=hover repeat=3 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=74.570667ms +interaction action=review-c repeat=3 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=237.816292ms +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 78 filtered out; finished in 3.16s + diff --git a/docs/performance/mark-resolution/interactions-red.log b/docs/performance/mark-resolution/interactions-red.log new file mode 100644 index 00000000..a913f804 --- /dev/null +++ b/docs/performance/mark-resolution/interactions-red.log @@ -0,0 +1,66 @@ + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Finished `test` profile [optimized + debuginfo] target(s) in 3.91s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 1 test +test tests::marked_interaction_timings ... interaction action=mark-space repeat=1 marks=0->1 renders=3 prepares=3 plans=6 refreshes=0 wall=3.866334ms +interaction action=hover repeat=1 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=2.832708ms +interaction action=review-c repeat=1 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=2.252292ms +interaction action=mark-space repeat=2 marks=0->1 renders=2 prepares=2 plans=4 refreshes=0 wall=2.443ms +interaction action=hover repeat=2 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=2.800667ms +interaction action=review-c repeat=2 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=2.211042ms +interaction action=mark-space repeat=3 marks=0->1 renders=2 prepares=2 plans=4 refreshes=0 wall=2.408125ms +interaction action=hover repeat=3 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=2.487209ms +interaction action=review-c repeat=3 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=2.212542ms +interaction action=mark-space repeat=1 marks=1->2 renders=3 prepares=3 plans=6 refreshes=0 wall=3.834833ms +interaction action=hover repeat=1 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=2.865791ms +interaction action=review-c repeat=1 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=2.445292ms +interaction action=mark-space repeat=2 marks=1->2 renders=2 prepares=2 plans=4 refreshes=0 wall=2.74825ms +interaction action=hover repeat=2 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=2.735875ms +interaction action=review-c repeat=2 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=2.421667ms +interaction action=mark-space repeat=3 marks=1->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.28325ms +interaction action=hover repeat=3 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=2.782625ms +interaction action=review-c repeat=3 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=2.733167ms +interaction action=mark-space repeat=1 marks=8->9 renders=3 prepares=3 plans=6 refreshes=0 wall=5.291709ms +interaction action=hover repeat=1 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=4.145167ms +interaction action=review-c repeat=1 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=4.556709ms +interaction action=mark-space repeat=2 marks=8->9 renders=2 prepares=2 plans=4 refreshes=0 wall=3.612542ms +interaction action=hover repeat=2 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=3.624666ms +interaction action=review-c repeat=2 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=4.150625ms +interaction action=mark-space repeat=3 marks=8->9 renders=2 prepares=2 plans=4 refreshes=0 wall=3.549459ms +interaction action=hover repeat=3 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=3.6055ms +interaction action=review-c repeat=3 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=4.159167ms +interaction action=mark-space repeat=1 marks=32->33 renders=3 prepares=3 plans=6 refreshes=0 wall=9.137625ms +interaction action=hover repeat=1 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=6.42875ms +interaction action=review-c repeat=1 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=9.801417ms +interaction action=mark-space repeat=2 marks=32->33 renders=2 prepares=2 plans=4 refreshes=0 wall=6.260209ms +interaction action=hover repeat=2 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=6.274417ms +interaction action=review-c repeat=2 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=9.739625ms +interaction action=mark-space repeat=3 marks=32->33 renders=2 prepares=2 plans=4 refreshes=0 wall=6.120542ms +interaction action=hover repeat=3 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=6.348584ms +interaction action=review-c repeat=3 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=9.569958ms +interaction action=mark-space repeat=1 marks=128->129 renders=3 prepares=3 plans=6 refreshes=0 wall=41.574ms +interaction action=hover repeat=1 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=29.242583ms +interaction action=review-c repeat=1 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=44.395917ms +interaction action=mark-space repeat=2 marks=128->129 renders=2 prepares=2 plans=4 refreshes=0 wall=27.956334ms +interaction action=hover repeat=2 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=27.605834ms +interaction action=review-c repeat=2 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=44.052416ms +interaction action=mark-space repeat=3 marks=128->129 renders=2 prepares=2 plans=4 refreshes=0 wall=29.216375ms +interaction action=hover repeat=3 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=27.679958ms +interaction action=review-c repeat=3 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=44.189917ms +interaction action=mark-space repeat=1 marks=1024->1025 renders=3 prepares=3 plans=6 refreshes=0 wall=1.726271166s +interaction action=hover repeat=1 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=1.09936375s +interaction action=review-c repeat=1 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=1.237629458s +interaction action=mark-space repeat=2 marks=1024->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=1.154407584s +interaction action=hover repeat=2 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=1.109142875s +interaction action=review-c repeat=2 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=1.229517333s +interaction action=mark-space repeat=3 marks=1024->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=1.064909416s +interaction action=hover repeat=3 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=1.068550917s +interaction action=review-c repeat=3 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=1.224605875s +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 73 filtered out; finished in 26.34s + diff --git a/docs/performance/mark-resolution/interactions.md b/docs/performance/mark-resolution/interactions.md new file mode 100644 index 00000000..1eeefec9 --- /dev/null +++ b/docs/performance/mark-resolution/interactions.md @@ -0,0 +1,92 @@ +# What is a mark, and when does the delay happen? + +A **mark** is one file or directory queued for removal. Marking a directory +marks that path once, regardless of how many files it contains. For example, +marking a cache directory containing a million files is M=1, not M=1,000,000. +Marking 1,024 unrelated paths means 1,024 separate queued targets. Main has no +usage telemetry establishing that this is a frequent manual workload; it is +a large-queue stress case, also relevant to programmatic/bulk control. + +The earlier **6.60 seconds** measured **four forced window draws, which caused +eight `prepare` calls** in the GPUI harness. It was not a scan time, a single +click time, or a measured delay for every item. The ~359 ms fixed result has +the same denominator. These draws were of the Explore screen with many marks +already present; they were not the act of selecting 1,024 items. + +## Where it repeats + +- Explore calls `prepare` whenever the app renders, resolving marked paths on + main. Its Marked section and disk projection each call `app.plan()` when + those panels are shown, evaluating the entire queue again. +- Marking/unmarking, keyboard selection, zoom and pointer movement over a + tile can request another render. Pointer movement within the same hovered + tile also requests rendering because the tooltip follows the pointer. +- Opening Review calls `app.plan()` for the queue again. Review does not use + the treemap's `prepare`, but it still pays the original quadratic removal + nesting cost, plus constructing the list rows. +- `Marks::refresh` is different: it runs when a scan result arrives and when + the metric changes. It is not run by every mark action or hover. +- An unchanged idle window need not redraw. The free-space ticker checks + every 1.2 seconds outside removal, but only requests rendering if the value + changes. This is not an unconditional six-second periodic stall. + +## Individual user actions, with smaller queues + +An additional opt-in real-window test measures actual Space, pointer-move and +C inputs. The fixture exists on disk: 1,025 sibling directories, each with one +64-byte file, scanned before timing. The tree size stays fixed at 2,050 +non-root nodes while the queued count changes. The window is 1,400 by 900, +with the selection/Marked panel and disk projection visible. No files are +deleted or trashed. + +Each row seeds M folder marks, then measures adding one more folder with +Space. Hover and Review are measured with M+1 marks afterward. Values are +medians of three input repetitions in milliseconds; each cell is +**main → fixed**. Both snapshots use the same fixture and test-only counters. + +| Already queued M | Add one mark (Space) | Move pointer over a tile | Open Review (C) | +| ---: | ---: | ---: | ---: | +| 0 | 2.44 → 2.51 | 2.80 → 2.55 | 2.21 → 2.18 | +| 1 | 3.28 → 2.59 | 2.78 → 2.67 | 2.45 → 2.57 | +| 8 | 3.61 → 3.82 | 3.62 → 3.61 | 4.16 → 4.04 | +| 32 | 6.26 → 5.45 | 6.35 → 5.40 | 9.74 → 8.76 | +| 128 | 29.22 → 14.72 | 27.68 → 13.02 | 44.19 → 30.69 | +| 1,024 | 1,154.41 → 77.31 | 1,099.36 → 74.57 | 1,229.52 → 242.71 | + +For a handful of marks, these operations were already a few milliseconds on +main; this measurement does not establish a substantial common-case speedup. +At 128 marks the repeated work is more noticeable, and the large queue causes +roughly a one-second delay for ordinary interactions in this fixture. The +problem repeats while interacting with that queue, rather than being paid +once when the scan finishes. Opening a large Review list remains more costly +than hovering because it constructs many rows as well as evaluating the plan. + +The measured Space input caused two renders in repetitions 2–3, and three in +the first repetition. Hover caused two renders. Review caused three renders +(one Explore prepare plus the switch to Review). With panels visible, these +inputs evaluated four plans, or six for the first Space repetition. Framework +render scheduling can vary, so these are end-to-end harness input timings, +not a universal mapping of one physical event to one frame. + +Every timed input asserts that real rendering and removal projections ran, +and that **zero `Marks::refresh` calls** occurred. No scan is part of these +individual-action numbers. Raw output includes exact render, prepare, plan +and refresh counts: + +- [Main interaction log](interactions-red.log): main algorithms from the + red commit `5a3c92e`, with the new test/counters temporarily applied. +- [Fixed interaction log](interactions-green.log): fixed algorithms with + the identical test/counters. + +These are synthetic macOS arm64 test-profile measurements, not release-app +latency guarantees or evidence of how often users select a given queue size. +The previous deterministic N/8N budgets remain necessary to protect scaling; +this additional test is intentionally ignored by normal CI because elapsed +latency is recorded as evidence, not asserted as a portable threshold. + +Reproduce on the final branch: + +```sh +cargo test -p disktree-app marked_interaction_timings -- \ + --ignored --nocapture --test-threads=1 +``` diff --git a/docs/performance/mark-resolution/lint.log b/docs/performance/mark-resolution/lint.log index 363cf891..1687dd61 100644 --- a/docs/performance/mark-resolution/lint.log +++ b/docs/performance/mark-resolution/lint.log @@ -1,7 +1,6 @@ Finished `dev` profile [optimized + debuginfo] target(s) in 0.09s Running `target/debug/xtask lint` - Checking disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) - Finished `dev` profile [optimized + debuginfo] target(s) in 1.77s + Finished `dev` profile [optimized + debuginfo] target(s) in 1.10s warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` diff --git a/docs/performance/mark-resolution/tests.log b/docs/performance/mark-resolution/tests.log index 158bcd31..af929fb7 100644 --- a/docs/performance/mark-resolution/tests.log +++ b/docs/performance/mark-resolution/tests.log @@ -1,41 +1,41 @@ - Finished `dev` profile [optimized + debuginfo] target(s) in 0.13s + Finished `dev` profile [optimized + debuginfo] target(s) in 0.08s Running `target/debug/xtask test` Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) - Finished `test` profile [optimized + debuginfo] target(s) in 3.58s + Finished `test` profile [optimized + debuginfo] target(s) in 4.25s warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) -running 78 tests +running 79 tests test appearance::tests::a_linux_desktop_is_followed_only_if_it_states_a_preference ... ok -test marks::tests::display_path_shortens_the_home_prefix ... ok -test argument_tests::invalid_worker_budgets_are_rejected ... ok -test marks::tests::marking_the_same_path_twice_keeps_one_entry ... ok test git::tests::summaries_say_what_would_be_lost ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok test marks::tests::toggling_marks_and_unmarks ... ok -test marks::tests::resolver_preserves_lossy_non_utf8_components ... ok test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok -test palette::tests::colourful_categories_share_one_level ... ok +test marks::tests::display_path_shortens_the_home_prefix ... ok +test marks::tests::marking_the_same_path_twice_keeps_one_entry ... ok +test marks::tests::resolver_preserves_lossy_non_utf8_components ... ok +test palette::tests::every_legend_category_has_its_own_hue ... ok test palette::tests::age_buckets_cover_every_age_in_order ... ok -test palette::tests::deeper_tiles_lift_away_from_the_background ... ok test palette::tests::mix_clamps_its_parameter ... ok +test palette::tests::colourful_categories_share_one_level ... ok +test palette::tests::deeper_tiles_lift_away_from_the_background ... ok test palette::tests::mixing_toward_a_grey_keeps_the_hue ... ok -test palette::tests::every_legend_category_has_its_own_hue ... ok test palette::tests::the_highlight_is_not_a_category_colour ... ok test power::tests::presets_are_fixed_and_show_the_effective_cpu_limit ... ok test power::tests::presets_that_add_no_workers_are_not_offered ... ok -test marks::tests::find_matches_whole_components_only ... ok -test marks::tests::resolver_preserves_scan_name_semantics ... ok -test marks::tests::refresh_re_reads_sizes_from_a_new_tree ... ok test state::tests::a_transition_grows_a_tile_that_is_being_entered ... ok -test marks::tests::cached_crumbs_follow_every_invalidation_boundary ... ok test state::tests::a_transition_starts_where_the_region_was_and_ends_where_it_is ... ok +test marks::tests::refresh_re_reads_sizes_from_a_new_tree ... ok test state::tests::clamping_never_leaves_a_margin ... ok +test marks::tests::find_matches_whole_components_only ... ok +test marks::tests::resolver_preserves_scan_name_semantics ... ok +test marks::tests::cached_crumbs_follow_every_invalidation_boundary ... ok test state::tests::directions_only_see_gaps_on_their_own_side ... ok +test argument_tests::saved_power_is_loaded_before_cli_overrides ... ok test state::tests::projecting_and_unprojecting_are_inverses ... ok test state::tests::zooming_keeps_the_point_under_the_cursor_still ... ok -test argument_tests::saved_power_is_loaded_before_cli_overrides ... ok test marks::tests::refresh_dimension_budget ... ok test power::tests::saved_choice_survives_replacement_and_bad_data_is_reported ... ok test marks::tests::refresh_resolution_budget ... ok @@ -43,80 +43,81 @@ test git::tests::reads_a_real_checkout ... ok test git::tests::a_checkout_cannot_make_git_run_its_programs ... ok test git::tests::a_checkout_cannot_make_git_verify_a_signature_with_its_program ... ok test git::tests::a_checkouts_own_filter_is_never_run ... ok +test tests::command_chords_are_not_read_as_plain_letters ... ok +test tests::marked_interaction_timings ... ignored, manual interaction timing on synthetic temporary files +test tests::enter_before_the_search_lands_applies_it_when_it_does ... ok +test tests::escape_in_the_delete_dialog_cancels ... ok +test tests::a_view_without_flags_or_marks_still_draws ... ok +test tests::back_and_forward_retrace_where_you_have_been ... ok +test tests::interface_zoom_scales_the_rem_and_the_header_band ... ok +test tests::restart_arguments_parse_back_to_the_same_scan ... ok +test tests::after_descending_every_tile_is_inside_the_directory_drawn ... ok test tests::enter_opens_the_selected_directory_at_any_depth ... ok test tests::escape_cancels_the_first_scan_and_r_starts_it_again ... ok -test tests::escape_in_the_delete_dialog_cancels ... ok +test tests::enter_in_the_focused_volume_picker_scans_the_selected_root ... ok test tests::a_permanent_deletion_asks_in_an_alert_dialog_then_removes ... ok +test tests::escape_cancels_widening_and_keeps_the_tree_on_screen ... ok +test tests::keys_walk_the_tree_and_mark_what_is_selected ... ok test tests::a_crumb_lists_its_siblings_and_jumps_sideways ... ok test tests::hovering_reports_the_tile_under_the_pointer ... ok -test tests::restart_arguments_parse_back_to_the_same_scan ... ok test tests::a_parents_name_gets_its_own_band_above_its_children ... ok -test tests::enter_before_the_search_lands_applies_it_when_it_does ... ok -test tests::escape_cancels_widening_and_keeps_the_tree_on_screen ... ok -test tests::a_view_without_flags_or_marks_still_draws ... ok -test tests::after_descending_every_tile_is_inside_the_directory_drawn ... ok -test tests::enter_in_the_focused_volume_picker_scans_the_selected_root ... ok -test tests::command_chords_are_not_read_as_plain_letters ... ok -test tests::both_appearances_draw ... ok -test tests::keys_walk_the_tree_and_mark_what_is_selected ... ok -test tests::back_and_forward_retrace_where_you_have_been ... ok test tests::dragging_the_panel_edge_resizes_it_within_its_limits ... ok -test tests::interface_zoom_scales_the_rem_and_the_header_band ... ok +test tests::both_appearances_draw ... ok test tests::marked_tiles_follow_metric_rescan_and_mark_changes ... ok -test tests::the_first_scan_shows_what_it_is_doing_then_the_treemap ... ok -test tests::marking_a_directory_marks_everything_inside_it ... ok -test tests::mouse_side_buttons_go_back_and_forward ... ok -test tests::showing_a_tile_that_is_gone_says_so_instead ... ok +test tests::the_help_overlay_opens_and_closes ... ok +test tests::the_review_screen_switches_removal_mode ... ok +test tests::the_review_screen_copies_the_list_as_an_agent_prompt ... ok +test tests::switching_the_metric_keeps_the_directory_and_the_selection ... ok test widgets::tests::ages_read_in_the_unit_a_person_would_use ... ok test widgets::tests::sizes_split_into_number_and_unit ... ok +test tests::right_click_selects_the_tile_and_reveals_it ... ok +test tests::showing_a_tile_that_is_gone_says_so_instead ... ok test tests::power_efficiency_menu_saves_without_discarding_the_tree ... ok -test tests::the_mark_key_follows_the_pointer_until_the_keyboard_moves ... ok -test tests::the_help_overlay_opens_and_closes ... ok -test tests::the_review_screen_copies_the_list_as_an_agent_prompt ... ok -test tests::the_treemap_zooms_with_the_wheel_and_resets ... ok -test tests::the_review_screen_switches_removal_mode ... ok +test tests::mouse_side_buttons_go_back_and_forward ... ok +test tests::marking_a_directory_marks_everything_inside_it ... ok +test tests::the_first_scan_shows_what_it_is_doing_then_the_treemap ... ok test tests::the_trash_is_the_default_when_there_is_one ... ok +test tests::the_mark_key_follows_the_pointer_until_the_keyboard_moves ... ok test tests::the_power_menu_offers_only_what_the_cpus_can_tell_apart ... ok -test tests::right_click_selects_the_tile_and_reveals_it ... ok test tests::the_volume_picker_is_centred_and_still_dismisses_from_outside ... ok +test tests::the_treemap_zooms_with_the_wheel_and_resets ... ok test tests::the_volume_picker_opens_moves_and_closes ... ok test tests::the_window_draws_a_treemap_with_tiles ... ok -test tests::switching_the_metric_keeps_the_directory_and_the_selection ... ok test tests::the_wheel_goes_into_the_directory_it_zoomed_into ... ok test tests::widening_reuses_the_tree_it_has_and_reads_only_the_rest ... ok test tests::typing_filters_live_and_enter_shows_only_the_matches ... ok test tests::marked_window_resolution_budget ... ok -test result: ok. 78 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 2.02s +test result: ok. 78 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 1.46s Running unittests src/lib.rs (target/debug/deps/disktree_core-9300e5ba20d6b25f) running 152 tests -test classify::tests::macos_developer_leftovers_are_reclaimable_and_its_risks_are_not ... ok -test classify::tests::the_legend_lists_every_named_category_once ... ok test classify::tests::steam_libraries_and_emulation_are_media_not_disposable_caches ... ok +test classify::tests::macos_developer_leftovers_are_reclaimable_and_its_risks_are_not ... ok test classify::tests::windows_caches_and_sync_folders_are_recognized ... ok +test classify::tests::the_legend_lists_every_named_category_once ... ok test filter::tests::case_is_ignored_without_allocating ... ok test export::tests::the_list_is_one_path_per_line ... ok -test export::tests::a_name_that_is_not_unicode_is_flagged_not_listed ... ok test export::tests::a_name_with_a_newline_cannot_smuggle_in_a_path ... ok +test export::tests::a_name_that_is_not_unicode_is_flagged_not_listed ... ok test export::tests::the_prompt_lists_every_path_with_its_size_and_the_rules ... ok -test filter::tests::a_search_below_the_root_leaves_the_rest_alone ... ok +test access::tests::only_macos_has_an_answer_and_a_missing_database_is_not_a_refusal ... ok test filter::tests::matches_are_kept_whole_and_their_ancestors_by_what_matched ... ok +test filter::tests::a_search_below_the_root_leaves_the_rest_alone ... ok test classify::tests::a_steam_library_on_another_drive_is_media_not_its_temp_folder ... ok +test insights::tests::ranks_findings_largest_first_and_skips_the_tiny ... ok +test insights::tests::documents_are_never_suggested ... ok test filter::tests::an_empty_needle_filters_nothing ... ok test insights::tests::the_limit_keeps_the_largest ... ok -test insights::tests::documents_are_never_suggested ... ok -test insights::tests::ranks_findings_largest_first_and_skips_the_tiny ... ok -test access::tests::only_macos_has_an_answer_and_a_missing_database_is_not_a_refusal ... ok -test classify::tests::target_is_build_output_only_beside_a_cargo_manifest ... ok +test classify::tests::an_unknown_top_level_directory_takes_its_largest_known_child ... ok test classify::tests::caches_are_reclaimable_all_the_way_down ... ok test insights::tests::crumbs_address_the_finding_from_the_root ... ok -test classify::tests::an_unknown_top_level_directory_takes_its_largest_known_child ... ok test classify::tests::names_announce_their_kind_and_children_inherit_it ... ok -test classify::tests::sandbox_layers_and_snapshots_are_reclaimable_only_in_sandbox_state ... ok +test classify::tests::target_is_build_output_only_beside_a_cargo_manifest ... ok test removal::tests::a_directory_holding_a_system_tree_is_refused ... ok test removal::tests::a_directory_holding_the_home_directory_is_refused ... ok +test classify::tests::sandbox_layers_and_snapshots_are_reclaimable_only_in_sandbox_state ... ok test macos::tests::a_missing_directory_is_an_error ... ok test removal::tests::macos_always_uses_the_system_trash ... ok test removal::tests::macos_guard_keys_fold_the_data_volume_and_case ... ok @@ -125,115 +126,115 @@ test removal::tests::a_name_that_is_not_utf8_is_refused_before_the_trash_sees_it test removal::tests::macos_system_trees_are_refused_but_apps_are_not ... ok test removal::tests::names_win32_would_misread_are_recognized ... ok test macos::tests::a_listing_has_names_kinds_lengths_and_no_dot_entries ... ok -test macos::tests::names_are_text_and_open ... ok -test removal::tests::nesting_uses_guard_case_order_even_when_the_child_sorts_first ... ok -test removal::tests::normalize_resolves_dots_without_the_filesystem ... ok test removal::tests::nesting_keeps_component_boundaries_duplicates_and_output_order ... ok -test macos::tests::every_leaf_reads_as_lstat_reads_it ... ok -test removal::tests::percent_encoding_escapes_what_the_spec_requires ... ok -test removal::tests::a_target_with_a_mount_inside_is_refused ... ok +test removal::tests::normalize_resolves_dots_without_the_filesystem ... ok +test removal::tests::nesting_uses_guard_case_order_even_when_the_child_sorts_first ... ok +test macos::tests::names_are_text_and_open ... ok test removal::tests::a_mount_below_a_target_is_caught_again_at_removal_time ... ok -test removal::tests::a_plan_keeps_only_the_outermost_targets ... ok -test removal::tests::a_missing_target_reports_not_found ... ok +test removal::tests::percent_encoding_escapes_what_the_spec_requires ... ok test removal::tests::a_volume_is_judged_by_its_mount_where_the_kernel_names_one ... ok +test removal::tests::a_missing_target_reports_not_found ... ok +test removal::tests::a_plan_keeps_only_the_outermost_targets ... ok test removal::tests::mount_points_are_recognized ... ok +test macos::tests::every_leaf_reads_as_lstat_reads_it ... ok +test removal::tests::a_target_with_a_mount_inside_is_refused ... ok test removal::tests::a_trash_info_path_is_restorable ... ok test removal::tests::paths_outside_the_root_are_refused ... ok test removal::tests::a_target_reached_through_a_link_is_refused ... ok -test removal::tests::home_is_found_by_its_real_path_too ... ok +test removal::tests::a_removal_run_reports_every_item_and_a_total ... ok test removal::tests::permanent_removal_takes_directories_and_leaves_siblings ... ok test removal::tests::the_macos_trash_takes_a_file_and_leaves_a_linked_target ... ignored, puts a file in this Mac's real Trash -test removal::tests::a_removal_run_reports_every_item_and_a_total ... ok +test removal::tests::home_is_found_by_its_real_path_too ... ok +test removal::tests::permanent_removal_follows_links_above_the_root ... ok test removal::tests::nothing_is_mounted_below_a_temp_dir ... ok +test removal::tests::permanent_removal_unlinks_a_symlink_instead_of_following_it ... ok test removal::tests::permanent_removal_unlinks_nested_symlinks_without_following_them ... ok -test removal::tests::a_link_put_on_the_way_down_after_planning_stops_the_worker ... ok -test removal::tests::permanent_removal_follows_links_above_the_root ... ok -test removal::tests::the_mount_table_is_read_off_the_calling_thread_and_then_kept ... ok test removal::tests::read_only_files_are_removed_too ... ok -test removal::tests::permanent_removal_does_not_follow_a_link_put_on_the_way_down ... ok -test removal::tests::permanent_removal_unlinks_a_symlink_instead_of_following_it ... ok test removal::tests::permanent_removal_takes_nested_trees_and_hidden_files ... ok -test removal::tests::the_root_and_home_are_refused ... ok +test removal::tests::permanent_removal_does_not_follow_a_link_put_on_the_way_down ... ok +test removal::tests::a_link_put_on_the_way_down_after_planning_stops_the_worker ... ok test removal::tests::trash_names_avoid_collisions ... ok +test removal::tests::the_root_and_home_are_refused ... ok test removal::tests::sibling_prefixes_are_not_treated_as_containment ... ok +test removal::tests::the_mount_table_is_read_off_the_calling_thread_and_then_kept ... ok test scan::tests::a_file_root_is_rejected ... ok -test removal::tests::this_macs_mount_table_is_read_without_proc ... ok -test removal::tests::the_xdg_trash_moves_a_file_and_records_where_it_came_from ... ok test scan::tests::fixed_worker_pools_change_size_without_reinitializing_rayon ... ok +test removal::tests::the_xdg_trash_moves_a_file_and_records_where_it_came_from ... ok +test scan::tests::a_directory_reports_its_own_bytes_separately ... ok +test removal::tests::this_macs_mount_table_is_read_without_proc ... ok test scan::tests::an_unreadable_directory_is_recorded_not_fatal ... ok test scan::tests::disk_usage_is_whole_allocation_units ... ok -test scan::tests::a_directory_reports_its_own_bytes_separately ... ok test scan::tests::children_are_ranked_largest_first ... ok test scan::tests::a_spawned_scan_reports_progress_and_a_result ... ok test scan::tests::whole_disk_smoke ... ignored, walks the whole disk -test scan::tests::followed_symlink_loops_do_not_hang_the_scan ... ok -test scan_threads::tests::governor_waits_for_retirement_and_recovers_without_exceeding_target ... ok test scan::tests::hardlinks_are_charged_once_by_default ... ok +test scan_threads::tests::governor_waits_for_retirement_and_recovers_without_exceeding_target ... ok test scan_threads::tests::inconclusive_windows_cannot_accept_a_reduction ... ok +test scan::tests::hidden_entries_are_included_by_default_and_excluded_on_request ... ok test scan_threads::tests::increased_limit_is_not_supplied_until_jobs_are_dispatched ... ok test scan_threads::tests::long_directory_never_becomes_a_fully_supplied_window ... ok +test scan::tests::followed_symlink_loops_do_not_hang_the_scan ... ok test scan_threads::tests::search_refines_one_reference_without_compounding_loss ... ok test size::tests::a_value_that_rounds_up_to_the_next_unit_is_shown_in_it ... ok test size::tests::bytes_use_binary_units_with_one_decimal_below_ten ... ok -test scan::tests::a_wider_scan_reuses_the_subtree_it_already_knows ... ok test size::tests::counts_switch_to_metric_suffixes ... ok test size::tests::share_bar_is_always_the_requested_width ... ok test size::tests::share_survives_a_zero_total ... ok test size::tests::short_form_drops_the_space ... ok -test space::tests::a_bind_mount_inside_the_scan_is_left_out ... ok test space::tests::a_home_scan_has_no_foreign_mounts_here ... ok +test space::tests::a_bind_mount_inside_the_scan_is_left_out ... ok +test scan::tests::a_wider_scan_reuses_the_subtree_it_already_knows ... ok test space::tests::a_missing_path_reports_the_io_error ... ok test space::tests::a_real_volume_reports_plausible_numbers ... ok -test scan::tests::hidden_entries_are_included_by_default_and_excluded_on_request ... ok test space::tests::a_root_booted_from_a_snapshot_keeps_its_subvolumes ... ok test space::tests::a_table_that_does_not_place_a_path_claims_nothing ... ok -test scan_threads::tests::real_pool_retires_at_job_boundaries_and_drains_queued_descendants ... ok test space::tests::a_volume_includes_its_subvolumes_and_nothing_else ... ok test space::tests::a_volume_keeps_its_subvolumes_and_loses_everything_else ... ok +test scan::tests::max_depth_stops_descending_but_keeps_direct_files ... ok test space::tests::btrfs_subvolume_devices_can_differ ... ok test space::tests::excluded_views_cannot_replace_visible_files ... ok test space::tests::hidden_original_and_nested_mounts_are_preserved ... ok +test scan_threads::tests::real_pool_retires_at_job_boundaries_and_drains_queued_descendants ... ok test space::tests::macos_mount_points_keep_spaces_and_brackets ... ok test space::tests::mountinfo_escapes_are_decoded_once ... ok test space::tests::only_macos_skips_the_data_volume_and_in_the_roots_spelling ... ok test space::tests::projecting_removal_cannot_exceed_the_volume ... ok test space::tests::separate_home_disk_is_its_own_volume ... ok +test space::tests::subvolumes_are_not_repeats_but_a_second_mount_of_a_disk_is ... ok test space::tests::the_device_is_the_longest_matching_mount ... ok test space::tests::the_home_disk_on_macos_is_the_root_and_has_a_device ... ok -test space::tests::subvolumes_are_not_repeats_but_a_second_mount_of_a_disk_is ... ok -test space::tests::the_top_of_a_btrfs_disk_beside_its_subvolumes_is_counted_once ... ok test space::tests::the_whole_disk_is_the_top_of_the_home_volume ... ok +test space::tests::the_top_of_a_btrfs_disk_beside_its_subvolumes_is_counted_once ... ok test space::tests::unrelated_tmpfs_mounts_are_not_duplicates ... ok test space::tests::used_fraction_handles_an_empty_volume_report ... ok -test scan::tests::symlinks_are_not_followed_by_default ... ok -test space::tests::volume_candidates_are_real_disks_not_pseudo_filesystems ... ok test space::tests::volumes_with_least_space_come_first_and_unknowns_last ... ok +test space::tests::volume_candidates_are_real_disks_not_pseudo_filesystems ... ok +test scan::tests::symlinks_are_not_followed_by_default ... ok test tree::tests::a_node_stays_small ... ok +test tree::tests::depth_counts_edges ... ok test tree::tests::aggregate_derives_totals_and_orders_children ... ok test tree::tests::find_reports_crumbs_and_is_case_insensitive ... ok -test tree::tests::depth_counts_edges ... ok test tree::tests::identities_that_differ_only_in_their_top_bits_are_two_files ... ok -test tree::tests::path_of_joins_names_beneath_the_root ... ok test tree::tests::largest_child_is_the_first_child_because_children_are_sorted ... ok -test tree::tests::aggregate_can_rank_by_file_count ... ok +test tree::tests::path_of_joins_names_beneath_the_root ... ok test tree::tests::resolve_walks_child_indices ... ok -test scan::tests::file_count_metric_ranks_by_entries ... ok +test tree::tests::aggregate_can_rank_by_file_count ... ok test treemap::tests::a_leaf_never_claims_a_header ... ok +test treemap::tests::a_filtered_layout_shows_only_the_matches_at_their_size ... ok test treemap::tests::a_tile_without_room_for_a_header_stays_whole ... ok +test treemap::tests::a_subdivided_directory_keeps_a_header_for_its_own_name ... ok test treemap::tests::hit_returns_the_deepest_tile ... ok test treemap::tests::layout_depth_one_stops_at_direct_children ... ok -test treemap::tests::a_filtered_layout_shows_only_the_matches_at_their_size ... ok test treemap::tests::layout_merges_the_child_tail_into_one_tile ... ok -test treemap::tests::a_subdivided_directory_keeps_a_header_for_its_own_name ... ok test treemap::tests::squarify_fills_the_area ... ok -test treemap::tests::layout_nests_children_inside_their_parent ... ok test treemap::tests::squarify_keeps_aspect_ratios_reasonable ... ok +test treemap::tests::layout_nests_children_inside_their_parent ... ok test treemap::tests::squarify_survives_degenerate_input ... ok test treemap::tests::tile_crumbs_address_the_scanned_tree_not_the_drawn_node ... ok -test tree::tests::seen_charges_an_identity_once_in_either_store ... ok -test scan::tests::max_depth_stops_descending_but_keeps_direct_files ... ok -test space::tests::without_a_table_the_same_place_is_one_volume ... ok test scan::tests::totals_are_summed_bottom_up ... ok +test space::tests::without_a_table_the_same_place_is_one_volume ... ok +test tree::tests::seen_charges_an_identity_once_in_either_store ... ok +test scan::tests::file_count_metric_ranks_by_entries ... ok test removal::tests::system_trees_are_refused_in_a_whole_disk_scan ... ok test removal::tests::the_directories_above_home_are_refused ... ok test removal::tests::removal_nesting_budget ... ok @@ -245,14 +246,14 @@ test scan_threads::tests::controller_retires_recovers_after_search_and_cancels . test macos::tests::a_directory_larger_than_the_buffer_is_listed_whole ... ok test removal::tests::a_trash_tool_is_called_with_the_path_after_a_separator ... ok -test result: ok. 150 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.27s +test result: ok. 150 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.20s Running unittests src/main.rs (target/debug/deps/xtask-0dc2df263f51d89e) running 3 tests +test bundle::tests::signing_is_ad_hoc_unless_an_identity_is_given ... ok test bundle::tests::notarizing_needs_a_real_signature ... ok test bundle::tests::the_plist_template_names_the_executable_and_takes_the_version ... ok -test bundle::tests::signing_is_ad_hoc_unless_an_identity_is_given ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s From 04b33b364e4949dc536feaac29bd97618afc9ece Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 14:29:13 -0400 Subject: [PATCH 07/19] docs: qualify mark interaction performance evidence --- docs/performance/apfs-accounting/README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/performance/apfs-accounting/README.md b/docs/performance/apfs-accounting/README.md index 42c0f10d..dbc97f92 100644 --- a/docs/performance/apfs-accounting/README.md +++ b/docs/performance/apfs-accounting/README.md @@ -47,6 +47,14 @@ The main-only red/green proof is separate in [PR #9](https://github.com/huntharo/disktree/pull/9) and [mark-resolution](../mark-resolution/README.md). This branch incorporates that fix and reuses one lazy sibling resolver per accounting cache miss. +One mark is one queued path, independent of its descendant count. The +1,024-mark case is a stress workload, with no usage telemetry establishing +that it is common. The earlier 6.60 s measured four forced Explore draws +and eight prepares, not a scan or one click. +[Individual interaction evidence](../mark-resolution/interactions.md) +separates Space, hover and Review, includes smaller queues, and records +render/plan counts with zero scan refreshes. Its timings compare the +isolated main-only snapshots, not this APFS feature branch. Removal authorization remains fresh on every plan and in the worker. Selected subtrees merge bounded maps small to large: expected @@ -101,5 +109,7 @@ cargo run --release -p disktree-core --example scan_accounting -- PATH ``` Local validation: lint passed; 79 window/app, 174 core and 3 xtask tests passed; -two existing core tests remained ignored. No private file contents were +two existing core tests and the opt-in interaction timing test remained +ignored in the normal suite. The interaction timing test was also run +explicitly and passed on the combined branch. No private file contents were inspected and no user files were removed by this investigation. From 786ae6d9b45a8d4ac71047d068a407e92b86d88f Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 15:01:26 -0400 Subject: [PATCH 08/19] fix: make non-Unix link count fallback const --- crates/disktree-core/src/scan.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rs index c2e35511..a7430e02 100644 --- a/crates/disktree-core/src/scan.rs +++ b/crates/disktree-core/src/scan.rs @@ -600,7 +600,7 @@ fn link_count(meta: &Metadata) -> u64 { } #[cfg(not(unix))] -fn link_count(_meta: &Metadata) -> u64 { +const fn link_count(_meta: &Metadata) -> u64 { 1 } From 23bb26e44f0a5817a51fe8ff495415dbf41133f8 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 15:30:33 -0400 Subject: [PATCH 09/19] docs: audit remaining queue rendering and scan complexity --- docs/performance/mark-resolution/README.md | 8 + .../mark-resolution/additional-audit.log | 50 ++ .../mark-resolution/additional-audit.md | 186 ++++++ .../mark-resolution/additional-audit.patch | 561 ++++++++++++++++++ 4 files changed, 805 insertions(+) create mode 100644 docs/performance/mark-resolution/additional-audit.log create mode 100644 docs/performance/mark-resolution/additional-audit.md create mode 100644 docs/performance/mark-resolution/additional-audit.patch diff --git a/docs/performance/mark-resolution/README.md b/docs/performance/mark-resolution/README.md index b8838306..4bc40c0c 100644 --- a/docs/performance/mark-resolution/README.md +++ b/docs/performance/mark-resolution/README.md @@ -21,6 +21,14 @@ large improvement applies to large queues. At 1,024 queued folders, adding one more or hovering takes about 1.1 seconds on main in that fixture. These costs can recur on renders while interacting, rather than once per scan. +## Broader follow-up audit + +[Additional rendering/scanning findings](additional-audit.md) document +remaining queue-construction, parent-absorption, Review status-join, depth and +mount-topology cases, plus a case-sensitive classification edge case. These +are separately scoped follow-ups; PR #9's three original fixes do not remove +them. Reproducible test-only probes and raw counts are included. + ## Red before green Base: `6c8d4ce6211bf135ff4e42890faebff1040e1846` (upstream/main and fork main). diff --git a/docs/performance/mark-resolution/additional-audit.log b/docs/performance/mark-resolution/additional-audit.log new file mode 100644 index 00000000..935b5e92 --- /dev/null +++ b/docs/performance/mark-resolution/additional-audit.log @@ -0,0 +1,50 @@ + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 6.09s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 1 test +test tests::additional_render_complexity_audit ... audit bulk-mark M=128 ancestor-visits=8128 inside-visits=8256 wall=3.146875ms +audit absorb-parent M=128 retain-visits=8384 wall=294.333µs +audit blocked-review M=128 row-checks=66048 renders=8 per-render=8256 wall=119.267584ms +audit covered-review children=128 row-checks=67072 renders=8 per-render=8384 wall=135.083208ms +audit bulk-mark M=1024 ancestor-visits=523776 inside-visits=524800 wall=199.863084ms +audit absorb-parent M=1024 retain-visits=525824 wall=14.79025ms +audit blocked-review M=1024 row-checks=4198400 renders=8 per-render=524800 wall=1.101053583s +audit covered-review children=1024 row-checks=4206592 renders=8 per-render=525824 wall=1.23678775s +audit deep-breadcrumbs D=128 prefix-walks=65024 renders=8 per-render=8128 wall=12.958166ms +audit deep-breadcrumbs D=1024 prefix-walks=4190208 renders=8 per-render=523776 wall=33.87925ms +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 79 filtered out; finished in 3.29s + + Running unittests src/lib.rs (target/debug/deps/disktree_core-9300e5ba20d6b25f) + +running 4 tests +test classify::tests::additional_classification_case_variant_audit ... audit case-variant-classify F=128 sibling-checks=16384 wall=84.834µs +audit case-variant-classify F=1024 sibling-checks=1048576 wall=2.682292ms +ok +test scan::tests::additional_scan_complexity_audit ... audit real-scan nested=false F=128 listed=128 aggregate-visits=129 wall=574.375µs +audit real-scan nested=false F=1024 listed=1024 aggregate-visits=1025 wall=2.407709ms +audit real-scan nested=true F=128 listed=256 aggregate-visits=257 wall=2.721ms +audit real-scan nested=true F=1024 listed=2048 aggregate-visits=2049 wall=18.88725ms +ok +test space::tests::additional_mount_complexity_audit ... audit mount-startup K=128 pairs=8128 wall=83.875µs +audit mount-startup K=1024 pairs=523776 wall=2.992958ms +ok +test treemap::tests::additional_layout_complexity_audit ... audit uncapped-squarify F=128 ratio-visits=682 wall=4.5µs +audit capped-layout F=128 ratio-visits=482 wall=44.834µs +audit uncapped-squarify F=1024 ratio-visits=12525 wall=24.375µs +audit capped-layout F=1024 ratio-visits=319 wall=21.125µs +ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 152 filtered out; finished in 0.47s + + Running unittests src/main.rs (target/debug/deps/xtask-0dc2df263f51d89e) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 3 filtered out; finished in 0.00s + diff --git a/docs/performance/mark-resolution/additional-audit.md b/docs/performance/mark-resolution/additional-audit.md new file mode 100644 index 00000000..ce4f0675 --- /dev/null +++ b/docs/performance/mark-resolution/additional-audit.md @@ -0,0 +1,186 @@ +# Additional rendering and scanning complexity audit + +Audited on 2026-10-14 at `ab11b1fd884e950faa9902c0b3e0e870eb6815d9`: +main `6c8d4ce` plus PR #9's fixes. APFS feature code is outside this audit. +The cases below remain after the original three fixes. This follow-up records +findings; it adds no further runtime or API change. + +[Raw probe output](additional-audit.log) and the +[reproducible test-only probe patch](additional-audit.patch) preserve the +measurements. The probes drive production functions; Review and breadcrumb +measurements draw the real GPUI window. Fixtures are synthetic in-memory trees +or real private temporary trees. No real removal or whole-home scan ran. + +M = marks, B = blocked marks, C = covered marks, D = path depth, +F = entries in one directory, K = mount-table entries. All operation counts +below compare 128 with 1,024 of the named variable, an 8x increase. Counts are +deterministic; recorded wall times are single instrumented test-profile runs, +not latency budgets or production timing guarantees. + +## Confirmed follow-ups + +| Operation | Variable / measured scope | Work at 128 | Work at 1,024 | Growth | +| --- | --- | ---: | ---: | ---: | +| Build a queue by calling `toggle_mark` repeatedly | M, no renders | 16,384 mark visits | 1,048,576 | 64.0x | +| Mark a parent over marked children | M children, one action | 8,384 retain visits | 525,824 | 62.7x | +| Render Review with all paths blocked | M, per actual render | 8,256 comparisons | 524,800 | 63.6x | +| Render Review with children covered by one parent | M children, per actual render | 8,384 comparisons | 525,824 | 62.7x | +| Build deep breadcrumbs | D, per actual render | 8,128 prefix index steps | 523,776 | 64.4x | +| Detect repeated Linux mount views | K, one startup calculation | 8,128 mount-pair checks | 523,776 | 64.4x | +| Classify case variants of `node_modules` | F, one finishing pass | 16,384 sibling checks | 1,048,576 | 64.0x | + +### Queue construction and parent absorption: highest MCP relevance + +[`Disktree::toggle_mark`](../../../crates/disktree-app/src/state.rs) invokes +`marked_ancestor`, which scans all existing marks, then scans marks again for +paths inside the new target. Repeating the operation M times costs O(M^2) +even if every mark is a disjoint file and rendering is suppressed. The probe +calls the real state transition without drawing between insertions. At 1,024 +marks it measured about 200 ms with probes, versus 3 ms at 128. + +Marking a parent over K marked descendants then calls `Marks::remove` K +times. Each call compacts the ordered target vector with `retain`: +O(K*M), or O(M^2) when most marks are descendants. This is one user action, +not the cost of the parent's scan. The 1,024-child probe measured ~15 ms; +operation counts are the stronger scaling evidence. + +A future MCP batch API should normalize the entire queue once, preserving +mark ordering and path semantics. Merely suppressing redraws while looping +over the scalar marking API does not remove this cost. Parent absorption +should remove the whole descendant set with one compaction. Ancestor +membership can use indexed ancestor paths instead of scanning every mark. + +### Review: row-status joins still use repeated linear lookup + +[`views::review`](../../../crates/disktree-app/src/views.rs) runs +`covered.iter().any(...)` and `blocked.iter().find(...)` for every displayed +item. This is O(min(M, 1200) * (B+C)) in the general case. It behaves +quadratically below the 1,200-row cap when blocked/covered counts grow with +the queue; above the cap it has a bounded displayed-row factor. It is not an +unbounded quadratic pass over every file on disk. + +The ordinary accepted-path fixture in the original budget had B=C=0, so it +could not expose these joins. The new blocked fixture queues paths outside +the scanned root; guards refuse them, with no deletion attempt. The covered +fixture supplies an overlapping raw queue. Normal manual marking absorbs +such overlaps, so the latter represents a raw/programmatic queue or other +path-alias situations, rather than an ordinary manual selection sequence. + +Each test forces four Review draws and observes eight real renders. Logs +report both totals and comparisons per render. The ~1.10 s blocked and +~1.24 s covered wall samples at 1,024 are those **four-draw totals**, not +single clicks. A per-plan lookup map/set for row status would avoid repeated +searches while leaving removal guards freshly evaluated. + +### Breadcrumbs and deep filtering: depth-dependent costs + +[`Disktree::breadcrumbs`](../../../crates/disktree-app/src/state.rs) starts at +the scan root again for every successive prefix, producing D(D-1)/2 indexed +walk steps. It also clones crumb prefixes. The view later hides most of the +trail, after the full trail has already been materialized. Above-root path +ancestors are similarly copied before this display truncation. + +The probe sets current depth to 128 and 1,024 in memory and draws the actual +Explore window. Deep chains like this are an edge workload, not evidence +that a common shallow home directory suffers this cost. Walking the node +chain once and building only displayed breadcrumb targets would bound it. + +Separately, `Matches::keep` and marked-ancestor coloring hash successive crumb +prefixes. Hashing each prefix depends on its length, so these can also cost +O(D^2) per tile when the absolute path is deep and the filter/marks require +checking all ancestors. Those hash costs were source-audited, not separately +measured by this breadcrumb counter. They depend on path depth, not the +number of siblings in one folder. + +### Linux mount setup: topology-dependent, not file-count-dependent + +[`space::repeated_mounts_excluding`](../../../crates/disktree-core/src/space.rs) +compares each visible mount with previously kept mounts. The probe uses +independent synthetic devices so no early match masks K(K-1)/2 comparisons. +This runs while preparing a scan on systems with `/proc/self/mountinfo`. +The algorithm was measured on synthetic Linux mount metadata on macOS; +there was no actual Linux mount manipulation or Linux wall-time benchmark. + +Other code in that routine can scan the full mount table or exclusion list +inside these checks, so more complex topologies deserve separate bounds. +The volume picker has a similar linear search through already seen devices. +Removal guards and volume attribution also retain O(M*K) checks per plan. +None of these is quadratic in the number of files with a fixed mount table. +The simple 1,024-mount synthetic case took ~3 ms here, so this ranks below +queue construction and Review for the proposed MCP flow. + +### Directory classification: a real but unusual within-folder case + +[`classify::reclaim_of`](../../../crates/disktree-core/src/classify.rs) requests +a sibling search only for particular names: `target`, `node_modules`, and +`logs`. Ordinary folders do not trigger a sibling scan for every arbitrary +entry. That apparent generic F^2 pattern is therefore usually a false alarm. + +However, the directory-name match is ASCII case-insensitive. A case-sensitive +filesystem can contain many distinct spellings of `node_modules`, each of +which searches all siblings for an absent `package.json`. The probe creates +128/1,024 such names in memory and calls full `classify`, observing F^2 +comparisons. There are only 2^11 case variants of this particular name, so +this is a quadratic window up to a finite namespace bound, not an unbounded +quadratic algorithm for arbitrary folder sizes. These variants cannot all +coexist on a normal case-insensitive APFS volume; this is chiefly a Linux or +case-sensitive-volume edge case. It is scan finishing, not per-frame work. + +Precomputing the few manifest/sibling flags once per parent would remove the +case without adding fields to every scanned node. + +## Normal scanning and bounded layout did not show this problem + +The regular walk appends each directory entry once. Directory completion +moves completed children into their parent, and aggregation visits each node +once before sorting each folder's children. Hardlink de-duplication and +followed-directory detection use a bitmap/hash sets, not a search through all +previous files. The admission queue removes an admitted tail, avoiding a +front-removal shift per job. + +Actual fixed-one-thread scans of private temporary trees produced: + +| Fixture | Listed entries at 128 / 1,024 items | Aggregate node visits | +| --- | ---: | ---: | +| One directory of files | 128 / 1,024 | 129 / 1,025 | +| One directory of folders, one file in each | 256 / 2,048 | 257 / 2,049 | + +Those counts are linear. Sorting is O(sum(F_i log F_i)) over directories, +with additional filename/path-length and filesystem costs. This does not +prove every input or OS path is linear; it rules out a general sibling-pair +loop in the normal walk/aggregation paths exercised here. + +`treemap::squarify` recomputes worst aspect ratios across a growing row, which +is quadratic within a row. With unbounded equal-weight input, ratio-element +visits rose from 682 to 12,525 (18.4x at 8x input): superlinear, not a measured +64x file-count quadratic. The real layout caller keeps only 96 children plus +one merged tail per directory before this step. Its measured ratio work was +482 versus 319 at 128/1,024 source entries. Ranking all source entries still +costs O(F log F) on a layout cache miss; normal unchanged frames reuse layout. +The internal row rescan therefore is not currently an unbounded wide-folder +rendering blocker. + +Windows MFT enumeration builds parent indexes and sorts names rather than +repeatedly searching every sibling. One additional **unmeasured** metadata +candidate is `mft::table_layout`'s vector `numbers.contains` while collecting +extension records from the MFT's own attribute list. That can be O(E^2) in +extension-record count E on a fragmented MFT; E is not the total file count. +Windows runtime behavior was not tested in this macOS audit. + +## Reproduce without changing runtime code + +From clean source at `ab11b1f` (or later evidence-only descendants): + +```sh +git apply docs/performance/mark-resolution/additional-audit.patch +cargo test --workspace additional_ -- --ignored --nocapture --test-threads=1 +git apply -R docs/performance/mark-resolution/additional-audit.patch +``` + +The patch adds counters gated by `cfg(test)` and ignored diagnostic tests. +All five diagnostic tests passed their correctness/call-path assertions. It +records counts rather than adding passing assertions that would enshrine +quadratic behavior. The patch was then removed: source remains byte-identical +to `ab11b1f`. The original lint/full-test result therefore still applies to +the unchanged runtime implementation. These findings need separate fixes +and corresponding failing budgets before their scaling can be called bounded. diff --git a/docs/performance/mark-resolution/additional-audit.patch b/docs/performance/mark-resolution/additional-audit.patch new file mode 100644 index 00000000..99a7e5f8 --- /dev/null +++ b/docs/performance/mark-resolution/additional-audit.patch @@ -0,0 +1,561 @@ +diff --git a/crates/disktree-app/src/marks.rs b/crates/disktree-app/src/marks.rs +index 3a86060..8d9578a 100644 +--- a/crates/disktree-app/src/marks.rs ++++ b/crates/disktree-app/src/marks.rs +@@ -18,6 +18,11 @@ pub mod work { + thread_local! { + pub static RESOLUTION: Cell = const { Cell::new(0) }; + pub static FRAMES: Cell = const { Cell::new(0) }; ++ pub static AUDIT_ROWS: Cell = const { Cell::new(0) }; ++ pub static AUDIT_RETAIN: Cell = const { Cell::new(0) }; ++ pub static AUDIT_ANCESTORS: Cell = const { Cell::new(0) }; ++ pub static AUDIT_INSIDE: Cell = const { Cell::new(0) }; ++ pub static AUDIT_CRUMBS: Cell = const { Cell::new(0) }; + pub static RENDERS: Cell = const { Cell::new(0) }; + pub static PLANS: Cell = const { Cell::new(0) }; + pub static REFRESHES: Cell = const { Cell::new(0) }; +@@ -31,6 +36,11 @@ pub mod work { + pub fn reset() { + RESOLUTION.set(0); + FRAMES.set(0); ++ AUDIT_ROWS.set(0); ++ AUDIT_RETAIN.set(0); ++ AUDIT_ANCESTORS.set(0); ++ AUDIT_INSIDE.set(0); ++ AUDIT_CRUMBS.set(0); + RENDERS.set(0); + PLANS.set(0); + REFRESHES.set(0); +@@ -91,7 +101,11 @@ impl Marks { + pub fn remove(&mut self, path: &Path) { + if self.index.remove(path) { + self.resolved = None; +- self.items.retain(|item| item.path != path); ++ self.items.retain(|item| { ++ #[cfg(test)] ++ work::AUDIT_RETAIN.set(work::AUDIT_RETAIN.get() + 1); ++ item.path != path ++ }); + } + } + +diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs +index 9713e0c..ce234ec 100644 +--- a/crates/disktree-app/src/state.rs ++++ b/crates/disktree-app/src/state.rs +@@ -1120,6 +1120,9 @@ impl Disktree { + }; + let mut crumbs: Vec = Vec::new(); + for &index in &self.crumbs { ++ #[cfg(test)] ++ crate::marks::work::AUDIT_CRUMBS ++ .set(crate::marks::work::AUDIT_CRUMBS.get() + crumbs.len()); + let Some(node) = + tree.resolve(&crumbs).and_then(|node| node.child(index)) + else { +@@ -1711,6 +1714,9 @@ impl Disktree { + .items() + .iter() + .filter(|item| { ++ #[cfg(test)] ++ crate::marks::work::AUDIT_INSIDE ++ .set(crate::marks::work::AUDIT_INSIDE.get() + 1); + item.path != path && item.path.starts_with(&path) + }) + .map(|item| item.path.clone()) +@@ -1738,7 +1744,12 @@ impl Disktree { + self.marks + .items() + .iter() +- .filter(|item| item.path != path && path.starts_with(&item.path)) ++ .filter(|item| { ++ #[cfg(test)] ++ crate::marks::work::AUDIT_ANCESTORS ++ .set(crate::marks::work::AUDIT_ANCESTORS.get() + 1); ++ item.path != path && path.starts_with(&item.path) ++ }) + .map(|item| item.path.clone()) + .min_by_key(|ancestor| ancestor.as_os_str().len()) + } +diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs +index 04bcc6f..3d08a34 100644 +--- a/crates/disktree-app/src/tests.rs ++++ b/crates/disktree-app/src/tests.rs +@@ -2118,3 +2118,141 @@ fn marked_interaction_timings(cx: &mut TestAppContext) { + window.update(|window, _| window.remove_window()); + } + } ++ ++#[gpui_kit::test] ++#[ignore = "manual additional complexity audit on synthetic state/windows"] ++fn additional_render_complexity_audit(cx: &mut TestAppContext) { ++ use crate::marks::work; ++ use disktree_core::removal::Target; ++ use disktree_core::tree::{Node, NodeKind}; ++ ++ cx.update(gpui_omarchy::init); ++ let temp = tempfile::tempdir().expect("tempdir"); ++ let scan_root = temp.path().join("scan"); ++ std::fs::create_dir(&scan_root).expect("mkdir"); ++ for count in [128, 1024] { ++ let root_path = scan_root.clone(); ++ let mut tree = Node::directory("root"); ++ let mut parent = Node::directory("container"); ++ for index in 0..count { ++ parent.children.push(Node::entry( ++ format!("file-{index}"), ++ NodeKind::File, ++ 1, ++ )); ++ } ++ tree.children.push(parent); ++ let (view, window) = cx.add_window_view(move |_, cx| { ++ Disktree::with_tree(root_path.clone(), tree, options(), 3, cx) ++ }); ++ update(&view, window, |app, cx| { ++ work::reset(); ++ let start = std::time::Instant::now(); ++ for index in 0..count { ++ app.toggle_mark(&[0, index], cx); ++ } ++ assert_eq!(app.marks.len(), count); ++ eprintln!( ++ "audit bulk-mark M={count} ancestor-visits={} inside-visits={} wall={:?}", ++ work::AUDIT_ANCESTORS.get(), ++ work::AUDIT_INSIDE.get(), ++ start.elapsed() ++ ); ++ work::reset(); ++ let start = std::time::Instant::now(); ++ app.toggle_mark(&[0], cx); ++ assert_eq!(app.marks.len(), 1); ++ assert!(app.marks.contains(&scan_root.join("container"))); ++ eprintln!( ++ "audit absorb-parent M={count} retain-visits={} wall={:?}", ++ work::AUDIT_RETAIN.get(), ++ start.elapsed() ++ ); ++ app.marks.clear(); ++ for index in 0..count { ++ app.marks.toggle(Target { ++ path: temp.path().join(format!("outside-{index}")), ++ bytes: 1, ++ is_dir: false, ++ hidden: false, ++ }); ++ } ++ app.screen = Screen::Review; ++ }); ++ work::reset(); ++ let start = std::time::Instant::now(); ++ for _ in 0..4 { ++ update(&view, window, |_, cx| cx.notify()); ++ draw(window); ++ } ++ let frames = work::RENDERS.get(); ++ assert!(frames >= 4); ++ eprintln!( ++ "audit blocked-review M={count} row-checks={} renders={frames} per-render={} wall={:?}", ++ work::AUDIT_ROWS.get(), ++ work::AUDIT_ROWS.get() / frames, ++ start.elapsed() ++ ); ++ update(&view, window, |app, cx| { ++ // Manual marking absorbs these; raw/programmatic queues can ++ // still present overlapping targets to the review renderer. ++ app.marks.clear(); ++ let parent = app.target_at(&[0]).expect("parent"); ++ app.marks.toggle(parent); ++ for index in 0..count { ++ let child = app.target_at(&[0, index]).expect("child"); ++ app.marks.toggle(child); ++ } ++ cx.notify(); ++ }); ++ work::reset(); ++ let start = std::time::Instant::now(); ++ for _ in 0..4 { ++ update(&view, window, |_, cx| cx.notify()); ++ draw(window); ++ } ++ let frames = work::RENDERS.get(); ++ assert!(frames >= 4); ++ eprintln!( ++ "audit covered-review children={count} row-checks={} renders={frames} per-render={} wall={:?}", ++ work::AUDIT_ROWS.get(), ++ work::AUDIT_ROWS.get() / frames, ++ start.elapsed() ++ ); ++ window.update(|window, _| window.remove_window()); ++ } ++ for depth in [128, 1024] { ++ let root_path = scan_root.clone(); ++ let mut tree = Node::directory("x"); ++ tree.children ++ .push(Node::entry("payload", NodeKind::File, 1)); ++ for _ in 0..depth { ++ let mut parent = Node::directory("x"); ++ parent.children.push(tree); ++ tree = parent; ++ } ++ let (view, window) = cx.add_window_view(move |_, cx| { ++ Disktree::with_tree(root_path.clone(), tree, options(), 3, cx) ++ }); ++ update(&view, window, |app, cx| { ++ app.crumbs = vec![0; depth]; ++ app.selected = None; ++ cx.notify(); ++ }); ++ work::reset(); ++ let start = std::time::Instant::now(); ++ for _ in 0..4 { ++ update(&view, window, |_, cx| cx.notify()); ++ draw(window); ++ } ++ let frames = work::RENDERS.get(); ++ assert!(frames >= 4); ++ eprintln!( ++ "audit deep-breadcrumbs D={depth} prefix-walks={} renders={frames} per-render={} wall={:?}", ++ work::AUDIT_CRUMBS.get(), ++ work::AUDIT_CRUMBS.get() / frames, ++ start.elapsed() ++ ); ++ window.update(|window, _| window.remove_window()); ++ } ++} +diff --git a/crates/disktree-app/src/views.rs b/crates/disktree-app/src/views.rs +index c23e91b..ff3acad 100644 +--- a/crates/disktree-app/src/views.rs ++++ b/crates/disktree-app/src/views.rs +@@ -2544,11 +2544,20 @@ fn review( + } + + for (index, item) in items.iter().take(LIST_LIMIT).enumerate() { +- let is_covered = +- covered.iter().any(|covered| covered.path == item.path); ++ let is_covered = covered.iter().any(|covered| { ++ #[cfg(test)] ++ crate::marks::work::AUDIT_ROWS ++ .set(crate::marks::work::AUDIT_ROWS.get() + 1); ++ covered.path == item.path ++ }); + let blocked_reason = blocked + .iter() +- .find(|blocked| blocked.path == item.path) ++ .find(|blocked| { ++ #[cfg(test)] ++ crate::marks::work::AUDIT_ROWS ++ .set(crate::marks::work::AUDIT_ROWS.get() + 1); ++ blocked.path == item.path ++ }) + .map(|blocked| blocked.reason.clone()); + list = list.child(mark_row( + index, +diff --git a/crates/disktree-core/src/classify.rs b/crates/disktree-core/src/classify.rs +index bfe24a7..82ffd9e 100644 +--- a/crates/disktree-core/src/classify.rs ++++ b/crates/disktree-core/src/classify.rs +@@ -17,6 +17,10 @@ use rayon::prelude::*; + + use crate::tree::Node; + ++#[cfg(test)] ++static AUDIT_SIBLINGS: std::sync::atomic::AtomicUsize = ++ std::sync::atomic::AtomicUsize::new(0); ++ + /// A kind of data, for colour. + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] + pub enum Category { +@@ -231,8 +235,14 @@ pub fn classify(root: &mut Node) { + root.reclaim = None; + for index in 0..root.children.len() { + let siblings = &root.children; +- let has_sibling = +- |wanted: &str| siblings.iter().any(|name| &*name.name == wanted); ++ let has_sibling = |wanted: &str| { ++ siblings.iter().any(|name| { ++ #[cfg(test)] ++ AUDIT_SIBLINGS ++ .fetch_add(1, std::sync::atomic::Ordering::Relaxed); ++ &*name.name == wanted ++ }) ++ }; + let child = &siblings[index]; + // A top-level directory with an unknown name takes the kind of its + // largest recognisable child: `~/world` is mostly `.git`. +@@ -303,8 +313,13 @@ fn kind_of( + if !child.is_dir() { + return (category, reclaim); + } +- let has_sibling = +- |wanted: &str| siblings.iter().any(|name| &*name.name == wanted); ++ let has_sibling = |wanted: &str| { ++ siblings.iter().any(|name| { ++ #[cfg(test)] ++ AUDIT_SIBLINGS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); ++ &*name.name == wanted ++ }) ++ }; + let child_category = category_of_name(&child.name) + .or_else(|| is_git_store(child).then_some(Category::Git)) + .unwrap_or(category); +@@ -616,4 +631,39 @@ mod tests { + assert!(!category.label().is_empty()); + } + } ++ #[test] ++ #[ignore = "manual case-sensitive sibling classification edge-case audit"] ++ fn additional_classification_case_variant_audit() { ++ for count in [128, 1024] { ++ let mut root = Node::directory("root"); ++ for index in 0..count { ++ let mut mask = 1; ++ let name: String = "node_modules" ++ .chars() ++ .map(|letter| { ++ if letter == '_' { ++ return letter; ++ } ++ let upper = index & mask != 0; ++ mask <<= 1; ++ if upper { ++ letter.to_ascii_uppercase() ++ } else { ++ letter ++ } ++ }) ++ .collect(); ++ root.children.push(Node::directory(name)); ++ } ++ AUDIT_SIBLINGS.store(0, std::sync::atomic::Ordering::Relaxed); ++ let start = std::time::Instant::now(); ++ classify(&mut root); ++ assert!(root.children.iter().all(|child| child.reclaim.is_none())); ++ eprintln!( ++ "audit case-variant-classify F={count} sibling-checks={} wall={:?}", ++ AUDIT_SIBLINGS.load(std::sync::atomic::Ordering::Relaxed), ++ start.elapsed() ++ ); ++ } ++ } + } +diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rs +index 693253a..ab27ea7 100644 +--- a/crates/disktree-core/src/scan.rs ++++ b/crates/disktree-core/src/scan.rs +@@ -283,6 +283,9 @@ pub fn scan(root: &Path, options: ScanOptions) -> io::Result { + Ok(node) + } + ++#[cfg(test)] ++static AUDIT_CLASSIFIED: AtomicUsize = AtomicUsize::new(0); ++ + struct WalkContext { + /// A subtree to reuse rather than walk. + known: Option, +@@ -328,6 +331,8 @@ impl WalkContext { + /// files, and a disk has millions of them. The name is moved out of + /// the entry last, so nothing copies it. + fn classify(&self, dir: &Path, entry: &mut impl Listed) -> Classified { ++ #[cfg(test)] ++ AUDIT_CLASSIFIED.fetch_add(1, Ordering::Relaxed); + let listing = match entry.listing() { + Ok(listing) => listing, + Err(error) => { +@@ -1855,4 +1860,49 @@ mod tests { + ); + } + } ++ #[test] ++ #[ignore = "manual normal-scan complexity audit on real temporary fixtures"] ++ fn additional_scan_complexity_audit() { ++ for nested in [false, true] { ++ for count in [128, 1024] { ++ let temp = tempfile::tempdir().expect("tempdir"); ++ for index in 0..count { ++ let name = format!("entry-{index:05}"); ++ let file = if nested { ++ let folder = temp.path().join(name); ++ fs::create_dir(&folder).expect("mkdir"); ++ folder.join("payload") ++ } else { ++ temp.path().join(name) ++ }; ++ fs::write(file, [b'x'; 64]).expect("write"); ++ } ++ AUDIT_CLASSIFIED.store(0, Ordering::Relaxed); ++ crate::tree::AUDIT_AGGREGATE.store(0, Ordering::Relaxed); ++ let start = std::time::Instant::now(); ++ let result = scan( ++ temp.path(), ++ ScanOptions { ++ apparent_size: true, ++ threads: crate::scan_threads::ScanThreads { ++ max_threads: 1, ++ ..crate::scan_threads::ScanThreads::default() ++ }, ++ ..ScanOptions::default() ++ }, ++ ) ++ .expect("scan"); ++ assert_eq!(result.files, count as u64); ++ let entries = AUDIT_CLASSIFIED.load(Ordering::Relaxed); ++ let aggregated = ++ crate::tree::AUDIT_AGGREGATE.load(Ordering::Relaxed); ++ assert_eq!(entries, count * if nested { 2 } else { 1 }); ++ assert_eq!(aggregated, entries + 1); ++ eprintln!( ++ "audit real-scan nested={nested} F={count} listed={entries} aggregate-visits={aggregated} wall={:?}", ++ start.elapsed() ++ ); ++ } ++ } ++ } + } +diff --git a/crates/disktree-core/src/space.rs b/crates/disktree-core/src/space.rs +index 8a2bc4e..5e2f8af 100644 +--- a/crates/disktree-core/src/space.rs ++++ b/crates/disktree-core/src/space.rs +@@ -7,6 +7,11 @@ + use std::io; + use std::path::{Path, PathBuf}; + ++#[cfg(test)] ++thread_local! { ++ static AUDIT_MOUNT_PAIRS: std::cell::Cell = const { std::cell::Cell::new(0) }; ++} ++ + /// A volume's capacity in bytes. + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub struct SpaceInfo { +@@ -515,6 +520,8 @@ fn repeated_mounts_excluding( + let mut repeated: Vec = Vec::new(); + for mount in visible { + let elsewhere = kept.iter().find_map(|wider| { ++ #[cfg(test)] ++ AUDIT_MOUNT_PAIRS.set(AUDIT_MOUNT_PAIRS.get() + 1); + if !same_filesystem(wider, mount) { + return None; + } +@@ -1327,4 +1334,28 @@ map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse) + let space = SpaceInfo::default(); + assert!((space.used_fraction() - 0.0).abs() < f32::EPSILON); + } ++ #[test] ++ #[ignore = "manual complexity audit on synthetic mount metadata"] ++ fn additional_mount_complexity_audit() { ++ for count in [128, 1024] { ++ let mounts: Vec<_> = (0..count) ++ .map(|index| MountView { ++ device: format!("8:{index}"), ++ source: format!("/dev/audit-{index}"), ++ fstype: "ext4".into(), ++ point: PathBuf::from(format!("/audit/mount-{index:05}")), ++ fs_root: PathBuf::from("/"), ++ }) ++ .collect(); ++ AUDIT_MOUNT_PAIRS.set(0); ++ let start = std::time::Instant::now(); ++ let duplicates = repeated_mounts(&mounts, Path::new("/")); ++ assert!(duplicates.is_empty()); ++ eprintln!( ++ "audit mount-startup K={count} pairs={} wall={:?}", ++ AUDIT_MOUNT_PAIRS.get(), ++ start.elapsed() ++ ); ++ } ++ } + } +diff --git a/crates/disktree-core/src/tree.rs b/crates/disktree-core/src/tree.rs +index 6991a86..8724560 100644 +--- a/crates/disktree-core/src/tree.rs ++++ b/crates/disktree-core/src/tree.rs +@@ -10,6 +10,10 @@ use rustc_hash::FxHashSet; + + use crate::classify::{Category, Reclaim}; + ++#[cfg(test)] ++pub(crate) static AUDIT_AGGREGATE: std::sync::atomic::AtomicUsize = ++ std::sync::atomic::AtomicUsize::new(0); ++ + /// What a node represents on disk. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum NodeKind { +@@ -352,6 +356,8 @@ fn aggregate_at( + depth: usize, + seen: Option<&Seen>, + ) { ++ #[cfg(test)] ++ AUDIT_AGGREGATE.fetch_add(1, Ordering::Relaxed); + if !node.is_dir() { + // Every name of a file has the file's size, so one that weighs + // nothing need not be remembered to be charged once. +diff --git a/crates/disktree-core/src/treemap.rs b/crates/disktree-core/src/treemap.rs +index 598b559..ce2fc50 100644 +--- a/crates/disktree-core/src/treemap.rs ++++ b/crates/disktree-core/src/treemap.rs +@@ -13,6 +13,11 @@ + use crate::filter::{Keep, Matches}; + use crate::tree::{Metric, Node}; + ++#[cfg(test)] ++thread_local! { ++ static AUDIT_RATIO: std::cell::Cell = const { std::cell::Cell::new(0) }; ++} ++ + /// An axis-aligned rectangle in viewport pixels. + #[derive(Clone, Copy, Debug, Default, PartialEq)] + pub struct Rect { +@@ -401,6 +406,8 @@ fn worst_ratio(areas: &[f64], row_sum: f64, side: f64) -> f64 { + } + let thickness = row_sum / side; + areas.iter().fold(0.0_f64, |worst, area| { ++ #[cfg(test)] ++ AUDIT_RATIO.set(AUDIT_RATIO.get() + 1); + if *area <= 0.0 || thickness <= 0.0 { + return worst; + } +@@ -722,4 +729,43 @@ mod tests { + let ratio = area_of("src") / area_of("unit_test.txt"); + assert!((ratio - 3.0).abs() < 0.05, "ratio {ratio}"); + } ++ #[test] ++ #[ignore = "manual row-complexity audit on synthetic layout"] ++ fn additional_layout_complexity_audit() { ++ for count in [128, 1024] { ++ let values = vec![1.; count]; ++ AUDIT_RATIO.set(0); ++ let start = std::time::Instant::now(); ++ let rects = squarify(&values, Rect::new(0., 0., 1024., 1024.)); ++ assert_eq!(rects.len(), count); ++ eprintln!( ++ "audit uncapped-squarify F={count} ratio-visits={} wall={:?}", ++ AUDIT_RATIO.get(), ++ start.elapsed() ++ ); ++ let mut root = Node::directory("root"); ++ for index in 0..count { ++ root.children.push(Node::entry( ++ format!("file-{index}"), ++ crate::tree::NodeKind::File, ++ 1, ++ )); ++ } ++ AUDIT_RATIO.set(0); ++ let start = std::time::Instant::now(); ++ let tiles = layout( ++ &root, ++ &[], ++ Rect::new(0., 0., 1024., 1024.), ++ Metric::Bytes, ++ &LayoutOptions::default(), ++ ); ++ assert!(!tiles.is_empty()); ++ eprintln!( ++ "audit capped-layout F={count} ratio-visits={} wall={:?}", ++ AUDIT_RATIO.get(), ++ start.elapsed() ++ ); ++ } ++ } + } From 55665087d9aa26a39598cf2796db10397268d83b Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 15:55:06 -0400 Subject: [PATCH 10/19] test: expose quadratic queue mutations and review status lookups --- crates/disktree-app/src/marks.rs | 18 +- crates/disktree-app/src/state.rs | 10 +- crates/disktree-app/src/tests.rs | 203 ++++++++++++++++++ crates/disktree-app/src/views.rs | 26 ++- .../mark-resolution/expanded/red.log | 61 ++++++ 5 files changed, 313 insertions(+), 5 deletions(-) create mode 100644 docs/performance/mark-resolution/expanded/red.log diff --git a/crates/disktree-app/src/marks.rs b/crates/disktree-app/src/marks.rs index 12fc5f01..346009b6 100644 --- a/crates/disktree-app/src/marks.rs +++ b/crates/disktree-app/src/marks.rs @@ -22,6 +22,12 @@ pub mod work { pub static PLANS: Cell = const { Cell::new(0) }; pub static REFRESHES: Cell = const { Cell::new(0) }; pub static TILES: Cell = const { Cell::new(0) }; + pub static MARKING: Cell = const { Cell::new(0) }; + pub static COMPACTION: Cell = const { Cell::new(0) }; + pub static REVIEW: Cell = const { Cell::new(0) }; + pub static REVIEW_ROWS: Cell = const { Cell::new(0) }; + pub static COVERED_ROWS: Cell = const { Cell::new(0) }; + pub static BLOCKED_ROWS: Cell = const { Cell::new(0) }; } pub fn step() { @@ -35,6 +41,12 @@ pub mod work { PLANS.set(0); REFRESHES.set(0); TILES.set(0); + MARKING.set(0); + COMPACTION.set(0); + REVIEW.set(0); + REVIEW_ROWS.set(0); + COVERED_ROWS.set(0); + BLOCKED_ROWS.set(0); } } @@ -91,7 +103,11 @@ impl Marks { pub fn remove(&mut self, path: &Path) { if self.index.remove(path) { self.resolved = None; - self.items.retain(|item| item.path != path); + self.items.retain(|item| { + #[cfg(test)] + work::COMPACTION.set(work::COMPACTION.get() + 1); + item.path != path + }); } } diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs index 4b750e6d..67531444 100644 --- a/crates/disktree-app/src/state.rs +++ b/crates/disktree-app/src/state.rs @@ -1722,6 +1722,9 @@ impl Disktree { .items() .iter() .filter(|item| { + #[cfg(test)] + crate::marks::work::MARKING + .set(crate::marks::work::MARKING.get() + 1); item.path != path && item.path.starts_with(&path) }) .map(|item| item.path.clone()) @@ -1749,7 +1752,12 @@ impl Disktree { self.marks .items() .iter() - .filter(|item| item.path != path && path.starts_with(&item.path)) + .filter(|item| { + #[cfg(test)] + crate::marks::work::MARKING + .set(crate::marks::work::MARKING.get() + 1); + item.path != path && path.starts_with(&item.path) + }) .map(|item| item.path.clone()) .min_by_key(|ancestor| ancestor.as_os_str().len()) } diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index a1236506..5350f494 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -2223,3 +2223,206 @@ fn marked_interaction_timings(cx: &mut TestAppContext) { window.update(|window, _| window.remove_window()); } } + +/// The disk root is private and empty; only the displayed tree is synthetic. +fn queue_budget_view( + count: usize, + cx: &mut TestAppContext, +) -> (tempfile::TempDir, Entity, &mut Window) { + use disktree_core::tree::{Metric, Node, NodeKind, aggregate}; + + let temp = tempfile::tempdir().expect("tempdir"); + let root_path = temp.path().to_path_buf(); + let mut tree = Node::directory("root"); + let mut parent = Node::directory("container"); + for index in 0..count { + parent.children.push(Node::entry( + format!("file-{index:05}"), + NodeKind::File, + 1, + )); + } + tree.children.push(parent); + tree.children.push(Node::entry("keep", NodeKind::File, 1)); + aggregate(&mut tree, Metric::Bytes); + let (view, window) = cx.add_window_view(move |_, cx| { + Disktree::with_tree(root_path.clone(), tree, options(), 3, cx) + }); + (temp, view, window) +} + +#[gpui_kit::test] +fn queue_construction_budget(cx: &mut TestAppContext) { + use crate::marks::work; + + cx.update(gpui_omarchy::init); + let mut measure = |count| { + let (_temp, view, window) = queue_budget_view(count, cx); + let operations = update(&view, window, |app, cx| { + work::reset(); + let start = std::time::Instant::now(); + for index in 0..count { + app.toggle_mark(&[0, index], cx); + } + assert_eq!(app.marks.len(), count); + assert_eq!( + work::FRAMES.get(), + 0, + "measure insertion before rendering" + ); + let operations = work::MARKING.get() + work::COMPACTION.get(); + assert!(operations > 0); + eprintln!( + "queue-construction M={count} work={operations} wall={:?}", + start.elapsed() + ); + operations + }); + window.update(|window, _| window.remove_window()); + operations + }; + let small = measure(128); + let large = measure(1024); + assert!(large <= small * 16, "8x marks: {small} -> {large}"); + assert!(large <= 1024 * 32, "absolute insertion budget: {large}"); +} + +#[gpui_kit::test] +fn parent_absorption_budget(cx: &mut TestAppContext) { + use crate::marks::work; + + cx.update(gpui_omarchy::init); + let mut measure = |count| { + let (temp, view, window) = queue_budget_view(count, cx); + let operations = update(&view, window, |app, cx| { + let keep = app.target_at(&[1]).expect("neighbour"); + app.marks.toggle(keep.clone()); + for index in 0..count { + let target = app.target_at(&[0, index]).expect("file"); + app.marks.toggle(target); + } + work::reset(); + let start = std::time::Instant::now(); + app.toggle_mark(&[0], cx); + assert_eq!(app.marks.len(), 2); + assert_eq!(app.marks.items()[0], keep, "neighbour stays in order"); + assert_eq!( + app.marks.items()[1].path, + temp.path().join("container") + ); + assert_eq!(app.marks.items()[1].bytes, count as u64); + let operations = work::MARKING.get() + work::COMPACTION.get(); + assert!(operations > 0); + eprintln!( + "parent-absorption M={count} work={operations} compactions={} wall={:?}", + work::COMPACTION.get(), + start.elapsed() + ); + operations + }); + window.update(|window, _| window.remove_window()); + operations + }; + let small = measure(128); + let large = measure(1024); + assert!(large <= small * 16, "8x descendants: {small} -> {large}"); + assert!(large <= 1024 * 32, "absolute absorption budget: {large}"); +} + +fn review_status_work( + count: usize, + covered: bool, + cx: &mut TestAppContext, +) -> (usize, usize) { + use crate::marks::work; + use disktree_core::removal::Target; + + let (temp, view, window) = queue_budget_view(count, cx); + update(&view, window, |app, _| { + if covered { + let parent = app.target_at(&[0]).expect("parent"); + app.marks.toggle(parent); + } + for index in 0..count { + let target = if covered { + app.target_at(&[0, index]).expect("file") + } else { + Target { + path: temp + .path() + .with_extension("outside") + .join(format!("file-{index:05}")), + bytes: 1, + is_dir: false, + hidden: false, + } + }; + app.marks.toggle(target); + } + app.screen = Screen::Review; + }); + work::reset(); + let start = std::time::Instant::now(); + for _ in 0..4 { + update(&view, window, |_, cx| cx.notify()); + draw(window); + } + let elapsed = start.elapsed(); + let frames = work::RENDERS.get(); + let operations = work::REVIEW.get(); + assert!( + frames >= 4 && work::PLANS.get() >= 4, + "must render and plan Review" + ); + let rows = (count + usize::from(covered)).min(1200); + assert_eq!( + work::REVIEW_ROWS.get(), + rows * frames, + "retain the display cap" + ); + assert_eq!( + work::BLOCKED_ROWS.get(), + if covered { 0 } else { rows * frames } + ); + assert_eq!( + work::COVERED_ROWS.get(), + if covered { (rows - 1) * frames } else { 0 } + ); + assert!(operations > 0); + eprintln!( + "review-status covered={covered} M={count} work={operations} renders={frames} rows={rows} per-render={} wall={elapsed:?}", + operations / frames + ); + window.update(|window, _| window.remove_window()); + (operations, frames) +} + +fn review_status_budget(cx: &mut TestAppContext, covered: bool) { + let (small, small_frames) = review_status_work(128, covered, cx); + let (large, large_frames) = review_status_work(1024, covered, cx); + assert!( + large * small_frames <= small * large_frames * 16, + "8x status rows: {small}/{small_frames} -> {large}/{large_frames}" + ); + assert!( + large <= 1024 * large_frames * 8, + "absolute row-status budget: {large}" + ); + let (capped, capped_frames) = review_status_work(8192, covered, cx); + assert!( + capped <= 8192 * capped_frames * 8, + "status budget above row cap: {capped}" + ); +} + +#[gpui_kit::test] +fn blocked_review_status_budget(cx: &mut TestAppContext) { + cx.update(gpui_omarchy::init); + review_status_budget(cx, false); +} + +#[gpui_kit::test] +fn covered_review_status_budget(cx: &mut TestAppContext) { + cx.update(gpui_omarchy::init); + review_status_budget(cx, true); +} diff --git a/crates/disktree-app/src/views.rs b/crates/disktree-app/src/views.rs index 2041d91e..a1b1aae3 100644 --- a/crates/disktree-app/src/views.rs +++ b/crates/disktree-app/src/views.rs @@ -2574,12 +2574,32 @@ fn review( } for (index, item) in items.iter().take(LIST_LIMIT).enumerate() { - let is_covered = - covered.iter().any(|covered| covered.path == item.path); + let is_covered = covered.iter().any(|covered| { + #[cfg(test)] + crate::marks::work::REVIEW + .set(crate::marks::work::REVIEW.get() + 1); + covered.path == item.path + }); let blocked_reason = blocked .iter() - .find(|blocked| blocked.path == item.path) + .find(|blocked| { + #[cfg(test)] + crate::marks::work::REVIEW + .set(crate::marks::work::REVIEW.get() + 1); + blocked.path == item.path + }) .map(|blocked| blocked.reason.clone()); + #[cfg(test)] + { + use crate::marks::work; + work::REVIEW_ROWS.set(work::REVIEW_ROWS.get() + 1); + work::COVERED_ROWS + .set(work::COVERED_ROWS.get() + usize::from(is_covered)); + work::BLOCKED_ROWS.set( + work::BLOCKED_ROWS.get() + + usize::from(blocked_reason.is_some()), + ); + } list = list.child(mark_row( index, item, diff --git a/docs/performance/mark-resolution/expanded/red.log b/docs/performance/mark-resolution/expanded/red.log new file mode 100644 index 00000000..81a829b1 --- /dev/null +++ b/docs/performance/mark-resolution/expanded/red.log @@ -0,0 +1,61 @@ + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Finished `test` profile [optimized + debuginfo] target(s) in 7.61s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 9 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=62.75µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=45µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=58.083µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=28.875µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=62.834µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=205.458µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.533583ms +ok +test tests::blocked_review_status_budget ... review-status covered=false M=128 work=66048 renders=8 rows=128 per-render=8256 wall=113.90025ms +review-status covered=false M=1024 work=4198400 renders=8 rows=1024 per-render=524800 wall=1.038960833s + +thread 'tests::blocked_review_status_budget' (404491031) panicked at crates/disktree-app/src/tests.rs:2298:5: +8x status rows: 66048/8 -> 4198400/8 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED +test tests::covered_review_status_budget ... review-status covered=true M=128 work=67072 renders=8 rows=129 per-render=8384 wall=135.896292ms +review-status covered=true M=1024 work=4206592 renders=8 rows=1025 per-render=525824 wall=1.29864975s + +thread 'tests::covered_review_status_budget' (404491108) panicked at crates/disktree-app/src/tests.rs:2298:5: +8x status rows: 67072/8 -> 4206592/8 +FAILED +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=63.308583ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=64.349292ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=342.802833ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=360.040792ms +ok +test tests::parent_absorption_budget ... parent-absorption M=128 work=8771 compactions=8512 wall=347.334µs +parent-absorption M=1024 work=528899 compactions=526848 wall=15.802708ms + +thread 'tests::parent_absorption_budget' (404491437) panicked at crates/disktree-app/src/tests.rs:2223:5: +8x descendants: 8771 -> 528899 +FAILED +test tests::queue_construction_budget ... queue-construction M=128 work=16384 wall=2.851667ms +queue-construction M=1024 work=1048576 wall=183.52175ms + +thread 'tests::queue_construction_budget' (404491448) panicked at crates/disktree-app/src/tests.rs:2181:5: +8x marks: 16384 -> 1048576 +FAILED + +failures: + +failures: + tests::blocked_review_status_budget + tests::covered_review_status_budget + tests::parent_absorption_budget + tests::queue_construction_budget + +test result: FAILED. 5 passed; 4 failed; 0 ignored; 0 measured; 74 filtered out; finished in 4.00s + +error: test failed, to rerun pass `-p disktree-app --bin disktree` From 2a7be0bc3d8998b7b05708567238bae513fcd87a Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 15:56:47 -0400 Subject: [PATCH 11/19] fix: index marked paths and absorb descendants in one pass --- crates/disktree-app/src/marks.rs | 159 +++++++++++++++++- crates/disktree-app/src/state.rs | 34 +--- .../expanded/green-marking.log | 54 ++++++ 3 files changed, 217 insertions(+), 30 deletions(-) create mode 100644 docs/performance/mark-resolution/expanded/green-marking.log diff --git a/crates/disktree-app/src/marks.rs b/crates/disktree-app/src/marks.rs index 346009b6..bc7ed46a 100644 --- a/crates/disktree-app/src/marks.rs +++ b/crates/disktree-app/src/marks.rs @@ -4,6 +4,8 @@ //! re-scanned after a removal, and a mark must survive that (or be reported as //! gone) rather than silently pointing at a different node. +use std::collections::BTreeSet; +use std::ops::Bound::{Excluded, Unbounded}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Weak}; @@ -54,7 +56,10 @@ pub mod work { #[derive(Debug, Default)] pub struct Marks { items: Vec, - index: FxHashSet, + // Path's component ordering makes descendants contiguous. The same + // index answers exact membership and whether a parent has any marks + // inside, without storing ancestor counts or scanning every mark. + index: BTreeSet, resolved: Option, } @@ -73,6 +78,54 @@ impl Marks { self.index.contains(path) } + /// Outermost marked ancestor, excluding the path itself. Keep the + /// spelling stored with the mark, using the same Path semantics as + /// exact membership and the old `starts_with` scan. + pub fn ancestor(&self, path: &Path) -> Option<&Path> { + path.ancestors() + .skip(1) + .filter_map(|ancestor| { + #[cfg(test)] + work::MARKING.set(work::MARKING.get() + 1); + self.index.get(ancestor) + }) + .last() + .map(PathBuf::as_path) + } + + /// Drop strict descendants in one stable compaction, retaining the + /// parent's mark and unrelated marks in their existing order. + pub fn absorb(&mut self, path: &Path) -> usize { + #[cfg(test)] + work::MARKING.set(work::MARKING.get() + 1); + let has_descendant = self + .index + .range::((Excluded(path), Unbounded)) + .next() + .is_some_and(|next| next.starts_with(path)); + // Most insertions are disjoint; proving that from the index keeps + // queuing M paths from doing M full scans of the ordered item list. + if !has_descendant { + return 0; + } + self.resolved = None; + let before = self.items.len(); + let index = &mut self.index; + self.items.retain(|item| { + #[cfg(test)] + work::COMPACTION.set(work::COMPACTION.get() + 1); + if item.path != path && item.path.starts_with(path) { + #[cfg(test)] + work::MARKING.set(work::MARKING.get() + 1); + index.remove(&item.path); + false + } else { + true + } + }); + before - self.items.len() + } + pub fn items(&self) -> &[Target] { &self.items } @@ -425,6 +478,110 @@ mod tests { ); } + #[test] + fn ordered_marks_preserve_components_case_and_outermost_ancestor() { + let mut marks = Marks::default(); + for path in [ + "/synthetic/a/child", + "/synthetic/a", + "/synthetic/a-b", + "/synthetic/a.b", + "/synthetic/ab", + "/synthetic/A", + ] { + marks.toggle(target(path, 1)); + } + assert_eq!( + marks.ancestor(Path::new("/synthetic/a/child/file")), + Some(Path::new("/synthetic/a")) + ); + assert_eq!(marks.ancestor(Path::new("/synthetic/a")), None); + assert_eq!( + marks.ancestor(Path::new("/synthetic/a-b/file")), + Some(Path::new("/synthetic/a-b")) + ); + assert_eq!( + marks.ancestor(Path::new("/synthetic/A/file")), + Some(Path::new("/synthetic/A")) + ); + assert_eq!(marks.absorb(Path::new("/synthetic/a")), 1); + let expected: Vec<_> = [ + "/synthetic/a", + "/synthetic/a-b", + "/synthetic/a.b", + "/synthetic/ab", + "/synthetic/A", + ] + .map(PathBuf::from) + .into(); + assert_eq!( + marks + .items() + .iter() + .map(|item| item.path.clone()) + .collect::>(), + expected + ); + assert!(marks.items().iter().all(|item| marks.contains(&item.path))); + assert!(!marks.contains(Path::new("/synthetic/a/child"))); + marks.remove(Path::new("/synthetic/a")); + assert_eq!(marks.ancestor(Path::new("/synthetic/a/file")), None); + assert_eq!( + marks.absorb(Path::new("/synthetic/a")), + 0, + "prefix lookalikes stay marked" + ); + marks.clear(); + assert_eq!(marks.absorb(Path::new("/synthetic")), 0); + } + + #[test] + fn absorption_invalidates_cached_crumbs_and_keeps_missing_paths_indexed() { + let root_path = Path::new("/home/tobi"); + let root = Arc::new(tree()); + let mut marks = Marks::default(); + marks.toggle(target("/home/tobi/.cache/blob.bin", 0)); + marks.toggle(target("/home/tobi/.cache", 0)); + marks.toggle(target("/home/tobi/gone/nested", 100)); + let before = marks.crumbs(root_path, &root, Metric::Bytes); + assert_eq!(before.len(), 2); + assert_eq!(marks.absorb(Path::new("/home/tobi/.cache")), 1); + let after = marks.crumbs(root_path, &root, Metric::Bytes); + assert!(!Arc::ptr_eq(&before, &after)); + assert_eq!(after.len(), 1); + assert!(after.contains(&vec![0])); + marks.refresh(root_path, &root, Metric::Files); + assert!(marks.contains(Path::new("/home/tobi/gone/nested"))); + assert_eq!(marks.absorb(Path::new("/home/tobi/gone")), 1); + assert!(!marks.contains(Path::new("/home/tobi/gone/nested"))); + } + + #[cfg(unix)] + #[test] + fn ordered_marks_keep_exact_non_utf8_paths() { + use std::os::unix::ffi::OsStrExt as _; + let root = Path::new("/synthetic"); + let parent = root.join(std::ffi::OsStr::from_bytes(b"not-\xff")); + let child = parent.join("file"); + let mut marks = Marks::default(); + marks.toggle(Target { + path: child.clone(), + bytes: 1, + is_dir: false, + hidden: false, + }); + marks.toggle(Target { + path: parent.clone(), + bytes: 1, + is_dir: true, + hidden: false, + }); + assert_eq!(marks.ancestor(&child), Some(parent.as_path())); + assert_eq!(marks.absorb(&parent), 1); + assert!(marks.contains(&parent)); + assert!(!marks.contains(&child)); + } + #[test] fn toggling_marks_and_unmarks() { let mut marks = Marks::default(); diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs index 67531444..142e84d5 100644 --- a/crates/disktree-app/src/state.rs +++ b/crates/disktree-app/src/state.rs @@ -1717,28 +1717,14 @@ impl Disktree { let path = target.path.clone(); if self.marks.toggle(target) { self.space_baseline = self.space_baseline.or(self.space); - let inside: Vec = self - .marks - .items() - .iter() - .filter(|item| { - #[cfg(test)] - crate::marks::work::MARKING - .set(crate::marks::work::MARKING.get() + 1); - item.path != path && item.path.starts_with(&path) - }) - .map(|item| item.path.clone()) - .collect(); - for inner in &inside { - self.marks.remove(inner); - } - if !inside.is_empty() { + let inside = self.marks.absorb(&path); + if inside > 0 { self.notice = Some(( format!( "{} now covers {} mark{} inside it", display_path(&path, self.home.as_deref()), - inside.len(), - if inside.len() == 1 { "" } else { "s" } + inside, + if inside == 1 { "" } else { "s" } ), Status::Neutral, )); @@ -1749,17 +1735,7 @@ impl Disktree { /// The marked directory `path` is inside, if any; never `path` itself. pub fn marked_ancestor(&self, path: &Path) -> Option { - self.marks - .items() - .iter() - .filter(|item| { - #[cfg(test)] - crate::marks::work::MARKING - .set(crate::marks::work::MARKING.get() + 1); - item.path != path && path.starts_with(&item.path) - }) - .map(|item| item.path.clone()) - .min_by_key(|ancestor| ancestor.as_os_str().len()) + self.marks.ancestor(path).map(Path::to_path_buf) } pub fn target_at(&self, crumbs: &[usize]) -> Option { diff --git a/docs/performance/mark-resolution/expanded/green-marking.log b/docs/performance/mark-resolution/expanded/green-marking.log new file mode 100644 index 00000000..58d28114 --- /dev/null +++ b/docs/performance/mark-resolution/expanded/green-marking.log @@ -0,0 +1,54 @@ + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 7.25s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 1 test +test tests::queue_construction_budget ... queue-construction M=128 work=1152 wall=572.625µs +queue-construction M=1024 work=9216 wall=6.670083ms +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 85 filtered out; finished in 0.13s + + Finished `test` profile [optimized + debuginfo] target(s) in 0.26s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 1 test +test tests::parent_absorption_budget ... parent-absorption M=128 work=266 compactions=130 wall=58.625µs +parent-absorption M=1024 work=2058 compactions=1026 wall=459.125µs +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 85 filtered out; finished in 0.07s + + Finished `test` profile [optimized + debuginfo] target(s) in 0.27s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 13 tests +test marks::tests::display_path_shortens_the_home_prefix ... ok +test marks::tests::ordered_marks_keep_exact_non_utf8_paths ... ok +test marks::tests::ordered_marks_preserve_components_case_and_outermost_ancestor ... ok +test marks::tests::marking_the_same_path_twice_keeps_one_entry ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=23.041µs +ok +test marks::tests::resolver_preserves_lossy_non_utf8_components ... ok +test marks::tests::toggling_marks_and_unmarks ... ok +test marks::tests::find_matches_whole_components_only ... ok +test marks::tests::resolver_preserves_scan_name_semantics ... ok +test marks::tests::refresh_re_reads_sizes_from_a_new_tree ... ok +test marks::tests::absorption_invalidates_cached_crumbs_and_keeps_missing_paths_indexed ... ok +test marks::tests::cached_crumbs_follow_every_invalidation_boundary ... ok +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=50.417µs +refresh M=128 N=256 R=4 work=2560 wall=232.083µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=58.625µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=27.083µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=60.416µs +test marks::tests::refresh_dimension_budget ... ok +refresh M=1024 N=2048 R=4 work=20480 wall=1.545542ms +test marks::tests::refresh_resolution_budget ... ok + +test result: ok. 13 passed; 0 failed; 0 ignored; 0 measured; 73 filtered out; finished in 0.01s + From 94a2c4027045321adc0ee2b765ecedd581f4fa4d Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 15:59:31 -0400 Subject: [PATCH 12/19] fix: index review row status from each fresh removal plan --- crates/disktree-app/src/views.rs | 134 +++++++++++++++--- .../mark-resolution/expanded/green-review.log | 29 ++++ 2 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 docs/performance/mark-resolution/expanded/green-review.log diff --git a/crates/disktree-app/src/views.rs b/crates/disktree-app/src/views.rs index a1b1aae3..cd5cd70f 100644 --- a/crates/disktree-app/src/views.rs +++ b/crates/disktree-app/src/views.rs @@ -4,6 +4,8 @@ //! always exactly what the state says — there is no second copy of anything to //! keep in sync. +use std::path::Path; + use disktree_core::classify::Category; use disktree_core::insights::{Candidate, Finding, STALE_DAYS}; use disktree_core::removal::{Plan, RemovalMode, Target}; @@ -24,6 +26,7 @@ use gpui_omarchy::{ }; use gpui_kit::prelude::FluentBuilder as _; +use rustc_hash::{FxHashMap, FxHashSet}; use crate::palette; use crate::state::{ @@ -37,6 +40,54 @@ use crate::widgets; /// removed; the list only stops being exhaustive, which it says out loud. const LIST_LIMIT: usize = 1200; +/// Row decoration borrows this render's freshly evaluated plan. It must +/// not cache removal authorization, and each displayed row needs constant +/// expected hash lookups rather than scans of all blocked/covered targets. +#[derive(Debug)] +struct ReviewStatus<'a> { + covered: FxHashSet<&'a Path>, + blocked: FxHashMap<&'a Path, &'a str>, +} + +impl<'a> ReviewStatus<'a> { + fn new(plan: &'a Plan) -> Self { + let mut status = Self { + covered: FxHashSet::default(), + blocked: FxHashMap::default(), + }; + for target in &plan.covered { + #[cfg(test)] + crate::marks::work::REVIEW + .set(crate::marks::work::REVIEW.get() + 1); + status.covered.insert(target.path.as_path()); + } + for target in &plan.blocked { + #[cfg(test)] + crate::marks::work::REVIEW + .set(crate::marks::work::REVIEW.get() + 1); + // A raw queue may repeat a path. Keep the first explanation, + // matching the original linear find's behavior. + status + .blocked + .entry(target.path.as_path()) + .or_insert(target.reason.as_str()); + } + status + } + + fn covered(&self, path: &Path) -> bool { + #[cfg(test)] + crate::marks::work::REVIEW.set(crate::marks::work::REVIEW.get() + 1); + self.covered.contains(path) + } + + fn blocked(&self, path: &Path) -> Option<&str> { + #[cfg(test)] + crate::marks::work::REVIEW.set(crate::marks::work::REVIEW.get() + 1); + self.blocked.get(path).copied() + } +} + /// The whole window. pub fn root( app: &mut Disktree, @@ -2549,9 +2600,8 @@ fn review( ) -> Div { let theme = cx.omarchy().clone(); let plan = app.plan(); - let items: Vec = app.marks.items().to_vec(); - let covered: Vec = plan.covered.clone(); - let blocked = plan.blocked.clone(); + let items = app.marks.items(); + let status = ReviewStatus::new(&plan); let mut list = div() .id("marked-list") @@ -2574,21 +2624,8 @@ fn review( } for (index, item) in items.iter().take(LIST_LIMIT).enumerate() { - let is_covered = covered.iter().any(|covered| { - #[cfg(test)] - crate::marks::work::REVIEW - .set(crate::marks::work::REVIEW.get() + 1); - covered.path == item.path - }); - let blocked_reason = blocked - .iter() - .find(|blocked| { - #[cfg(test)] - crate::marks::work::REVIEW - .set(crate::marks::work::REVIEW.get() + 1); - blocked.path == item.path - }) - .map(|blocked| blocked.reason.clone()); + let is_covered = status.covered(&item.path); + let blocked_reason = status.blocked(&item.path).map(str::to_owned); #[cfg(test)] { use crate::marks::work; @@ -3699,3 +3736,64 @@ fn savings(lower: u64, upper: u64) -> String { format!("{}–{}", human_bytes(lower), human_bytes(upper)) } } + +#[cfg(test)] +mod tests { + use super::*; + use disktree_core::removal::Blocked; + use std::path::PathBuf; + + #[test] + fn review_status_preserves_exact_paths_and_first_blocked_reason() { + let path = PathBuf::from("/synthetic/a/child"); + let mut plan = Plan::default(); + plan.covered.push(Target { + path: path.clone(), + bytes: 1, + is_dir: false, + hidden: false, + }); + plan.blocked.push(Blocked { + path: path.clone(), + reason: "first".into(), + }); + plan.blocked.push(Blocked { + path: path.clone(), + reason: "second".into(), + }); + let status = ReviewStatus::new(&plan); + assert!(status.covered(&path)); + assert!(!status.covered(Path::new("/synthetic/a/childish"))); + assert!(!status.covered(Path::new("/synthetic/A/child"))); + assert_eq!(status.blocked(&path), Some("first")); + assert_eq!( + status.blocked(Path::new("/synthetic/a/./child")), + Some("first") + ); + assert_eq!(status.blocked(Path::new("/synthetic/A/child")), None); + plan.blocked.clear(); + plan.covered.clear(); + let refreshed = ReviewStatus::new(&plan); + assert_eq!(refreshed.blocked(&path), None); + assert!(!refreshed.covered(&path)); + } + + #[cfg(unix)] + #[test] + fn review_status_keeps_non_utf8_paths_distinct() { + use std::os::unix::ffi::OsStrExt as _; + let path = Path::new("/synthetic") + .join(std::ffi::OsStr::from_bytes(b"not-\xff")); + let mut plan = Plan::default(); + plan.blocked.push(Blocked { + path: path.clone(), + reason: "blocked".into(), + }); + let status = ReviewStatus::new(&plan); + assert_eq!(status.blocked(&path), Some("blocked")); + assert_eq!( + status.blocked(Path::new(&path.to_string_lossy().into_owned())), + None + ); + } +} diff --git a/docs/performance/mark-resolution/expanded/green-review.log b/docs/performance/mark-resolution/expanded/green-review.log new file mode 100644 index 00000000..b9d59fa2 --- /dev/null +++ b/docs/performance/mark-resolution/expanded/green-review.log @@ -0,0 +1,29 @@ + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 4.06s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 2 tests +test tests::blocked_review_status_budget ... review-status covered=false M=128 work=3072 renders=8 rows=128 per-render=384 wall=117.141208ms +review-status covered=false M=1024 work=24576 renders=8 rows=1024 per-render=3072 wall=918.023375ms +review-status covered=false M=8192 work=84736 renders=8 rows=1200 per-render=10592 wall=1.251237917s +ok +test tests::covered_review_status_budget ... review-status covered=true M=128 work=3088 renders=8 rows=129 per-render=386 wall=124.676334ms +review-status covered=true M=1024 work=24592 renders=8 rows=1025 per-render=3074 wall=1.058290834s +review-status covered=true M=8192 work=84736 renders=8 rows=1200 per-render=10592 wall=2.1404995s +ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 86 filtered out; finished in 5.87s + + Finished `test` profile [optimized + debuginfo] target(s) in 0.26s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 2 tests +test views::tests::review_status_preserves_exact_paths_and_first_blocked_reason ... ok +test views::tests::review_status_keeps_non_utf8_paths_distinct ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 86 filtered out; finished in 0.00s + From 8740401d295f16498bd93e277e4b79d888c8079d Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 16:04:14 -0400 Subject: [PATCH 13/19] test: cover directory insertion in the queue construction budget --- crates/disktree-app/src/tests.rs | 47 +++++++++++++++++++ .../mark-resolution/expanded/red-folders.log | 23 +++++++++ 2 files changed, 70 insertions(+) create mode 100644 docs/performance/mark-resolution/expanded/red-folders.log diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index 5350f494..4d257c9f 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -2426,3 +2426,50 @@ fn covered_review_status_budget(cx: &mut TestAppContext) { cx.update(gpui_omarchy::init); review_status_budget(cx, true); } + +#[gpui_kit::test] +fn folder_queue_construction_budget(cx: &mut TestAppContext) { + use crate::marks::work; + use disktree_core::tree::{Node, NodeKind, aggregate}; + use std::sync::Arc; + + cx.update(gpui_omarchy::init); + let mut measure = |count| { + let (_temp, view, window) = queue_budget_view(count, cx); + let operations = update(&view, window, |app, cx| { + let metric = app.options.metric; + let tree = Arc::make_mut(app.tree.as_mut().expect("tree")); + for child in &mut tree.children[0].children { + child.kind = NodeKind::Directory; + child + .children + .push(Node::entry("payload", NodeKind::File, 1)); + } + aggregate(tree, metric); + work::reset(); + let start = std::time::Instant::now(); + for index in 0..count { + app.toggle_mark(&[0, index], cx); + } + assert_eq!(app.marks.len(), count); + assert!(app.marks.items().iter().all(|item| item.is_dir)); + assert_eq!(work::FRAMES.get(), 0); + let operations = work::MARKING.get() + work::COMPACTION.get(); + assert!(operations > 0); + eprintln!( + "folder-queue M={count} work={operations} wall={:?}", + start.elapsed() + ); + operations + }); + window.update(|window, _| window.remove_window()); + operations + }; + let small = measure(128); + let large = measure(1024); + assert!(large <= small * 16, "8x folders: {small} -> {large}"); + assert!( + large <= 1024 * 32, + "absolute folder insertion budget: {large}" + ); +} diff --git a/docs/performance/mark-resolution/expanded/red-folders.log b/docs/performance/mark-resolution/expanded/red-folders.log new file mode 100644 index 00000000..fd240b02 --- /dev/null +++ b/docs/performance/mark-resolution/expanded/red-folders.log @@ -0,0 +1,23 @@ + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 4.21s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 1 test +test tests::folder_queue_construction_budget ... folder-queue M=128 work=16384 wall=2.888959ms +folder-queue M=1024 work=1048576 wall=177.786542ms + +thread 'tests::folder_queue_construction_budget' (404519213) panicked at crates/disktree-app/src/tests.rs:2365:5: +8x folders: 16384 -> 1048576 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED + +failures: + +failures: + tests::folder_queue_construction_budget + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 83 filtered out; finished in 0.32s + +error: test failed, to rerun pass `-p disktree-app --bin disktree` From 24bf81bc32d9abc15a284129509e39b26b762327 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 16:08:08 -0400 Subject: [PATCH 14/19] fix: preserve ancestor selection for raw mark spellings --- crates/disktree-app/src/marks.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/disktree-app/src/marks.rs b/crates/disktree-app/src/marks.rs index bc7ed46a..1d34484f 100644 --- a/crates/disktree-app/src/marks.rs +++ b/crates/disktree-app/src/marks.rs @@ -78,9 +78,9 @@ impl Marks { self.index.contains(path) } - /// Outermost marked ancestor, excluding the path itself. Keep the - /// spelling stored with the mark, using the same Path semantics as - /// exact membership and the old `starts_with` scan. + /// Marked ancestor with the shortest stored spelling, excluding the + /// path itself: the outermost one for paths built by the scan. Preserve + /// that choice even for raw queues with redundant path separators. pub fn ancestor(&self, path: &Path) -> Option<&Path> { path.ancestors() .skip(1) @@ -89,7 +89,7 @@ impl Marks { work::MARKING.set(work::MARKING.get() + 1); self.index.get(ancestor) }) - .last() + .min_by_key(|ancestor| ancestor.as_os_str().len()) .map(PathBuf::as_path) } @@ -533,6 +533,15 @@ mod tests { ); marks.clear(); assert_eq!(marks.absorb(Path::new("/synthetic")), 0); + + // Raw control clients can supply redundant separators. Retain the + // original shortest-spelling rule rather than changing notices. + marks.toggle(target("/synthetic/a////////////////////", 1)); + marks.toggle(target("/synthetic/a/child", 1)); + assert_eq!( + marks.ancestor(Path::new("/synthetic/a/child/file")), + Some(Path::new("/synthetic/a/child")) + ); } #[test] From 2ca9fce60ffc4fc90a59677d69d7e09b24a8f5d7 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 16:08:16 -0400 Subject: [PATCH 15/19] docs: explain the combined queue fixes with red-green evidence --- docs/performance/mark-resolution/README.md | 29 +- .../mark-resolution/additional-audit.md | 6 + .../mark-resolution/expanded/README.md | 211 ++++++++++++++ .../mark-resolution/expanded/green.log | 57 ++++ .../expanded/interactions-before.log | 65 +++++ .../expanded/interactions-final.log | 66 +++++ .../expanded/interactions-main.log | 66 +++++ .../mark-resolution/expanded/lint.log | 7 + .../mark-resolution/expanded/tests.log | 275 ++++++++++++++++++ .../mark-resolution/expanded/wall-after.log | 127 ++++++++ .../mark-resolution/expanded/wall-before.log | 187 ++++++++++++ .../mark-resolution/interactions.md | 4 + 12 files changed, 1094 insertions(+), 6 deletions(-) create mode 100644 docs/performance/mark-resolution/expanded/README.md create mode 100644 docs/performance/mark-resolution/expanded/green.log create mode 100644 docs/performance/mark-resolution/expanded/interactions-before.log create mode 100644 docs/performance/mark-resolution/expanded/interactions-final.log create mode 100644 docs/performance/mark-resolution/expanded/interactions-main.log create mode 100644 docs/performance/mark-resolution/expanded/lint.log create mode 100644 docs/performance/mark-resolution/expanded/tests.log create mode 100644 docs/performance/mark-resolution/expanded/wall-after.log create mode 100644 docs/performance/mark-resolution/expanded/wall-before.log diff --git a/docs/performance/mark-resolution/README.md b/docs/performance/mark-resolution/README.md index 4bc40c0c..fa5168ee 100644 --- a/docs/performance/mark-resolution/README.md +++ b/docs/performance/mark-resolution/README.md @@ -1,4 +1,21 @@ -# Mark resolution and removal-plan performance +# Large removal-queue performance + +The final combined scope and current validation are documented in +[expanded evidence](expanded/README.md): the original path-resolution and +removal-plan fixes plus efficient queue construction, one-pass parent +absorption and indexed Review status. The expanded regular budgets fail +before those additions and pass afterward, including 8,192-entry Review +queues. Current validation: 241 enabled tests and the required lint gate pass. + +A mark is one queued file or folder, regardless of how many files a marked +folder contains. The original 6.60-second stress result was four forced draws +and eight prepares of Explore; it was not one scan or one click. The expanded +report separates whole-queue construction, one parent action, four Review +draws and individual UI inputs, and includes fresh combined-head timings. + +The sections below preserve the original three fixes' measurements and +reproduction history. They are historical stage results; the expanded report +is the source for the final combined implementation and validation. Measured on 2026-10-14, Darwin arm64, Rust 1.97.1, repository development/test profile (application opt-level 1, dependency opt-level 2). All trees and paths @@ -24,10 +41,10 @@ costs can recur on renders while interacting, rather than once per scan. ## Broader follow-up audit [Additional rendering/scanning findings](additional-audit.md) document -remaining queue-construction, parent-absorption, Review status-join, depth and -mount-topology cases, plus a case-sensitive classification edge case. These -are separately scoped follow-ups; PR #9's three original fixes do not remove -them. Reproducible test-only probes and raw counts are included. +the audit that identified queue construction, parent absorption and Review +status joins, now addressed in the expansion. Depth, mount-topology and the +case-sensitive classification edge case remain separate follow-ups. +Historical test-only probes and raw counts are included. ## Red before green @@ -177,7 +194,7 @@ removed when its production caller moved to this API. Display callers use `Marks::crumbs(&Path, &Arc, Metric) -> Arc>>`; callers have no revision-counter or manual invalidation obligation. -## Verification and reproduction +## Original-stage verification and reproduction - `cargo xtask lint`: pass; [lint.log](lint.log). - `cargo xtask test`: 78 app, 150 core and 3 xtask tests pass; diff --git a/docs/performance/mark-resolution/additional-audit.md b/docs/performance/mark-resolution/additional-audit.md index ce4f0675..a5663240 100644 --- a/docs/performance/mark-resolution/additional-audit.md +++ b/docs/performance/mark-resolution/additional-audit.md @@ -1,5 +1,11 @@ # Additional rendering and scanning complexity audit +**Historical audit:** queue construction, parent absorption and Review status +joins below are now fixed by the [expanded PR #9](expanded/README.md). The +remaining depth, mount-topology and classification findings stay outside +that queue-focused scope. Counts in this file describe the audited snapshot, +not the final combined implementation. + Audited on 2026-10-14 at `ab11b1fd884e950faa9902c0b3e0e870eb6815d9`: main `6c8d4ce` plus PR #9's fixes. APFS feature code is outside this audit. The cases below remain after the original three fixes. This follow-up records diff --git a/docs/performance/mark-resolution/expanded/README.md b/docs/performance/mark-resolution/expanded/README.md new file mode 100644 index 00000000..99750103 --- /dev/null +++ b/docs/performance/mark-resolution/expanded/README.md @@ -0,0 +1,211 @@ +# Expanded PR #9: construction, absorption and Review status + +This is the final evidence for the three queue issues discovered after the +original PR #9 fixes. The scope is queue construction, marking a parent over +marked descendants, and Review's blocked/covered row lookups. The existing +removal nesting, mark-refresh and cached-crumb fixes remain included. + +A mark is one independently queued file or folder. Marking a folder counts +once regardless of its descendant file count. Long queues are relevant to +programmatic/MCP control; no usage evidence establishes 1,024 independent +marks as a frequent manual workload. This PR does not add an MCP endpoint. + +## Trigger and denominator + +| Operation | What is timed | When the cost is paid | +| --- | --- | --- | +| Queue construction | All 1,024 state-level additions, before any drawing | Building a queue of disjoint paths; files and folders have separate budgets | +| Parent absorption | One call marking a parent over 1,024 child marks, plus an unrelated mark | That marking action, before rendering | +| Blocked/covered Review | Four forced draws, eight observed renders | Repeated Review rendering, including opening/updating the screen | +| Space, hover, C input | One input plus harness rendering/settling | Ordinary interaction with an already populated queue | + +The earlier 6.60 s number was four forced Explore draws/eight prepares on +main, not a single click, one scan, or the time to select all 1,024 paths. +Neither queue construction nor parent absorption here includes painting. +The Review wall measurements below do include row layout/painting and fresh +plan evaluation. Comparisons per render are reported separately from total +wall time. Timed input tests assert zero `Marks::refresh` calls: no rescan is +included in those numbers. + +## Red before green + +At `57d4efde372e7f08e0a2b91a906ff3b870abaddc`, the new probes and budgets run +against the original queue/Review algorithms, with the earlier PR #9 fixes +already present. Four regular budgets fail: queue construction, parent +absorption, blocked Review and covered Review. See [red.log](red.log). +The added folder-only variant was also replayed against that unchanged +algorithm and fails; see [red-folders.log](red-folders.log). + +Runtime fixes are separate commits: + +- `e88b9e1`: ordered mark index, indexed ancestor lookup and one-pass + descendant absorption. +- `b71718f`: Review row indexes borrowed from each newly computed plan. +- `1c36fae`: the separate directory-insertion regression budget; it changes + test coverage, not runtime behavior. +- `d886865`: compatibility coverage for raw paths with redundant separators, + preserving the original shortest-stored-spelling ancestor choice. This + does not affect the canonical paths in the timing fixtures. + +The same regular budgets pass after the fixes. They grow input from 128 to +1,024 and require <=16x logical work, plus absolute caps. Queue mutation +allows <=32 units per mark; row status allows <=8 units per queued item per +observed render. Both Review variants also run at 8,192 entries and verify +that exactly 1,200 rows are built per render. + +| Measured path | Before, 128 / 1,024 | Combined fix, 128 / 1,024 | +| --- | ---: | ---: | +| Queue construction, all additions | 16,384 / 1,048,576 | 1,152 / 9,216 | +| Folder queue construction, all additions | 16,384 / 1,048,576 | 1,152 / 9,216 | +| Parent absorption, one action | 8,771 / 528,899 | 266 / 2,058 | +| Blocked Review status, per render | 8,256 / 524,800 | 384 / 3,072 | +| Covered Review status, per render | 8,384 / 525,824 | 386 / 3,074 | + +At 8,192 items, both Review cases use 10,592 status units per render, while +keeping the 1,200-row cap. See [green.log](green.log). The tests drive +`Disktree::toggle_mark` or actual GPUI window draws, not an isolated index +helper. They verify surviving mark order, counts, directory/file kinds, +covered/blocked decorations, render counts and the row cap. + +The logical mutation units count mark visits before the fix; after the fix +they count ancestor/range requests, removals and compaction visits. They do +not count internal B-tree comparisons. The complexity guarantee below +includes that logarithmic factor. Review units count status construction and +row lookups after the fix, versus repeated comparison scans before it. These +are deterministic work budgets, not CPU instruction counts. + +## Three-run wall measurements + +Medians in milliseconds from three serial repetitions, macOS arm64, Rust +1.97.1, the repository test profile, `--test-threads=1`. Compilation and +fixture setup are excluded. These workloads use synthetic displayed trees +backed by empty private temporary roots; they establish scaling, not storage +throughput. Raw samples: [wall-before.log](wall-before.log) and +[wall-after.log](wall-after.log). + +| Workload at 1,024 marks | Before expansion | Combined fix | +| --- | ---: | ---: | +| Queue all 1,024 files, no drawing | 176.13 ms | 6.41 ms | +| One parent-absorption action | 15.60 ms | 0.47 ms | +| Blocked Review, four draws/eight renders | 1,003.88 ms | 883.69 ms | +| Covered Review, four draws/eight renders | 1,110.66 ms | 1,009.63 ms | + +Review's wall improvement is much smaller than its row-lookup work reduction: +the screen still lays out up to 1,200 rows and computes plans/guards. +Eliminating one quadratic join does not eliminate those linear and +O(M log M) costs. The larger 8,192-item capped cases took approximately +1.19 s blocked / 2.11 s covered for four draws in these instrumented runs. +No elapsed-time threshold or frame-rate claim is made. + +## Implementation and guarantees + +Let M be current marks, D path depth, K absorbed descendants, B blocked +entries, C covered entries and L displayed rows (L <= 1,200). Path lengths +add comparison/hashing/copying costs to the bounds below. + +`Marks` keeps its ordered `Vec` for display and replaces the private +exact-path hash set with a `BTreeSet`. Native `Path` ordering is by +components, so a strict descendant, if any, immediately follows a parent's +position in that ordered index. Prefix lookalikes such as `a-b`, `a.b` and +`ab` do not defeat that query. + +- Ancestor lookup visits path ancestors and queries the index, costing + O(D log M) index work rather than scanning M marks. +- A disjoint insertion proves that no descendants exist using an ordered + range query. Building M disjoint marks therefore costs O(M*D*log M), + instead of O(M^2). Both files and directories follow this path. +- A parent with K marked descendants compacts the item vector once and + removes those K keys: O(M + K log M), instead of K full compactions. +- Exact mark membership changes from expected hash O(1) to O(log M). The + index still stores one path per mark and adds no fields to scan nodes. +- Unmarking one item still compacts the ordered vector in O(M). Multiple + separately invoked parent actions can each require their own compaction; + this is a per-action bound, not a promise that arbitrary sequences of + scalar edits cost one pass in total. + +`Disktree::toggle_mark` and the existing public behavior stay intact: it +refuses a mark beneath an already marked ancestor, absorbs descendants, +keeps unrelated marks in their original order and reports the absorbed count. +Tree-to-crumb display caches are invalidated by the mutations, including the +new absorption operation. Refresh continues retaining missing paths at zero +bytes, and the ordered index stays consistent with those retained paths. + +`ReviewStatus` borrows covered paths and blocked reasons from the `Plan` +computed for the current render. Construction plus all row lookups costs +expected O(B + C + L) hashing work and O(B + C) temporary index storage. +The first blocked reason wins for duplicate paths, preserving the original +linear lookup's behavior. Item order, chips and the display cap remain. +No plan/status authorization is retained across renders. + +Tests preserve native path component/case semantics, exact non-UTF-8 paths, +first-reason behavior, parent mark retention, unrelated neighbours and cache +invalidation after absorption. Removal guards, measurement and execution +logic are unchanged; worker-side mount/link revalidation still occurs +immediately before deletion. Existing synthetic permanent-deletion tests +continue passing. + +## End-to-end input evidence + +The separate opt-in input benchmark uses real on-disk temporary directories: +1,025 sibling folders, one 64-byte file each, with the same 1,400x900 window +and both panels visible. Space queues one additional folder; hover and C +then run with that additional mark present. Three repetitions record the +actual render/prepare/plan counts and zero refreshes. These inputs cause two +or three renders in the harness, not universally one production frame. + +[interactions-main.log](interactions-main.log) replays main algorithms +(`5a3c92e`, tests/probes over main `6c8d4ce`) with the input test applied. +[interactions-final.log](interactions-final.log) runs the combined fixes +immediately afterward. [interactions-before.log](interactions-before.log) +is a separate control with only the original PR #9 fixes, before this +expansion. A generated table of the latest input medians is kept below. +These test-profile samples are observations, not release-app per-click +guarantees; small queues do not support a substantial speedup claim. + +| Already queued | Space: main → final | Hover: main → final | Review C: main → final | +| ---: | ---: | ---: | ---: | +| 0 | 3.52 → 3.52 ms | 3.64 → 3.58 ms | 3.10 → 3.07 ms | +| 1 | 3.79 → 3.76 ms | 3.94 → 3.87 ms | 3.47 → 3.43 ms | +| 8 | 5.14 → 5.01 ms | 5.28 → 5.14 ms | 5.82 → 5.74 ms | +| 32 | 8.70 → 7.23 ms | 8.85 → 7.43 ms | 13.53 → 12.19 ms | +| 128 | 41.54 → 16.13 ms | 40.85 → 16.26 ms | 62.99 → 38.19 ms | +| 1,024 | 1584.73 → 96.49 ms | 1575.83 → 97.78 ms | 1786.18 → 300.71 ms | + +## Validation and reproduction + +- `cargo xtask lint`: pass; [lint.log](lint.log). +- `cargo xtask test`: 88 app + 150 core + 3 xtask tests pass (241 total); + [tests.log](tests.log). Two pre-existing ignored core tests and the manual + timing test remain ignored by normal CI. +- `cargo test --workspace budget -- --nocapture --test-threads=1`: all + original and expanded budgets pass, including 8,192-entry Review cases. +- The opt-in input timing test passes when explicitly run. +- Local validation is macOS arm64. Other operating systems remain for the + repository's existing CI; no new cross-platform result is claimed. + +The first four red failures can be reproduced at `57d4efd`: + +```sh +cargo test -p disktree-app budget -- --nocapture --test-threads=1 +``` + +At the final head: + +```sh +cargo xtask lint +cargo xtask test +cargo test --workspace budget -- --nocapture --test-threads=1 +cargo test -p disktree-app marked_interaction_timings -- \ + --ignored --nocapture --test-threads=1 +``` + +These changes improve the existing state/render paths. Future programmatic +bulk control should still submit additions within one UI transaction rather +than require a redraw per tool call. Per-render plans continue processing +the queued items, and independently rendering R times still pays that cost +R times. No MCP transport/control implementation is included here. + +Deep breadcrumbs, Linux mount-topology checks and the unusual classification +case from the [broader audit](../additional-audit.md) remain separate +follow-ups. This PR is one main-based review unit for large removal queues; +it does not depend on the APFS feature branch. diff --git a/docs/performance/mark-resolution/expanded/green.log b/docs/performance/mark-resolution/expanded/green.log new file mode 100644 index 00000000..62924d9e --- /dev/null +++ b/docs/performance/mark-resolution/expanded/green.log @@ -0,0 +1,57 @@ + Finished `test` profile [optimized + debuginfo] target(s) in 0.35s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 10 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=24.375µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=48.083µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=52.167µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=26.25µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=92.083µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=203.5µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.533958ms +ok +test tests::blocked_review_status_budget ... review-status covered=false M=128 work=3072 renders=8 rows=128 per-render=384 wall=107.112541ms +review-status covered=false M=1024 work=24576 renders=8 rows=1024 per-render=3072 wall=896.730708ms +review-status covered=false M=8192 work=84736 renders=8 rows=1200 per-render=10592 wall=1.196129834s +ok +test tests::covered_review_status_budget ... review-status covered=true M=128 work=3088 renders=8 rows=129 per-render=386 wall=122.642875ms +review-status covered=true M=1024 work=24592 renders=8 rows=1025 per-render=3074 wall=1.006027084s +review-status covered=true M=8192 work=84736 renders=8 rows=1200 per-render=10592 wall=2.109495042s +ok +test tests::folder_queue_construction_budget ... folder-queue M=128 work=1152 wall=542µs +folder-queue M=1024 work=9216 wall=6.55425ms +ok +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=60.956167ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=61.972375ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=332.587667ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=333.501ms +ok +test tests::parent_absorption_budget ... parent-absorption M=128 work=266 compactions=130 wall=57.667µs +parent-absorption M=1024 work=2058 compactions=1026 wall=472.542µs +ok +test tests::queue_construction_budget ... queue-construction M=128 work=1152 wall=554.417µs +queue-construction M=1024 work=9216 wall=6.531ms +ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 79 filtered out; finished in 6.89s + + Running unittests src/lib.rs (target/debug/deps/disktree_core-9300e5ba20d6b25f) + +running 1 test +test removal::tests::removal_nesting_budget ... plan M=128 work=1314 wall=2.843041ms +plan M=1024 work=15298 wall=22.107625ms +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 151 filtered out; finished in 0.03s + + Running unittests src/main.rs (target/debug/deps/xtask-0dc2df263f51d89e) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 3 filtered out; finished in 0.00s + diff --git a/docs/performance/mark-resolution/expanded/interactions-before.log b/docs/performance/mark-resolution/expanded/interactions-before.log new file mode 100644 index 00000000..93d390f5 --- /dev/null +++ b/docs/performance/mark-resolution/expanded/interactions-before.log @@ -0,0 +1,65 @@ + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 6.06s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 1 test +test tests::marked_interaction_timings ... interaction action=mark-space repeat=1 marks=0->1 renders=3 prepares=3 plans=6 refreshes=0 wall=5.29ms +interaction action=hover repeat=1 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.693417ms +interaction action=review-c repeat=1 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=3.161125ms +interaction action=mark-space repeat=2 marks=0->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.493542ms +interaction action=hover repeat=2 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=4.177458ms +interaction action=review-c repeat=2 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=3.333041ms +interaction action=mark-space repeat=3 marks=0->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.531208ms +interaction action=hover repeat=3 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.598083ms +interaction action=review-c repeat=3 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=3.059417ms +interaction action=mark-space repeat=1 marks=1->2 renders=3 prepares=3 plans=6 refreshes=0 wall=6.413958ms +interaction action=hover repeat=1 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=4.211334ms +interaction action=review-c repeat=1 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=3.781ms +interaction action=mark-space repeat=2 marks=1->2 renders=2 prepares=2 plans=4 refreshes=0 wall=4.085666ms +interaction action=hover repeat=2 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=4.282084ms +interaction action=review-c repeat=2 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=3.605958ms +interaction action=mark-space repeat=3 marks=1->2 renders=2 prepares=2 plans=4 refreshes=0 wall=4.143666ms +interaction action=hover repeat=3 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=4.118042ms +interaction action=review-c repeat=3 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=3.837666ms +interaction action=mark-space repeat=1 marks=8->9 renders=3 prepares=3 plans=6 refreshes=0 wall=8.128709ms +interaction action=hover repeat=1 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.588583ms +interaction action=review-c repeat=1 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=6.106292ms +interaction action=mark-space repeat=2 marks=8->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.294584ms +interaction action=hover repeat=2 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.593625ms +interaction action=review-c repeat=2 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=6.023584ms +interaction action=mark-space repeat=3 marks=8->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.320958ms +interaction action=hover repeat=3 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.340792ms +interaction action=review-c repeat=3 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=5.888875ms +interaction action=mark-space repeat=1 marks=32->33 renders=3 prepares=3 plans=6 refreshes=0 wall=11.475584ms +interaction action=hover repeat=1 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=8.573458ms +interaction action=review-c repeat=1 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=13.598084ms +interaction action=mark-space repeat=2 marks=32->33 renders=2 prepares=2 plans=4 refreshes=0 wall=8.43825ms +interaction action=hover repeat=2 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=8.692292ms +interaction action=review-c repeat=2 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=13.951083ms +interaction action=mark-space repeat=3 marks=32->33 renders=2 prepares=2 plans=4 refreshes=0 wall=8.248416ms +interaction action=hover repeat=3 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=9.179667ms +interaction action=review-c repeat=3 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=14.361125ms +interaction action=mark-space repeat=1 marks=128->129 renders=3 prepares=3 plans=6 refreshes=0 wall=25.540333ms +interaction action=hover repeat=1 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=17.300917ms +interaction action=review-c repeat=1 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=41.825167ms +interaction action=mark-space repeat=2 marks=128->129 renders=2 prepares=2 plans=4 refreshes=0 wall=17.227209ms +interaction action=hover repeat=2 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=17.180542ms +interaction action=review-c repeat=2 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=39.642416ms +interaction action=mark-space repeat=3 marks=128->129 renders=2 prepares=2 plans=4 refreshes=0 wall=17.732042ms +interaction action=hover repeat=3 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=18.09625ms +interaction action=review-c repeat=3 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=45.706083ms +interaction action=mark-space repeat=1 marks=1024->1025 renders=3 prepares=3 plans=6 refreshes=0 wall=154.267542ms +interaction action=hover repeat=1 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=106.91175ms +interaction action=review-c repeat=1 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=322.193416ms +interaction action=mark-space repeat=2 marks=1024->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=97.879042ms +interaction action=hover repeat=2 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=99.106959ms +interaction action=review-c repeat=2 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=322.52575ms +interaction action=mark-space repeat=3 marks=1024->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=106.434417ms +interaction action=hover repeat=3 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=104.738875ms +interaction action=review-c repeat=3 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=320.722334ms +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 82 filtered out; finished in 4.32s + diff --git a/docs/performance/mark-resolution/expanded/interactions-final.log b/docs/performance/mark-resolution/expanded/interactions-final.log new file mode 100644 index 00000000..209a385f --- /dev/null +++ b/docs/performance/mark-resolution/expanded/interactions-final.log @@ -0,0 +1,66 @@ + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 4.71s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 1 test +test tests::marked_interaction_timings ... interaction action=mark-space repeat=1 marks=0->1 renders=3 prepares=3 plans=6 refreshes=0 wall=5.246292ms +interaction action=hover repeat=1 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.710917ms +interaction action=review-c repeat=1 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=3.242042ms +interaction action=mark-space repeat=2 marks=0->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.513208ms +interaction action=hover repeat=2 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.584084ms +interaction action=review-c repeat=2 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=3.063ms +interaction action=mark-space repeat=3 marks=0->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.520209ms +interaction action=hover repeat=3 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.578375ms +interaction action=review-c repeat=3 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=3.073ms +interaction action=mark-space repeat=1 marks=1->2 renders=3 prepares=3 plans=6 refreshes=0 wall=5.520041ms +interaction action=hover repeat=1 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.890709ms +interaction action=review-c repeat=1 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=3.48725ms +interaction action=mark-space repeat=2 marks=1->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.755416ms +interaction action=hover repeat=2 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.86925ms +interaction action=review-c repeat=2 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=3.427584ms +interaction action=mark-space repeat=3 marks=1->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.751084ms +interaction action=hover repeat=3 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.86175ms +interaction action=review-c repeat=3 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=3.408333ms +interaction action=mark-space repeat=1 marks=8->9 renders=3 prepares=3 plans=6 refreshes=0 wall=7.470209ms +interaction action=hover repeat=1 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.143125ms +interaction action=review-c repeat=1 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=5.73275ms +interaction action=mark-space repeat=2 marks=8->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.014375ms +interaction action=hover repeat=2 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.164583ms +interaction action=review-c repeat=2 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=5.76775ms +interaction action=mark-space repeat=3 marks=8->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.006375ms +interaction action=hover repeat=3 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.13825ms +interaction action=review-c repeat=3 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=5.735209ms +interaction action=mark-space repeat=1 marks=32->33 renders=3 prepares=3 plans=6 refreshes=0 wall=11.012916ms +interaction action=hover repeat=1 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=7.431459ms +interaction action=review-c repeat=1 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=13.125583ms +interaction action=mark-space repeat=2 marks=32->33 renders=2 prepares=2 plans=4 refreshes=0 wall=7.209709ms +interaction action=hover repeat=2 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=7.418334ms +interaction action=review-c repeat=2 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=12.185875ms +interaction action=mark-space repeat=3 marks=32->33 renders=2 prepares=2 plans=4 refreshes=0 wall=7.22675ms +interaction action=hover repeat=3 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=7.432875ms +interaction action=review-c repeat=3 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=12.091458ms +interaction action=mark-space repeat=1 marks=128->129 renders=3 prepares=3 plans=6 refreshes=0 wall=24.179958ms +interaction action=hover repeat=1 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=16.337667ms +interaction action=review-c repeat=1 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=39.234959ms +interaction action=mark-space repeat=2 marks=128->129 renders=2 prepares=2 plans=4 refreshes=0 wall=16.130417ms +interaction action=hover repeat=2 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=16.262167ms +interaction action=review-c repeat=2 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=37.713833ms +interaction action=mark-space repeat=3 marks=128->129 renders=2 prepares=2 plans=4 refreshes=0 wall=16.075125ms +interaction action=hover repeat=3 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=16.138417ms +interaction action=review-c repeat=3 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=38.188041ms +interaction action=mark-space repeat=1 marks=1024->1025 renders=3 prepares=3 plans=6 refreshes=0 wall=143.247792ms +interaction action=hover repeat=1 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=99.35325ms +interaction action=review-c repeat=1 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=302.670959ms +interaction action=mark-space repeat=2 marks=1024->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=96.486625ms +interaction action=hover repeat=2 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=95.861042ms +interaction action=review-c repeat=2 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=300.70975ms +interaction action=mark-space repeat=3 marks=1024->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=95.268667ms +interaction action=hover repeat=3 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=97.78325ms +interaction action=review-c repeat=3 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=298.794208ms +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 88 filtered out; finished in 4.04s + diff --git a/docs/performance/mark-resolution/expanded/interactions-main.log b/docs/performance/mark-resolution/expanded/interactions-main.log new file mode 100644 index 00000000..800e1841 --- /dev/null +++ b/docs/performance/mark-resolution/expanded/interactions-main.log @@ -0,0 +1,66 @@ + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 4.93s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 1 test +test tests::marked_interaction_timings ... interaction action=mark-space repeat=1 marks=0->1 renders=3 prepares=3 plans=6 refreshes=0 wall=5.255584ms +interaction action=hover repeat=1 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.762417ms +interaction action=review-c repeat=1 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=3.218458ms +interaction action=mark-space repeat=2 marks=0->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.503375ms +interaction action=hover repeat=2 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.638375ms +interaction action=review-c repeat=2 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=3.096042ms +interaction action=mark-space repeat=3 marks=0->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.517333ms +interaction action=hover repeat=3 marks=1->1 renders=2 prepares=2 plans=4 refreshes=0 wall=3.6145ms +interaction action=review-c repeat=3 marks=1->1 renders=3 prepares=1 plans=4 refreshes=0 wall=3.080083ms +interaction action=mark-space repeat=1 marks=1->2 renders=3 prepares=3 plans=6 refreshes=0 wall=5.511084ms +interaction action=hover repeat=1 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.941834ms +interaction action=review-c repeat=1 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=3.503667ms +interaction action=mark-space repeat=2 marks=1->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.788ms +interaction action=hover repeat=2 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.90975ms +interaction action=review-c repeat=2 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=3.466166ms +interaction action=mark-space repeat=3 marks=1->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.76825ms +interaction action=hover repeat=3 marks=2->2 renders=2 prepares=2 plans=4 refreshes=0 wall=3.96025ms +interaction action=review-c repeat=3 marks=2->2 renders=3 prepares=1 plans=4 refreshes=0 wall=3.458125ms +interaction action=mark-space repeat=1 marks=8->9 renders=3 prepares=3 plans=6 refreshes=0 wall=7.608625ms +interaction action=hover repeat=1 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.284208ms +interaction action=review-c repeat=1 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=5.894708ms +interaction action=mark-space repeat=2 marks=8->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.141833ms +interaction action=hover repeat=2 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.286125ms +interaction action=review-c repeat=2 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=5.815959ms +interaction action=mark-space repeat=3 marks=8->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.117792ms +interaction action=hover repeat=3 marks=9->9 renders=2 prepares=2 plans=4 refreshes=0 wall=5.267ms +interaction action=review-c repeat=3 marks=9->9 renders=3 prepares=1 plans=4 refreshes=0 wall=5.815375ms +interaction action=mark-space repeat=1 marks=32->33 renders=3 prepares=3 plans=6 refreshes=0 wall=12.998667ms +interaction action=hover repeat=1 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=8.816834ms +interaction action=review-c repeat=1 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=13.860541ms +interaction action=mark-space repeat=2 marks=32->33 renders=2 prepares=2 plans=4 refreshes=0 wall=8.695333ms +interaction action=hover repeat=2 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=8.8455ms +interaction action=review-c repeat=2 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=13.520791ms +interaction action=mark-space repeat=3 marks=32->33 renders=2 prepares=2 plans=4 refreshes=0 wall=8.686166ms +interaction action=hover repeat=3 marks=33->33 renders=2 prepares=2 plans=4 refreshes=0 wall=8.850291ms +interaction action=review-c repeat=3 marks=33->33 renders=3 prepares=1 plans=4 refreshes=0 wall=13.525542ms +interaction action=mark-space repeat=1 marks=128->129 renders=3 prepares=3 plans=6 refreshes=0 wall=58.431709ms +interaction action=hover repeat=1 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=39.571ms +interaction action=review-c repeat=1 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=62.991042ms +interaction action=mark-space repeat=2 marks=128->129 renders=2 prepares=2 plans=4 refreshes=0 wall=41.537667ms +interaction action=hover repeat=2 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=42.52325ms +interaction action=review-c repeat=2 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=63.378542ms +interaction action=mark-space repeat=3 marks=128->129 renders=2 prepares=2 plans=4 refreshes=0 wall=40.624666ms +interaction action=hover repeat=3 marks=129->129 renders=2 prepares=2 plans=4 refreshes=0 wall=40.849792ms +interaction action=review-c repeat=3 marks=129->129 renders=3 prepares=1 plans=4 refreshes=0 wall=62.616375ms +interaction action=mark-space repeat=1 marks=1024->1025 renders=3 prepares=3 plans=6 refreshes=0 wall=2.353868208s +interaction action=hover repeat=1 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=1.5475255s +interaction action=review-c repeat=1 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=1.786180708s +interaction action=mark-space repeat=2 marks=1024->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=1.558053083s +interaction action=hover repeat=2 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=1.588539167s +interaction action=review-c repeat=2 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=1.814801708s +interaction action=mark-space repeat=3 marks=1024->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=1.584729708s +interaction action=hover repeat=3 marks=1025->1025 renders=2 prepares=2 plans=4 refreshes=0 wall=1.575834417s +interaction action=review-c repeat=3 marks=1025->1025 renders=3 prepares=1 plans=4 refreshes=0 wall=1.766631333s +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 73 filtered out; finished in 37.74s + diff --git a/docs/performance/mark-resolution/expanded/lint.log b/docs/performance/mark-resolution/expanded/lint.log new file mode 100644 index 00000000..ee7838ac --- /dev/null +++ b/docs/performance/mark-resolution/expanded/lint.log @@ -0,0 +1,7 @@ + Finished `dev` profile [optimized + debuginfo] target(s) in 0.18s + Running `target/debug/xtask lint` + Checking disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `dev` profile [optimized + debuginfo] target(s) in 2.21s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` diff --git a/docs/performance/mark-resolution/expanded/tests.log b/docs/performance/mark-resolution/expanded/tests.log new file mode 100644 index 00000000..a27e00a6 --- /dev/null +++ b/docs/performance/mark-resolution/expanded/tests.log @@ -0,0 +1,275 @@ + Finished `dev` profile [optimized + debuginfo] target(s) in 0.11s + Running `target/debug/xtask test` + Compiling disktree-core v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-core) + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 3.26s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 89 tests +test appearance::tests::a_linux_desktop_is_followed_only_if_it_states_a_preference ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::display_path_shortens_the_home_prefix ... ok +test git::tests::summaries_say_what_would_be_lost ... ok +test marks::tests::marking_the_same_path_twice_keeps_one_entry ... ok +test marks::tests::ordered_marks_keep_exact_non_utf8_paths ... ok +test marks::tests::ordered_marks_preserve_components_case_and_outermost_ancestor ... ok +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test marks::tests::resolver_preserves_lossy_non_utf8_components ... ok +test marks::tests::toggling_marks_and_unmarks ... ok +test marks::tests::find_matches_whole_components_only ... ok +test marks::tests::refresh_re_reads_sizes_from_a_new_tree ... ok +test marks::tests::absorption_invalidates_cached_crumbs_and_keeps_missing_paths_indexed ... ok +test marks::tests::cached_crumbs_follow_every_invalidation_boundary ... ok +test marks::tests::resolver_preserves_scan_name_semantics ... ok +test palette::tests::age_buckets_cover_every_age_in_order ... ok +test palette::tests::colourful_categories_share_one_level ... ok +test palette::tests::deeper_tiles_lift_away_from_the_background ... ok +test palette::tests::every_legend_category_has_its_own_hue ... ok +test palette::tests::mix_clamps_its_parameter ... ok +test palette::tests::mixing_toward_a_grey_keeps_the_hue ... ok +test palette::tests::the_highlight_is_not_a_category_colour ... ok +test power::tests::presets_are_fixed_and_show_the_effective_cpu_limit ... ok +test power::tests::presets_that_add_no_workers_are_not_offered ... ok +test state::tests::a_transition_grows_a_tile_that_is_being_entered ... ok +test state::tests::a_transition_starts_where_the_region_was_and_ends_where_it_is ... ok +test state::tests::clamping_never_leaves_a_margin ... ok +test state::tests::directions_only_see_gaps_on_their_own_side ... ok +test argument_tests::saved_power_is_loaded_before_cli_overrides ... ok +test state::tests::projecting_and_unprojecting_are_inverses ... ok +test state::tests::zooming_keeps_the_point_under_the_cursor_still ... ok +test marks::tests::refresh_dimension_budget ... ok +test power::tests::saved_choice_survives_replacement_and_bad_data_is_reported ... ok +test marks::tests::refresh_resolution_budget ... ok +test git::tests::reads_a_real_checkout ... ok +test git::tests::a_checkout_cannot_make_git_run_its_programs ... ok +test git::tests::a_checkout_cannot_make_git_verify_a_signature_with_its_program ... ok +test git::tests::a_checkouts_own_filter_is_never_run ... ok +test tests::escape_cancels_widening_and_keeps_the_tree_on_screen ... ok +test tests::escape_in_the_delete_dialog_cancels ... ok +test tests::command_chords_are_not_read_as_plain_letters ... ok +test tests::enter_before_the_search_lands_applies_it_when_it_does ... ok +test tests::marked_interaction_timings ... ignored, manual interaction timing on synthetic temporary files +test tests::a_view_without_flags_or_marks_still_draws ... ok +test tests::a_parents_name_gets_its_own_band_above_its_children ... ok +test tests::a_permanent_deletion_asks_in_an_alert_dialog_then_removes ... ok +test tests::a_crumb_lists_its_siblings_and_jumps_sideways ... ok +test tests::enter_in_the_focused_volume_picker_scans_the_selected_root ... ok +test tests::folder_queue_construction_budget ... ok +test tests::enter_opens_the_selected_directory_at_any_depth ... ok +test tests::restart_arguments_parse_back_to_the_same_scan ... ok +test tests::after_descending_every_tile_is_inside_the_directory_drawn ... ok +test tests::both_appearances_draw ... ok +test tests::back_and_forward_retrace_where_you_have_been ... ok +test tests::dragging_the_panel_edge_resizes_it_within_its_limits ... ok +test tests::escape_cancels_the_first_scan_and_r_starts_it_again ... ok +test tests::hovering_reports_the_tile_under_the_pointer ... ok +test tests::interface_zoom_scales_the_rem_and_the_header_band ... ok +test tests::marked_tiles_follow_metric_rescan_and_mark_changes ... ok +test tests::marking_a_directory_marks_everything_inside_it ... ok +test tests::power_efficiency_menu_saves_without_discarding_the_tree ... ok +test tests::showing_a_tile_that_is_gone_says_so_instead ... ok +test tests::keys_walk_the_tree_and_mark_what_is_selected ... ok +test tests::switching_the_metric_keeps_the_directory_and_the_selection ... ok +test tests::mouse_side_buttons_go_back_and_forward ... ok +test tests::parent_absorption_budget ... ok +test tests::right_click_selects_the_tile_and_reveals_it ... ok +test tests::the_first_scan_shows_what_it_is_doing_then_the_treemap ... ok +test views::tests::review_status_keeps_non_utf8_paths_distinct ... ok +test views::tests::review_status_preserves_exact_paths_and_first_blocked_reason ... ok +test widgets::tests::ages_read_in_the_unit_a_person_would_use ... ok +test widgets::tests::sizes_split_into_number_and_unit ... ok +test tests::the_help_overlay_opens_and_closes ... ok +test tests::the_mark_key_follows_the_pointer_until_the_keyboard_moves ... ok +test tests::queue_construction_budget ... ok +test tests::the_power_menu_offers_only_what_the_cpus_can_tell_apart ... ok +test tests::the_review_screen_copies_the_list_as_an_agent_prompt ... ok +test tests::the_review_screen_switches_removal_mode ... ok +test tests::the_trash_is_the_default_when_there_is_one ... ok +test tests::the_volume_picker_is_centred_and_still_dismisses_from_outside ... ok +test tests::marked_window_resolution_budget ... ok +test tests::the_wheel_goes_into_the_directory_it_zoomed_into ... ok +test tests::the_treemap_zooms_with_the_wheel_and_resets ... ok +test tests::the_volume_picker_opens_moves_and_closes ... ok +test tests::typing_filters_live_and_enter_shows_only_the_matches ... ok +test tests::widening_reuses_the_tree_it_has_and_reads_only_the_rest ... ok +test tests::the_window_draws_a_treemap_with_tiles ... ok +test tests::blocked_review_status_budget ... ok +test tests::covered_review_status_budget ... ok + +test result: ok. 88 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 4.87s + + Running unittests src/lib.rs (target/debug/deps/disktree_core-9300e5ba20d6b25f) + +running 152 tests +test classify::tests::macos_developer_leftovers_are_reclaimable_and_its_risks_are_not ... ok +test export::tests::the_list_is_one_path_per_line ... ok +test classify::tests::steam_libraries_and_emulation_are_media_not_disposable_caches ... ok +test filter::tests::case_is_ignored_without_allocating ... ok +test classify::tests::the_legend_lists_every_named_category_once ... ok +test classify::tests::windows_caches_and_sync_folders_are_recognized ... ok +test export::tests::a_name_with_a_newline_cannot_smuggle_in_a_path ... ok +test export::tests::a_name_that_is_not_unicode_is_flagged_not_listed ... ok +test export::tests::the_prompt_lists_every_path_with_its_size_and_the_rules ... ok +test filter::tests::an_empty_needle_filters_nothing ... ok +test filter::tests::a_search_below_the_root_leaves_the_rest_alone ... ok +test filter::tests::matches_are_kept_whole_and_their_ancestors_by_what_matched ... ok +test access::tests::only_macos_has_an_answer_and_a_missing_database_is_not_a_refusal ... ok +test classify::tests::a_steam_library_on_another_drive_is_media_not_its_temp_folder ... ok +test classify::tests::sandbox_layers_and_snapshots_are_reclaimable_only_in_sandbox_state ... ok +test insights::tests::the_limit_keeps_the_largest ... ok +test insights::tests::ranks_findings_largest_first_and_skips_the_tiny ... ok +test insights::tests::documents_are_never_suggested ... ok +test insights::tests::crumbs_address_the_finding_from_the_root ... ok +test classify::tests::names_announce_their_kind_and_children_inherit_it ... ok +test classify::tests::target_is_build_output_only_beside_a_cargo_manifest ... ok +test classify::tests::an_unknown_top_level_directory_takes_its_largest_known_child ... ok +test classify::tests::caches_are_reclaimable_all_the_way_down ... ok +test removal::tests::a_directory_holding_a_system_tree_is_refused ... ok +test removal::tests::a_directory_holding_the_home_directory_is_refused ... ok +test macos::tests::a_missing_directory_is_an_error ... ok +test removal::tests::macos_always_uses_the_system_trash ... ok +test removal::tests::macos_guard_keys_fold_the_data_volume_and_case ... ok +test removal::tests::macos_guards_see_through_other_spellings ... ok +test removal::tests::macos_system_trees_are_refused_but_apps_are_not ... ok +test removal::tests::a_name_that_is_not_utf8_is_refused_before_the_trash_sees_it ... ok +test removal::tests::names_win32_would_misread_are_recognized ... ok +test macos::tests::a_listing_has_names_kinds_lengths_and_no_dot_entries ... ok +test removal::tests::nesting_keeps_component_boundaries_duplicates_and_output_order ... ok +test removal::tests::normalize_resolves_dots_without_the_filesystem ... ok +test macos::tests::names_are_text_and_open ... ok +test removal::tests::nesting_uses_guard_case_order_even_when_the_child_sorts_first ... ok +test macos::tests::every_leaf_reads_as_lstat_reads_it ... ok +test removal::tests::a_mount_below_a_target_is_caught_again_at_removal_time ... ok +test removal::tests::a_volume_is_judged_by_its_mount_where_the_kernel_names_one ... ok +test removal::tests::percent_encoding_escapes_what_the_spec_requires ... ok +test removal::tests::a_missing_target_reports_not_found ... ok +test removal::tests::a_plan_keeps_only_the_outermost_targets ... ok +test removal::tests::a_target_with_a_mount_inside_is_refused ... ok +test removal::tests::mount_points_are_recognized ... ok +test removal::tests::a_trash_info_path_is_restorable ... ok +test removal::tests::a_target_reached_through_a_link_is_refused ... ok +test removal::tests::paths_outside_the_root_are_refused ... ok +test removal::tests::permanent_removal_takes_directories_and_leaves_siblings ... ok +test removal::tests::a_removal_run_reports_every_item_and_a_total ... ok +test removal::tests::the_macos_trash_takes_a_file_and_leaves_a_linked_target ... ignored, puts a file in this Mac's real Trash +test removal::tests::home_is_found_by_its_real_path_too ... ok +test removal::tests::permanent_removal_follows_links_above_the_root ... ok +test removal::tests::permanent_removal_does_not_follow_a_link_put_on_the_way_down ... ok +test removal::tests::permanent_removal_unlinks_nested_symlinks_without_following_them ... ok +test removal::tests::permanent_removal_takes_nested_trees_and_hidden_files ... ok +test removal::tests::permanent_removal_unlinks_a_symlink_instead_of_following_it ... ok +test removal::tests::nothing_is_mounted_below_a_temp_dir ... ok +test removal::tests::read_only_files_are_removed_too ... ok +test removal::tests::sibling_prefixes_are_not_treated_as_containment ... ok +test removal::tests::a_link_put_on_the_way_down_after_planning_stops_the_worker ... ok +test removal::tests::the_mount_table_is_read_off_the_calling_thread_and_then_kept ... ok +test removal::tests::trash_names_avoid_collisions ... ok +test removal::tests::the_root_and_home_are_refused ... ok +test removal::tests::the_xdg_trash_moves_a_file_and_records_where_it_came_from ... ok +test scan::tests::a_file_root_is_rejected ... ok +test scan::tests::fixed_worker_pools_change_size_without_reinitializing_rayon ... ok +test removal::tests::this_macs_mount_table_is_read_without_proc ... ok +test scan::tests::disk_usage_is_whole_allocation_units ... ok +test scan::tests::followed_symlink_loops_do_not_hang_the_scan ... ok +test scan::tests::hardlinks_are_charged_once_by_default ... ok +test scan::tests::an_unreadable_directory_is_recorded_not_fatal ... ok +test scan::tests::a_spawned_scan_reports_progress_and_a_result ... ok +test scan::tests::a_directory_reports_its_own_bytes_separately ... ok +test scan::tests::whole_disk_smoke ... ignored, walks the whole disk +test scan::tests::children_are_ranked_largest_first ... ok +test scan_threads::tests::governor_waits_for_retirement_and_recovers_without_exceeding_target ... ok +test scan_threads::tests::inconclusive_windows_cannot_accept_a_reduction ... ok +test scan_threads::tests::increased_limit_is_not_supplied_until_jobs_are_dispatched ... ok +test scan_threads::tests::long_directory_never_becomes_a_fully_supplied_window ... ok +test scan_threads::tests::search_refines_one_reference_without_compounding_loss ... ok +test size::tests::a_value_that_rounds_up_to_the_next_unit_is_shown_in_it ... ok +test size::tests::bytes_use_binary_units_with_one_decimal_below_ten ... ok +test size::tests::counts_switch_to_metric_suffixes ... ok +test size::tests::share_bar_is_always_the_requested_width ... ok +test scan_threads::tests::real_pool_retires_at_job_boundaries_and_drains_queued_descendants ... ok +test size::tests::share_survives_a_zero_total ... ok +test size::tests::short_form_drops_the_space ... ok +test space::tests::a_bind_mount_inside_the_scan_is_left_out ... ok +test space::tests::a_home_scan_has_no_foreign_mounts_here ... ok +test space::tests::a_missing_path_reports_the_io_error ... ok +test space::tests::a_real_volume_reports_plausible_numbers ... ok +test space::tests::a_root_booted_from_a_snapshot_keeps_its_subvolumes ... ok +test space::tests::a_table_that_does_not_place_a_path_claims_nothing ... ok +test space::tests::a_volume_includes_its_subvolumes_and_nothing_else ... ok +test space::tests::a_volume_keeps_its_subvolumes_and_loses_everything_else ... ok +test space::tests::btrfs_subvolume_devices_can_differ ... ok +test space::tests::excluded_views_cannot_replace_visible_files ... ok +test space::tests::hidden_original_and_nested_mounts_are_preserved ... ok +test space::tests::macos_mount_points_keep_spaces_and_brackets ... ok +test space::tests::mountinfo_escapes_are_decoded_once ... ok +test space::tests::only_macos_skips_the_data_volume_and_in_the_roots_spelling ... ok +test space::tests::projecting_removal_cannot_exceed_the_volume ... ok +test space::tests::separate_home_disk_is_its_own_volume ... ok +test space::tests::subvolumes_are_not_repeats_but_a_second_mount_of_a_disk_is ... ok +test space::tests::the_device_is_the_longest_matching_mount ... ok +test scan::tests::file_count_metric_ranks_by_entries ... ok +test space::tests::the_home_disk_on_macos_is_the_root_and_has_a_device ... ok +test space::tests::the_top_of_a_btrfs_disk_beside_its_subvolumes_is_counted_once ... ok +test space::tests::the_whole_disk_is_the_top_of_the_home_volume ... ok +test space::tests::unrelated_tmpfs_mounts_are_not_duplicates ... ok +test space::tests::used_fraction_handles_an_empty_volume_report ... ok +test space::tests::volume_candidates_are_real_disks_not_pseudo_filesystems ... ok +test space::tests::volumes_with_least_space_come_first_and_unknowns_last ... ok +test tree::tests::a_node_stays_small ... ok +test tree::tests::aggregate_derives_totals_and_orders_children ... ok +test scan::tests::a_wider_scan_reuses_the_subtree_it_already_knows ... ok +test tree::tests::depth_counts_edges ... ok +test tree::tests::find_reports_crumbs_and_is_case_insensitive ... ok +test tree::tests::aggregate_can_rank_by_file_count ... ok +test tree::tests::identities_that_differ_only_in_their_top_bits_are_two_files ... ok +test tree::tests::largest_child_is_the_first_child_because_children_are_sorted ... ok +test tree::tests::path_of_joins_names_beneath_the_root ... ok +test tree::tests::resolve_walks_child_indices ... ok +test treemap::tests::a_leaf_never_claims_a_header ... ok +test treemap::tests::a_filtered_layout_shows_only_the_matches_at_their_size ... ok +test treemap::tests::a_subdivided_directory_keeps_a_header_for_its_own_name ... ok +test treemap::tests::a_tile_without_room_for_a_header_stays_whole ... ok +test treemap::tests::hit_returns_the_deepest_tile ... ok +test treemap::tests::layout_depth_one_stops_at_direct_children ... ok +test tree::tests::seen_charges_an_identity_once_in_either_store ... ok +test treemap::tests::layout_merges_the_child_tail_into_one_tile ... ok +test treemap::tests::squarify_fills_the_area ... ok +test treemap::tests::layout_nests_children_inside_their_parent ... ok +test space::tests::without_a_table_the_same_place_is_one_volume ... ok +test scan::tests::hidden_entries_are_included_by_default_and_excluded_on_request ... ok +test treemap::tests::squarify_keeps_aspect_ratios_reasonable ... ok +test treemap::tests::squarify_survives_degenerate_input ... ok +test treemap::tests::tile_crumbs_address_the_scanned_tree_not_the_drawn_node ... ok +test scan::tests::max_depth_stops_descending_but_keeps_direct_files ... ok +test scan::tests::symlinks_are_not_followed_by_default ... ok +test scan::tests::totals_are_summed_bottom_up ... ok +test removal::tests::system_trees_are_refused_in_a_whole_disk_scan ... ok +test removal::tests::the_directories_above_home_are_refused ... ok +test removal::tests::removal_nesting_budget ... ok +test scan::tests::cancelled_queued_jobs_release_parent_completion_tokens ... ok +test scan::tests::fixed_worker_pools_preserve_a_wide_deep_tree ... ok +test removal::tests::only_the_scanned_volumes_marks_are_projected ... ok +test removal::tests::a_failing_trash_tool_reports_its_stderr ... ok +test scan_threads::tests::controller_retires_recovers_after_search_and_cancels ... ok +test macos::tests::a_directory_larger_than_the_buffer_is_listed_whole ... ok +test removal::tests::a_trash_tool_is_called_with_the_path_after_a_separator ... ok + +test result: ok. 150 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.28s + + Running unittests src/main.rs (target/debug/deps/xtask-0dc2df263f51d89e) + +running 3 tests +test bundle::tests::signing_is_ad_hoc_unless_an_identity_is_given ... ok +test bundle::tests::the_plist_template_names_the_executable_and_takes_the_version ... ok +test bundle::tests::notarizing_needs_a_real_signature ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests disktree_core + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + diff --git a/docs/performance/mark-resolution/expanded/wall-after.log b/docs/performance/mark-resolution/expanded/wall-after.log new file mode 100644 index 00000000..21a9e7eb --- /dev/null +++ b/docs/performance/mark-resolution/expanded/wall-after.log @@ -0,0 +1,127 @@ +REPEAT 1: sources at combined fix b71718f + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 4.24s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 9 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=144.833µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=62.791µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=74.667µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=38.75µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=87.5µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=295.792µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.906625ms +ok +test tests::blocked_review_status_budget ... review-status covered=false M=128 work=3072 renders=8 rows=128 per-render=384 wall=107.559125ms +review-status covered=false M=1024 work=24576 renders=8 rows=1024 per-render=3072 wall=883.20225ms +review-status covered=false M=8192 work=84736 renders=8 rows=1200 per-render=10592 wall=1.196734875s +ok +test tests::covered_review_status_budget ... review-status covered=true M=128 work=3088 renders=8 rows=129 per-render=386 wall=122.079458ms +review-status covered=true M=1024 work=24592 renders=8 rows=1025 per-render=3074 wall=997.160667ms +review-status covered=true M=8192 work=84736 renders=8 rows=1200 per-render=10592 wall=2.106200625s +ok +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=59.356875ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=60.529083ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=331.029917ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=335.875875ms +ok +test tests::parent_absorption_budget ... parent-absorption M=128 work=266 compactions=130 wall=58.125µs +parent-absorption M=1024 work=2058 compactions=1026 wall=467.083µs +ok +test tests::queue_construction_budget ... queue-construction M=128 work=1152 wall=542µs +queue-construction M=1024 work=9216 wall=6.311584ms +ok + +test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 79 filtered out; finished in 6.71s + + +EXIT 0 +REPEAT 2: sources at combined fix b71718f + Finished `test` profile [optimized + debuginfo] target(s) in 0.27s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 9 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=25.25µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=42.666µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=51.75µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=24.5µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=56.75µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=198.292µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.479125ms +ok +test tests::blocked_review_status_budget ... review-status covered=false M=128 work=3072 renders=8 rows=128 per-render=384 wall=105.373709ms +review-status covered=false M=1024 work=24576 renders=8 rows=1024 per-render=3072 wall=883.687ms +review-status covered=false M=8192 work=84736 renders=8 rows=1200 per-render=10592 wall=1.188244s +ok +test tests::covered_review_status_budget ... review-status covered=true M=128 work=3088 renders=8 rows=129 per-render=386 wall=122.092208ms +review-status covered=true M=1024 work=24592 renders=8 rows=1025 per-render=3074 wall=1.010354417s +review-status covered=true M=8192 work=84736 renders=8 rows=1200 per-render=10592 wall=2.112680083s +ok +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=61.657666ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=61.290042ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=335.296459ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=333.089792ms +ok +test tests::parent_absorption_budget ... parent-absorption M=128 work=266 compactions=130 wall=57.125µs +parent-absorption M=1024 work=2058 compactions=1026 wall=481.292µs +ok +test tests::queue_construction_budget ... queue-construction M=128 work=1152 wall=547.333µs +queue-construction M=1024 work=9216 wall=6.518083ms +ok + +test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 79 filtered out; finished in 6.70s + + +EXIT 0 +REPEAT 3: sources at combined fix b71718f + Finished `test` profile [optimized + debuginfo] target(s) in 0.26s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 9 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=25.583µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=42.583µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=51.292µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=26.042µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=59.083µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=214µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.552917ms +ok +test tests::blocked_review_status_budget ... review-status covered=false M=128 work=3072 renders=8 rows=128 per-render=384 wall=108.225542ms +review-status covered=false M=1024 work=24576 renders=8 rows=1024 per-render=3072 wall=886.057917ms +review-status covered=false M=8192 work=84736 renders=8 rows=1200 per-render=10592 wall=1.19339425s +ok +test tests::covered_review_status_budget ... review-status covered=true M=128 work=3088 renders=8 rows=129 per-render=386 wall=122.399375ms +review-status covered=true M=1024 work=24592 renders=8 rows=1025 per-render=3074 wall=1.009629625s +review-status covered=true M=8192 work=84736 renders=8 rows=1200 per-render=10592 wall=2.127712375s +ok +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=61.806167ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=61.474458ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=351.863667ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=336.533542ms +ok +test tests::parent_absorption_budget ... parent-absorption M=128 work=266 compactions=130 wall=57.75µs +parent-absorption M=1024 work=2058 compactions=1026 wall=459.042µs +ok +test tests::queue_construction_budget ... queue-construction M=128 work=1152 wall=547.75µs +queue-construction M=1024 work=9216 wall=6.406375ms +ok + +test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 79 filtered out; finished in 6.76s + + +EXIT 0 diff --git a/docs/performance/mark-resolution/expanded/wall-before.log b/docs/performance/mark-resolution/expanded/wall-before.log new file mode 100644 index 00000000..d2c6de76 --- /dev/null +++ b/docs/performance/mark-resolution/expanded/wall-before.log @@ -0,0 +1,187 @@ +REPEAT 1: sources at red commit 57d4efd + Compiling disktree-app v0.11.0 (/Users/huntharo/.codex/worktrees/mv8epstr/disktree/crates/disktree-app) + Finished `test` profile [optimized + debuginfo] target(s) in 6.29s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 9 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=88.833µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=66.209µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=72.458µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=40.208µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=96.25µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=292.5µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.89825ms +ok +test tests::blocked_review_status_budget ... review-status covered=false M=128 work=66048 renders=8 rows=128 per-render=8256 wall=111.492791ms +review-status covered=false M=1024 work=4198400 renders=8 rows=1024 per-render=524800 wall=1.003875917s + +thread 'tests::blocked_review_status_budget' (404509605) panicked at crates/disktree-app/src/tests.rs:2298:5: +8x status rows: 66048/8 -> 4198400/8 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED +test tests::covered_review_status_budget ... review-status covered=true M=128 work=67072 renders=8 rows=129 per-render=8384 wall=122.635083ms +review-status covered=true M=1024 work=4206592 renders=8 rows=1025 per-render=525824 wall=1.110662959s + +thread 'tests::covered_review_status_budget' (404509677) panicked at crates/disktree-app/src/tests.rs:2298:5: +8x status rows: 67072/8 -> 4206592/8 +FAILED +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=60.112708ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=61.191625ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=333.839916ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=332.639625ms +ok +test tests::parent_absorption_budget ... parent-absorption M=128 work=8771 compactions=8512 wall=332.125µs +parent-absorption M=1024 work=528899 compactions=526848 wall=15.169208ms + +thread 'tests::parent_absorption_budget' (404509733) panicked at crates/disktree-app/src/tests.rs:2223:5: +8x descendants: 8771 -> 528899 +FAILED +test tests::queue_construction_budget ... queue-construction M=128 work=16384 wall=2.928667ms +queue-construction M=1024 work=1048576 wall=176.133125ms + +thread 'tests::queue_construction_budget' (404509745) panicked at crates/disktree-app/src/tests.rs:2181:5: +8x marks: 16384 -> 1048576 +FAILED + +failures: + +failures: + tests::blocked_review_status_budget + tests::covered_review_status_budget + tests::parent_absorption_budget + tests::queue_construction_budget + +test result: FAILED. 5 passed; 4 failed; 0 ignored; 0 measured; 74 filtered out; finished in 3.77s + +error: test failed, to rerun pass `-p disktree-app --bin disktree` + +EXIT 101 +REPEAT 2: sources at red commit 57d4efd + Finished `test` profile [optimized + debuginfo] target(s) in 0.26s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 9 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=19.958µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=43.5µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=49.75µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=29.916µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=59.417µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=206.042µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.547208ms +ok +test tests::blocked_review_status_budget ... review-status covered=false M=128 work=66048 renders=8 rows=128 per-render=8256 wall=113.726291ms +review-status covered=false M=1024 work=4198400 renders=8 rows=1024 per-render=524800 wall=1.030923916s + +thread 'tests::blocked_review_status_budget' (404509806) panicked at crates/disktree-app/src/tests.rs:2298:5: +8x status rows: 66048/8 -> 4198400/8 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED +test tests::covered_review_status_budget ... review-status covered=true M=128 work=67072 renders=8 rows=129 per-render=8384 wall=123.684209ms +review-status covered=true M=1024 work=4206592 renders=8 rows=1025 per-render=525824 wall=1.122654292s + +thread 'tests::covered_review_status_budget' (404509980) panicked at crates/disktree-app/src/tests.rs:2298:5: +8x status rows: 67072/8 -> 4206592/8 +FAILED +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=60.245708ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=61.801583ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=341.867208ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=350.583125ms +ok +test tests::parent_absorption_budget ... parent-absorption M=128 work=8771 compactions=8512 wall=327.959µs +parent-absorption M=1024 work=528899 compactions=526848 wall=15.596208ms + +thread 'tests::parent_absorption_budget' (404510061) panicked at crates/disktree-app/src/tests.rs:2223:5: +8x descendants: 8771 -> 528899 +FAILED +test tests::queue_construction_budget ... queue-construction M=128 work=16384 wall=2.95175ms +queue-construction M=1024 work=1048576 wall=184.469666ms + +thread 'tests::queue_construction_budget' (404510074) panicked at crates/disktree-app/src/tests.rs:2181:5: +8x marks: 16384 -> 1048576 +FAILED + +failures: + +failures: + tests::blocked_review_status_budget + tests::covered_review_status_budget + tests::parent_absorption_budget + tests::queue_construction_budget + +test result: FAILED. 5 passed; 4 failed; 0 ignored; 0 measured; 74 filtered out; finished in 3.81s + +error: test failed, to rerun pass `-p disktree-app --bin disktree` + +EXIT 101 +REPEAT 3: sources at red commit 57d4efd + Finished `test` profile [optimized + debuginfo] target(s) in 0.26s +warning: the following packages contain code that will be rejected by a future version of Rust: block v0.1.6 +note: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1` + Running unittests src/main.rs (target/debug/deps/disktree-be72e126ba0d25ca) + +running 9 tests +test argument_tests::adaptive_budget_and_fixed_override_are_explicit ... ok +test argument_tests::invalid_worker_budgets_are_rejected ... ok +test marks::tests::refresh_dimension_budget ... dimensions folders=128 M=8 D=2 R=1 work=160 wall=19.625µs +dimensions folders=1024 M=8 D=2 R=1 work=1056 wall=44.666µs +dimensions folders=1024 M=64 D=2 R=1 work=1280 wall=52.459µs +dimensions folders=128 M=8 D=16 R=1 work=384 wall=24.583µs +dimensions folders=128 M=8 D=2 R=8 work=1280 wall=61.791µs +ok +test marks::tests::refresh_resolution_budget ... refresh M=128 N=256 R=4 work=2560 wall=199.917µs +refresh M=1024 N=2048 R=4 work=20480 wall=1.529833ms +ok +test tests::blocked_review_status_budget ... review-status covered=false M=128 work=66048 renders=8 rows=128 per-render=8256 wall=108.861834ms +review-status covered=false M=1024 work=4198400 renders=8 rows=1024 per-render=524800 wall=999.468209ms + +thread 'tests::blocked_review_status_budget' (404510132) panicked at crates/disktree-app/src/tests.rs:2298:5: +8x status rows: 66048/8 -> 4198400/8 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED +test tests::covered_review_status_budget ... review-status covered=true M=128 work=67072 renders=8 rows=129 per-render=8384 wall=124.821916ms +review-status covered=true M=1024 work=4206592 renders=8 rows=1025 per-render=525824 wall=1.110591625s + +thread 'tests::covered_review_status_budget' (404510197) panicked at crates/disktree-app/src/tests.rs:2298:5: +8x status rows: 67072/8 -> 4206592/8 +FAILED +test tests::marked_window_resolution_budget ... warm window M=128 prepares=8 resolution=216 tiles=1544 wall=59.989667ms +window M=128 N=256 draws=4 prepares=8 work=856 wall=61.309958ms +warm window M=1024 prepares=8 resolution=216 tiles=776 wall=341.035208ms +window M=1024 N=2048 draws=4 prepares=8 work=5336 wall=332.707042ms +ok +test tests::parent_absorption_budget ... parent-absorption M=128 work=8771 compactions=8512 wall=336µs +parent-absorption M=1024 work=528899 compactions=526848 wall=15.915917ms + +thread 'tests::parent_absorption_budget' (404510244) panicked at crates/disktree-app/src/tests.rs:2223:5: +8x descendants: 8771 -> 528899 +FAILED +test tests::queue_construction_budget ... queue-construction M=128 work=16384 wall=2.838333ms +queue-construction M=1024 work=1048576 wall=175.767541ms + +thread 'tests::queue_construction_budget' (404510255) panicked at crates/disktree-app/src/tests.rs:2181:5: +8x marks: 16384 -> 1048576 +FAILED + +failures: + +failures: + tests::blocked_review_status_budget + tests::covered_review_status_budget + tests::parent_absorption_budget + tests::queue_construction_budget + +test result: FAILED. 5 passed; 4 failed; 0 ignored; 0 measured; 74 filtered out; finished in 3.72s + +error: test failed, to rerun pass `-p disktree-app --bin disktree` + +EXIT 101 diff --git a/docs/performance/mark-resolution/interactions.md b/docs/performance/mark-resolution/interactions.md index 1eeefec9..427e1b78 100644 --- a/docs/performance/mark-resolution/interactions.md +++ b/docs/performance/mark-resolution/interactions.md @@ -1,5 +1,9 @@ # What is a mark, and when does the delay happen? +This file preserves the first set of input measurements at the original +PR #9 fix. For the final combined scope, current validation and freshly +replayed main-versus-final inputs, see [expanded evidence](expanded/README.md). + A **mark** is one file or directory queued for removal. Marking a directory marks that path once, regardless of how many files it contains. For example, marking a cache directory containing a million files is M=1, not M=1,000,000. From e8c098e426a0ceefb3147812a8986f58fe5685d6 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 16:10:36 -0400 Subject: [PATCH 16/19] docs: clarify the blocked and overlapping review fixtures --- docs/performance/mark-resolution/expanded/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/performance/mark-resolution/expanded/README.md b/docs/performance/mark-resolution/expanded/README.md index 99750103..94c8916e 100644 --- a/docs/performance/mark-resolution/expanded/README.md +++ b/docs/performance/mark-resolution/expanded/README.md @@ -61,6 +61,11 @@ that exactly 1,200 rows are built per render. | Blocked Review status, per render | 8,256 / 524,800 | 384 / 3,072 | | Covered Review status, per render | 8,384 / 525,824 | 386 / 3,074 | +The blocked fixture queues paths outside the scanned root. The covered +fixture supplies a raw overlapping parent/children queue; normal manual +marking would absorb those overlaps. Both exercise status decoration through +planning and rendering only, with no removal attempt. + At 8,192 items, both Review cases use 10,592 status units per render, while keeping the 1,200-row cap. See [green.log](green.log). The tests drive `Disktree::toggle_mark` or actual GPUI window draws, not an isolated index From b0e48fbd1d6a559ccf3c36829e73e214b450eefb Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Wed, 14 Oct 2026 16:14:59 -0400 Subject: [PATCH 17/19] docs: record expanded queue fix integration and validation --- docs/performance/apfs-accounting/README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/performance/apfs-accounting/README.md b/docs/performance/apfs-accounting/README.md index dbc97f92..dcb8fac8 100644 --- a/docs/performance/apfs-accounting/README.md +++ b/docs/performance/apfs-accounting/README.md @@ -55,6 +55,15 @@ and eight prepares, not a scan or one click. separates Space, hover and Review, includes smaller queues, and records render/plan counts with zero scan refreshes. Its timings compare the isolated main-only snapshots, not this APFS feature branch. +The [expanded queue fixes](../mark-resolution/expanded/README.md) are also +integrated: an ordered mark index replaces repeated ancestor/descendant scans, +parent absorption compacts once, and Review builds status indexes from each +fresh plan. Disjoint construction is O(M D log M), absorption is +O(M + K log M) plus ancestor lookup, and row-status work is expected +O(B + C + L), with L <= 1,200. Exact membership now costs O(log M). +The saved main-only audit remains historical; its depth, mount and unusual +classification cases remain deferred. + Removal authorization remains fresh on every plan and in the worker. Selected subtrees merge bounded maps small to large: expected @@ -108,8 +117,11 @@ cargo test -p disktree-core macos::clone_tests -- --nocapture cargo run --release -p disktree-core --example scan_accounting -- PATH ``` -Local validation: lint passed; 79 window/app, 174 core and 3 xtask tests passed; +Local validation: lint passed; 89 window/app, 174 core and 3 xtask tests passed; two existing core tests and the opt-in interaction timing test remained ignored in the normal suite. The interaction timing test was also run -explicitly and passed on the combined branch. No private file contents were +explicitly and passed on the combined branch. The latest optional replay +passed on retry after GPUI reported filesystem-watcher thread activity as +nondeterministic in the first attempt; its local timings are not used for +the isolated main-only comparisons. No private file contents were inspected and no user files were removed by this investigation. From f747d1287570b18f483639741513f1c4faa6d952 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Fri, 23 Oct 2026 20:30:07 -0400 Subject: [PATCH 18/19] Use a native absolute path in remote acceptance fixture Keep the remote-root test independent of host path syntax. The fixture stays nonexistent locally so acceptance still proves local directory validation is skipped on Windows as well as Unix. --- crates/disktree-app/src/tests.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index 8aa56c3c..8e7919fa 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -434,6 +434,10 @@ fn remote_root_acceptance_never_stats_the_path_locally( ) { cx.update(init); let temp = fixture(); + // Use a native absolute path so Windows reaches the remote acceptance + // branch too. Its absence proves local directory validation is skipped. + let remote_root = temp.path().join("only-on-the-remote-host"); + assert!(remote_root.is_absolute() && !remote_root.exists()); let (view, cx) = view_over(temp.path(), cx); update(&view, cx, |app, cx| { let crumbs = app.crumbs_for_path(&temp.path().join("junk")).unwrap(); @@ -446,7 +450,7 @@ fn remote_root_acceptance_never_stats_the_path_locally( app.finish_volume_listing( Ok(( vec![Volume { - point: PathBuf::from("/only-on-the-remote-host"), + point: remote_root.clone(), device: Some("/dev/remote-disk".into()), space: None, }], @@ -459,7 +463,8 @@ fn remote_root_acceptance_never_stats_the_path_locally( update(&view, cx, |app, cx| { app.choose_volume(cx); assert_eq!(app.remote.as_deref(), Some("-test-fixture")); - assert_eq!(app.root_path, PathBuf::from("/only-on-the-remote-host")); + assert_eq!(app.root_path, remote_root); + assert!(app.picker_error.is_none()); assert!(app.marks.is_empty() && app.remote_identity.is_none()); assert!( app.tree.is_none() From 87c02c7bf1f253e081ff7d8996e8c21a4a496a21 Mon Sep 17 00:00:00 2001 From: Harold Hunt Date: Fri, 23 Oct 2026 20:53:49 -0400 Subject: [PATCH 19/19] Match remote preview test to helper platform support Keep final hardlink totals and received progress counters checked on every platform. Require live preview records from the Unix streaming walker while asserting their absence from the Windows completed-tree fallback. --- crates/disktree-core/src/remote.rs | 31 ++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/crates/disktree-core/src/remote.rs b/crates/disktree-core/src/remote.rs index a6299b71..a53ccbfb 100644 --- a/crates/disktree-core/src/remote.rs +++ b/crates/disktree-core/src/remote.rs @@ -1210,18 +1210,25 @@ mod tests { .unwrap(); assert_eq!(snapshot.tree.bytes, if dedup { 10 } else { 15 }); assert_eq!(snapshot.tree.files, 3); - let preview = shared.lock().unwrap().preview.clone().unwrap(); - let nested = preview - .entries - .iter() - .find(|entry| entry.name == "nested") - .unwrap(); - assert_eq!(nested.bytes, 5); - assert_eq!(nested.files, 1); - assert_eq!( - preview.entries.iter().map(|e| e.bytes).sum::(), - snapshot.tree.bytes - ); + let progress = shared.lock().unwrap(); + assert_eq!(progress.bytes, snapshot.tree.bytes); + assert_eq!(progress.files, snapshot.tree.files); + // Only the Unix streaming walker emits live-tree records; the + // Windows fallback sends the completed tree without a preview. + assert_eq!(progress.preview.is_some(), cfg!(unix)); + if let Some(preview) = &progress.preview { + let nested = preview + .entries + .iter() + .find(|entry| entry.name == "nested") + .unwrap(); + assert_eq!(nested.bytes, 5); + assert_eq!(nested.files, 1); + assert_eq!( + preview.entries.iter().map(|e| e.bytes).sum::(), + snapshot.tree.bytes + ); + } } }