From 15eb91224375b1dd0a8d2ad389dc013b1628063c Mon Sep 17 00:00:00 2001 From: Matt Date: Fri, 6 Nov 2026 07:44:54 -0500 Subject: [PATCH] Add Btrfs extent accounting and conservative reclaim estimates --- .github/workflows/ci.yml | 18 +- AGENTS.md | 9 +- README.md | 48 ++- crates/disktree-app/src/state.rs | 6 +- crates/disktree-app/src/tests.rs | 44 ++ crates/disktree-app/src/views.rs | 51 ++- crates/disktree-core/src/btrfs.rs | 599 ++++++++++++++++++++++++++++ crates/disktree-core/src/lib.rs | 2 + crates/disktree-core/src/removal.rs | 69 +++- crates/disktree-core/src/scan.rs | 23 +- crates/disktree-core/src/tree.rs | 30 +- 11 files changed, 885 insertions(+), 14 deletions(-) create mode 100644 crates/disktree-core/src/btrfs.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48079012..142b4053 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: sudo apt-get install -y --no-install-recommends \ pkg-config libxkbcommon-dev libxkbcommon-x11-dev libwayland-dev \ libx11-xcb-dev libxcb1-dev libfontconfig-dev libfreetype-dev \ - libvulkan-dev libasound2-dev libssl-dev libzstd-dev + libvulkan-dev libasound2-dev libssl-dev libzstd-dev btrfs-progs # Pinned: the lint gate denies clippy::all and ::pedantic, and a newer # clippy's new lints are a change to make on purpose, not by surprise. @@ -48,6 +48,22 @@ jobs: - name: Test run: cargo xtask test + # Extent sharing cannot be tested on the runner's ordinary filesystem. + # Only this disposable image is formatted; no host device is touched. + - name: Btrfs integration + run: | + btrfs_image="$RUNNER_TEMP/disktree-btrfs.img" + btrfs_fixture="$RUNNER_TEMP/disktree-btrfs" + truncate -s 512M "$btrfs_image" + mkfs.btrfs -f "$btrfs_image" + mkdir -p "$btrfs_fixture" + sudo mount -o loop,compress=zstd "$btrfs_image" "$btrfs_fixture" + trap 'sudo umount "$btrfs_fixture"' EXIT + sudo chown "$(id -u):$(id -g)" "$btrfs_fixture" + export DISKTREE_BTRFS_TEST_DIR="$btrfs_fixture" + cargo test -p disktree-core btrfs + cargo test -p disktree-app btrfs_review + check-macos: name: Lint and test (macOS) # Apple silicon. No system libraries to install, and no Metal toolchain diff --git a/AGENTS.md b/AGENTS.md index 0342b30c..5e9df6fb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,8 +62,13 @@ and `cargo build --release` directly; CI runs the gate on both systems. ## Invariants -1. **Sizes come from `st_blocks * 512` unless apparent size was asked for.** - That is the number that comes back when a file is deleted. On macOS the +1. **Sizes use allocated bytes unless apparent size was asked for.** + Local Btrfs scans deduplicate extents in `btrfs.rs`; compressed lengths + and measurement fallbacks are explicitly estimated. Other Unix scans + use `st_blocks * 512`. Neither number guarantees reclaimed space: + Btrfs keeps a separate conservative estimate in `Node::reclaimable`. + Shared extents, hardlinks, compressed and unknown data are excluded. + Trash projects no immediate saving. On macOS the directory listing reports it and `st_size` for a whole buffer of entries: every fork's allocation, and the data fork's length; see `macos.rs`. On Windows it is the allocation the directory listing reports; see diff --git a/README.md b/README.md index bf9bc21f..a2a9f1cc 100644 --- a/README.md +++ b/README.md @@ -399,9 +399,10 @@ escaped in the prompt, so it cannot pass for another path. ## What it measures -- **Disk usage** by default: `st_blocks × 512`, the number `du` reports and the - space that actually comes back when a file is deleted. Apparent size (what - `ls -l` shows) is one toggle away. +- **Disk usage** by default: allocated bytes, with extent-aware accounting + for local Btrfs scans (see below). Other Unix filesystems use + `st_blocks × 512`, like `du`. Allocation is not a guarantee of space freed + by deletion. Apparent size (`ls -l`) is one toggle away. - **Hardlinks once.** Two names for one inode cost one file. - **Hidden entries included**, because `~/.cache` is often the biggest thing in a home directory. Symlinks are not followed. @@ -419,6 +420,47 @@ scope per root, a completion counter per directory so no directory is built before its last subdirectory lands, and one bottom-up pass that aggregates sizes and removes duplicate hardlinks. +## Btrfs accounting on Linux + +Local Btrfs scans inspect file extents without root privileges. Reflinked +copies count once across the scanned tree, including partial overlaps and +copies in different subvolumes of the same filesystem. Shared data is +assigned to the first path in name order, so a duplicate tile can be zero; +that is its contribution to this scan, not the file's apparent length. +Scanning that directory alone can therefore give a larger contribution. +The **Apparent size** toggle still shows file lengths. + +**Size and reclaimable space are different measurements.** The removal +preview uses a conservative estimate of exclusive, uncompressed data, not +selected tile sizes. It excludes shared extents (including references in +snapshots or outside the scan), hardlinked files, compressed data and files +that cannot be measured. Selecting every visible copy does not establish +that no other reference exists: shared data remains excluded. A zero +estimate means no exclusive data was verified, not that deletion can never +free space. Trash shows no immediate saving; emptying it is a separate step. + +The kernel's unprivileged FIEMAP interface does not report compressed +physical lengths. Compressed extents are counted once by their extent +address using a **logical-size estimate**, visibly labelled in the scan +summary. Inline or unsupported mappings, inaccessible files and files +changed during measurement retain their ordinary size with the same +estimate label. These totals are not exact physical disk usage and can +still exceed the volume's used space. The volume meter always uses +`statvfs`; metadata, replicas, snapshots and other files also affect it. + +Accounting is a cancellable second pass after indexing. Btrfs scans are +remeasured when widening the root; extent addresses are never cached across +scans. Estimates describe scan-time data and can change after writes or +new snapshots, so rescan before acting on old results. Remote SSH scans +currently retain their existing allocated-byte accounting. + +To exercise real reflinks on a writable Btrfs filesystem (temporary files +only, no mounts or elevated access): + +```sh +DISKTREE_BTRFS_TEST_DIR=/path/on/btrfs cargo test -p disktree-core btrfs +``` + ## Choosing sources and roots Launch without a path to choose **This computer** or enter an SSH alias / diff --git a/crates/disktree-app/src/state.rs b/crates/disktree-app/src/state.rs index efb9730a..987884a2 100644 --- a/crates/disktree-app/src/state.rs +++ b/crates/disktree-app/src/state.rs @@ -2210,7 +2210,11 @@ impl Disktree { // this computer's mounts or project an unmeasured saving. return selection; } - plan(self.marks.items(), &self.root_path) + let mut result = plan(self.marks.items(), &self.root_path); + if let Some(tree) = self.tree() { + result.account_storage(tree, self.removal_mode); + } + result } // ── layout and hit-testing ────────────────────────────────────────── diff --git a/crates/disktree-app/src/tests.rs b/crates/disktree-app/src/tests.rs index 0520e4cb..61dec92a 100644 --- a/crates/disktree-app/src/tests.rs +++ b/crates/disktree-app/src/tests.rs @@ -2357,3 +2357,47 @@ fn the_power_menu_offers_only_what_the_cpus_can_tell_apart( draw(cx); assert!(read(&view, cx, |app| app.power_menu.is_none())); } + +/// The same real-Btrfs opt-in as the core fixtures, rendered through the UI. +#[gpui_kit::test] +#[cfg(target_os = "linux")] +fn btrfs_review_uses_reclaim_estimates_in_apparent_mode( + cx: &mut TestAppContext, +) { + let Some(parent) = std::env::var_os("DISKTREE_BTRFS_TEST_DIR") else { + return; + }; + cx.update(gpui_omarchy::init); + let temp = tempfile::tempdir_in(parent).unwrap(); + let first = temp.path().join("first"); + std::fs::write(&first, vec![b'x'; 1024 * 1024]).unwrap(); + std::fs::File::open(&first).unwrap().sync_all().unwrap(); + assert!( + std::process::Command::new("cp") + .arg("--reflink=always") + .arg(&first) + .arg(temp.path().join("second")) + .status() + .unwrap() + .success() + ); + let (view, cx) = view_over(temp.path(), cx); + update(&view, cx, |app, cx| { + let crumbs = app.crumbs_for_path(&first).unwrap(); + app.select(Some(crumbs), cx); + app.toggle_mark_selected(cx); + app.removal_mode = RemovalMode::Permanent; + assert_eq!(app.plan().bytes(), 1024 * 1024); + assert_eq!(app.plan().reclaim(), 0); + assert!(app.plan().btrfs_estimate); + app.screen = Screen::Review; + }); + draw(cx); + update(&view, cx, |app, _| { + app.removal_mode = RemovalMode::Trash; + assert_eq!(app.plan().reclaim(), 0); + assert!(app.plan().deferred_to_trash); + }); + draw(cx); + assert!(first.exists(), "preview never deletes the selection"); +} diff --git a/crates/disktree-app/src/views.rs b/crates/disktree-app/src/views.rs index 32aedf5f..366730dc 100644 --- a/crates/disktree-app/src/views.rs +++ b/crates/disktree-app/src/views.rs @@ -1365,6 +1365,18 @@ fn scan_totals(app: &Disktree, theme: &Theme) -> Div { .text_size(text::CAPTION) .text_color(theme.secondary) .child(div().text_color(theme.foreground).child(human_bytes(bytes))) + .when( + tree.is_some_and(|node| node.reclaimable().is_some()), + |this| { + this.child(if tree.is_some_and(|node| node.size_estimated) { + "· Btrfs estimated size" + } else if app.options.apparent_size { + "· apparent size" + } else { + "· Btrfs data (shared blocks counted once)" + }) + }, + ) .child(format!( "· {} files · {} dirs", widgets::human_count(files), @@ -1584,7 +1596,25 @@ fn selection_section( text::TITLE, cx, )) - .child(widgets::bar(share, highlight, cx)); + .child(widgets::bar(share, highlight, cx)) + .children(node.reclaimable().map(|bytes| { + widgets::row( + "Conservative reclaim estimate", + human_bytes(bytes), + cx, + ) + })) + .when(node.reclaimable().is_some(), |this| { + this.child( + div() + .text_size(text::CAPTION) + .text_color(theme.secondary) + .child( + "Size is this path’s contribution to the scan. \ + Reclaim excludes shared, compressed and unknown data.", + ), + ) + }); let fourth = if app.remote.is_none() && node.is_dir() @@ -2293,6 +2323,16 @@ fn disk_section( /// this volume's, which is the usual case. fn not_counted(plan: &Plan) -> Option { let mut parts: Vec = Vec::new(); + if plan.btrfs_estimate { + parts.push( + "Conservative Btrfs estimate: shared, compressed and \ + unknown data excluded; rescan after changes" + .into(), + ); + } + if plan.deferred_to_trash { + parts.push("Space is released only after the trash is emptied".into()); + } if plan.foreign() > 0 { parts.push(format!("{} on other volumes", human_bytes(plan.foreign()))); } @@ -2584,6 +2624,8 @@ fn indexing_progress( .child(div().h(size::PROGRESS_ROW).truncate().child( if app.remote.is_some() { app.remote_progress.stage.label() + } else if app.progress.accounting { + "Measuring shared disk space (sizes are provisional)" } else { "Reading this computer" }, @@ -3042,10 +3084,15 @@ fn review_summary( cx, )) .child(widgets::row( - "Space freed", + "Selected size", human_bytes(plan.bytes()), cx, )) + .child(widgets::row( + "Estimated freed here", + human_bytes(plan.reclaim()), + cx, + )) .children(left_out), ); diff --git a/crates/disktree-core/src/btrfs.rs b/crates/disktree-core/src/btrfs.rs new file mode 100644 index 00000000..f32aae45 --- /dev/null +++ b/crates/disktree-core/src/btrfs.rs @@ -0,0 +1,599 @@ +//! Unprivileged Btrfs accounting. FIEMAP identifies reflinked data, including +//! references outside the scan, without mounting anything or reading contents. +//! +//! Encoded mappings expose their logical length, not their compressed length. +//! They are charged once by extent start as an estimate, never as reclaimable +//! bytes. Shared extents are likewise excluded from the conservative deletion +//! estimate: FIEMAP cannot prove that a selection contains every reference. + +#![allow( + unsafe_code, + reason = "FIEMAP and FS_INFO have no rustix wrapper; each ioctl documents \ + its buffer and ABI" +)] + +use std::collections::BTreeMap; +use std::fs::{File, OpenOptions}; +use std::io; +use std::os::fd::AsRawFd as _; +use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; +use std::path::Path; + +use rustc_hash::FxHashMap; + +use crate::scan::ScanProgress; +use crate::tree::{Node, NodeKind}; + +const BTRFS_MAGIC: u64 = 0x9123_683e; +const LAST: u32 = 1; +const ENCODED: u32 = 8; +const SHARED: u32 = 0x2000; +const UNWRITTEN: u32 = 0x800; +const BATCH: usize = 128; +// Unknown, delayed, inline, tail-packed, merged and future extent flags must +// not be interpreted as ordinary physical ranges. +const KNOWN: u32 = LAST | ENCODED | SHARED | UNWRITTEN; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct Extent { + logical: u64, + physical: u64, + length: u64, + reserved64: [u64; 2], + flags: u32, + reserved: [u32; 3], +} + +#[repr(C)] +struct Map { + start: u64, + length: u64, + flags: u32, + mapped: u32, + count: u32, + reserved: u32, + extents: [Extent; BATCH], +} + +unsafe extern "C" { + fn ioctl( + fd: std::ffi::c_int, + request: std::ffi::c_ulong, + ... + ) -> std::ffi::c_int; +} + +/// A disjoint union. Inserting a partially overlapping reflink charges only +/// the new part, even when the files were split into different extents. +#[derive(Default)] +struct Ranges(BTreeMap); + +impl Ranges { + fn insert(&mut self, start: u64, end: u64) -> u64 { + let mut first = start; + let mut last = end; + let mut added = end - start; + if let Some((&left, &right)) = self.0.range(..=start).next_back() + && right >= start + { + added -= right.min(end).saturating_sub(start); + first = left; + last = last.max(right); + self.0.remove(&left); + } + while let Some((&left, &right)) = self.0.range(first..=last).next() { + added -= right.min(end).saturating_sub(left.max(start)); + last = last.max(right); + self.0.remove(&left); + } + self.0.insert(first, last); + added + } +} + +#[derive(Default)] +struct Volume { + plain: Ranges, + encoded: FxHashMap, +} + +impl Volume { + fn charge(&mut self, extents: &[Extent], links: u64) -> (u64, u64, bool) { + let mut bytes = 0_u64; + let mut reclaimable = 0_u64; + let mut estimated = false; + for e in extents { + if e.flags & ENCODED != 0 { + // Encoded logical lengths are NOT physical address ranges. + // Different compressed extents can appear to overlap. Only + // equal starts identify the same compressed extent. + let seen = self.encoded.entry(e.physical).or_default(); + bytes = bytes.saturating_add(e.length.saturating_sub(*seen)); + *seen = (*seen).max(e.length); + estimated = true; + } else { + let added = + self.plain.insert(e.physical, e.physical + e.length); + bytes = bytes.saturating_add(added); + if e.flags & SHARED == 0 && links == 1 { + reclaimable = reclaimable.saturating_add(added); + } + } + } + (bytes, reclaimable, estimated) + } +} + +#[derive(Default)] +struct Accountant { + // FS_INFO's UUID, not st_dev: different Btrfs subvolumes share extents. + volumes: FxHashMap<[u8; 16], Volume>, + devices: FxHashMap>, +} + +/// Re-measure completed local leaves before their totals are aggregated. +/// Sorting by name makes allocation to the first referencing file stable. +/// A rescan constructs a new accountant; no physical addresses outlive it. +pub fn measure( + root: &Path, + node: &mut Node, + progress: &ScanProgress, + options: &crate::scan::ScanOptions, +) -> io::Result<()> { + let btrfs = rustix::fs::statfs(root)?.f_type as u64 == BTRFS_MAGIC; + if options.one_filesystem && !btrfs { + return Ok(()); + } + progress + .accounting + .store(true, std::sync::atomic::Ordering::Relaxed); + Accountant::default().visit( + root, + node, + progress, + btrfs, + options.apparent_size, + ) +} + +impl Accountant { + fn volume( + &mut self, + file: &File, + device: u64, + ) -> io::Result> { + if let Some(id) = self.devices.get(&device) { + return Ok(*id); + } + let stat = rustix::fs::fstatfs(file)?; + let id = if stat.f_type as u64 == BTRFS_MAGIC { + // FS_INFO is a fixed 1024-byte UAPI buffer; UUID is bytes 16..32. + let mut info = [0_u64; 128]; + // SAFETY: the kernel writes at most 1024 aligned, initialized + // bytes into this fixed-size buffer. + if unsafe { + ioctl( + file.as_raw_fd(), + rustix::ioctl::opcode::read::<[u64; 128]>(0x94, 31).into(), + info.as_mut_ptr(), + ) + } < 0 + { + return Err(io::Error::last_os_error()); + } + let mut id = [0; 16]; + id[..8].copy_from_slice(&info[2].to_ne_bytes()); + id[8..].copy_from_slice(&info[3].to_ne_bytes()); + Some(id) + } else { + None + }; + self.devices.insert(device, id); + Ok(id) + } + + fn visit( + &mut self, + path: &Path, + node: &mut Node, + progress: &ScanProgress, + inherited: bool, + apparent: bool, + ) -> io::Result<()> { + if progress.is_cancelled() { + return Err(io::Error::new( + io::ErrorKind::Interrupted, + "Btrfs accounting cancelled", + )); + } + // Nodes use lossy UTF-8 names for display. A replacement character + // may stand for a different on-disk name; do not measure its alias. + if node.name.contains('\u{fffd}') { + if inherited { + unknown(node, apparent); + } + return Ok(()); + } + if node.is_dir() { + // Do not follow a directory replaced with a symlink since the walk. + let volume = open(path) + .and_then(|file| self.volume(&file, file.metadata()?.dev())); + if volume.is_err() { + if inherited { + unknown(node, apparent); + } + return Ok(()); + } + let btrfs = matches!(volume, Ok(Some(_))); + if btrfs { + node.set_reclaimable(0); + } + node.children.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + for child in &mut node.children { + self.visit( + &path.join(&*child.name), + child, + progress, + btrfs, + apparent, + )?; + } + return Ok(()); + } + if node.kind != NodeKind::File { + if inherited { + node.set_reclaimable(0); + } + return Ok(()); + } + // Check the parent first: non-Btrfs files need no open or extent query. + if !inherited { + return Ok(()); + } + // On any failure retain the ordinary size, explicitly estimated, and + // promise no saving. Opening nonblocking avoids a replaced FIFO hang. + node.set_reclaimable(0); + node.size_estimated = !apparent; + let Ok(file) = open(path) else { + return Ok(()); + }; + let Ok(before) = file.metadata() else { + return Ok(()); + }; + if !before.is_file() { + return Ok(()); + } + let Ok(Some(id)) = self.volume(&file, before.dev()) else { + return Ok(()); + }; + let Ok(extents) = mappings(&file, progress) else { + if progress.is_cancelled() { + return Err(io::Error::new( + io::ErrorKind::Interrupted, + "Btrfs accounting cancelled", + )); + } + return Ok(()); + }; + let Ok(after) = file.metadata() else { + return Ok(()); + }; + if before.len() != after.len() + || before.mtime_nsec() != after.mtime_nsec() + || before.mtime() != after.mtime() + || before.ctime() != after.ctime() + || before.ctime_nsec() != after.ctime_nsec() + { + return Ok(()); + } + // An unsupported mapping invalidates the whole file, before committing + // any ranges: otherwise a failed file could hide bytes in later files. + if extents.iter().any(|e| { + e.flags & !KNOWN != 0 + || e.physical == 0 + || e.length == 0 + || e.physical.checked_add(e.length).is_none() + }) { + return Ok(()); + } + let volume = self.volumes.entry(id).or_default(); + let (bytes, reclaimable, estimated) = + volume.charge(&extents, before.nlink()); + if !apparent { + node.bytes = bytes; + } + node.set_reclaimable(reclaimable); + node.size_estimated = estimated && !apparent; + Ok(()) + } +} + +fn unknown(node: &mut Node, apparent: bool) { + node.set_reclaimable(0); + node.size_estimated = !apparent; + for child in &mut node.children { + unknown(child, apparent); + } +} + +fn open(path: &Path) -> io::Result { + OpenOptions::new() + .read(true) + .custom_flags( + i32::try_from( + (rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK) + .bits(), + ) + .expect("open flags fit a C int"), + ) + .open(path) +} + +fn mappings(file: &File, progress: &ScanProgress) -> io::Result> { + let mut result = Vec::new(); + let mut start = 0; + loop { + if progress.is_cancelled() { + return Err(io::Error::new( + io::ErrorKind::Interrupted, + "cancelled", + )); + } + let mut map = Map { + start, + length: u64::MAX - start, + // Do not force writeback of the user's files just to measure them. + flags: 0, + mapped: 0, + count: BATCH as u32, + reserved: 0, + extents: [Extent::default(); BATCH], + }; + // SAFETY: repr(C) header is 32 bytes, followed by count 56-byte + // extents. The ioctl size is the header size, not the array size. + if unsafe { + ioctl( + file.as_raw_fd(), + rustix::ioctl::opcode::read_write::<[u64; 4]>(0x66, 11).into(), + &raw mut map, + ) + } < 0 + { + return Err(io::Error::last_os_error()); + } + let count = map.mapped as usize; + if count > BATCH { + return Err(io::Error::other("invalid FIEMAP count")); + } + let Some(last) = map.extents[..count].last() else { + return Ok(result); + }; + let next = last + .logical + .checked_add(last.length) + .filter(|next| *next > start) + .ok_or_else(|| io::Error::other("FIEMAP did not advance"))?; + if result.len() > 1_000_000 { + return Err(io::Error::other("FIEMAP extent limit exceeded")); + } + result.extend_from_slice(&map.extents[..count]); + if last.flags & LAST != 0 { + return Ok(result); + } + start = next; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ioctl_layout_matches_linux_uapi() { + assert_eq!(size_of::(), 56); + assert_eq!(std::mem::offset_of!(Map, extents), 32); + assert_eq!(size_of::(), 32 + BATCH * 56); + } + + #[test] + fn partial_reflinks_and_bridging_ranges_are_counted_once() { + let mut ranges = Ranges::default(); + assert_eq!(ranges.insert(100, 200), 100); + assert_eq!(ranges.insert(150, 250), 50); + assert_eq!(ranges.insert(300, 400), 100); + assert_eq!(ranges.insert(50, 450), 150); + assert_eq!(ranges.insert(100, 200), 0); + assert_eq!(ranges.0.into_iter().collect::>(), vec![(50, 450)]); + } + + fn extent(physical: u64, length: u64, flags: u32) -> Extent { + Extent { + physical, + length, + flags, + ..Extent::default() + } + } + + #[test] + fn encoded_extents_do_not_overlap_by_their_logical_lengths() { + let mut volume = Volume::default(); + assert_eq!( + volume.charge(&[extent(100, 1000, ENCODED)], 1), + (1000, 0, true) + ); + assert_eq!( + volume.charge(&[extent(200, 1000, ENCODED)], 1), + (1000, 0, true) + ); + assert_eq!( + volume.charge(&[extent(100, 1000, ENCODED)], 1), + (0, 0, true) + ); + assert_eq!( + volume.charge(&[extent(100, 1500, ENCODED)], 1), + (500, 0, true) + ); + } + + #[test] + fn exclusive_data_is_separate_from_shared_and_hardlinked_data() { + let mut volume = Volume::default(); + let extents = [extent(100, 100, SHARED), extent(200, 100, 0)]; + assert_eq!(volume.charge(&extents, 1), (200, 100, false)); + assert_eq!(Volume::default().charge(&extents, 2), (200, 0, false)); + // Another filesystem's identical addresses must not be deduplicated. + assert_eq!(Volume::default().charge(&extents, 1), (200, 100, false)); + } + + #[test] + fn cancelled_accounting_does_not_publish_a_partial_result() { + let temp = tempfile::tempdir().unwrap(); + let progress = ScanProgress::default(); + progress.cancel(); + let mut node = Node::directory("root"); + let error = Accountant::default() + .visit(temp.path(), &mut node, &progress, true, false) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::Interrupted); + } + + #[test] + fn live_compression_is_labelled_and_sparse_holes_are_not_charged() { + let Some(parent) = std::env::var_os("DISKTREE_BTRFS_TEST_DIR") else { + return; + }; + let temp = tempfile::tempdir_in(parent).unwrap(); + let compressed = temp.path().join("compressed"); + std::fs::write(&compressed, vec![b'x'; 1024 * 1024]).unwrap(); + File::open(&compressed).unwrap().sync_all().unwrap(); + let sparse = File::create(temp.path().join("sparse")).unwrap(); + sparse.set_len(32 * 1024 * 1024).unwrap(); + sparse.sync_all().unwrap(); + let tree = + crate::scan::scan(temp.path(), crate::scan::ScanOptions::default()) + .unwrap(); + assert_eq!(tree.child_named("sparse").unwrap().bytes, 0); + let extents = mappings( + &File::open(compressed).unwrap(), + &ScanProgress::default(), + ) + .unwrap(); + if extents.iter().any(|extent| extent.flags & ENCODED != 0) { + assert!(tree.size_estimated); + assert_eq!(tree.reclaimable(), Some(0)); + } + } + + #[test] + fn live_unreadable_files_keep_estimated_size_without_a_saving() { + use std::os::unix::fs::PermissionsExt as _; + let Some(parent) = std::env::var_os("DISKTREE_BTRFS_TEST_DIR") else { + return; + }; + let temp = tempfile::tempdir_in(parent).unwrap(); + let file = temp.path().join("unreadable"); + std::fs::write(&file, vec![b'x'; 1024 * 1024]).unwrap(); + File::open(&file).unwrap().sync_all().unwrap(); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o000)) + .unwrap(); + // Running tests as root defeats the unreadable-file fixture. + if File::open(&file).is_ok() { + return; + } + let tree = + crate::scan::scan(temp.path(), crate::scan::ScanOptions::default()) + .unwrap(); + assert!(tree.bytes > 0); + assert!(tree.size_estimated); + assert_eq!(tree.reclaimable(), Some(0)); + } + + /// Set `DISKTREE_BTRFS_TEST_DIR` to a writable Btrfs directory. No mounts, + /// sudo, or changes to existing files are required. + #[test] + fn live_reflinks_count_once_and_do_not_promise_shared_space() { + let Some(parent) = std::env::var_os("DISKTREE_BTRFS_TEST_DIR") else { + return; + }; + let temp = tempfile::tempdir_in(parent).unwrap(); + let original = temp.path().join("a"); + // Deterministic noise prevents transparent compression in this fixture. + let mut seed = 42_u64; + let data: Vec = (0..1024 * 1024) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed as u8 + }) + .collect(); + std::fs::write(&original, data).unwrap(); + File::open(&original).unwrap().sync_all().unwrap(); + for name in ["b", "c"] { + assert!( + std::process::Command::new("cp") + .arg("--reflink=always") + .arg(&original) + .arg(temp.path().join(name)) + .status() + .unwrap() + .success() + ); + } + let tree = + crate::scan::scan(temp.path(), crate::scan::ScanOptions::default()) + .unwrap(); + assert_eq!(tree.bytes, 1024 * 1024); + assert_eq!(tree.reclaimable(), Some(0)); + assert!(!tree.size_estimated); + assert_eq!(tree.child_named("a").unwrap().bytes, 1024 * 1024); + assert_eq!(tree.child_named("b").unwrap().bytes, 0); + // Both apparent-size mode and partial selections keep sharing separate + // from the layout weights. An alias outside the scanned root still + // prevents an exclusive-space claim through FIEMAP_EXTENT_SHARED. + let apparent = crate::scan::scan( + temp.path(), + crate::scan::ScanOptions { + apparent_size: true, + ..crate::scan::ScanOptions::default() + }, + ) + .unwrap(); + assert_eq!(apparent.bytes, 3 * 1024 * 1024); + assert_eq!(apparent.reclaimable(), Some(0)); + let target = crate::removal::Target { + path: original.clone(), + bytes: tree.child_named("a").unwrap().bytes, + is_dir: false, + hidden: false, + }; + let mut plan = crate::removal::plan(&[target], temp.path()); + plan.account_storage(&tree, crate::removal::RemovalMode::Permanent); + assert_eq!(plan.reclaim(), 0); + assert!(plan.btrfs_estimate); + plan.account_storage(&tree, crate::removal::RemovalMode::Permanent); + assert_eq!(plan.reclaim(), 0, "projection is idempotent"); + + std::fs::remove_file(temp.path().join("b")).unwrap(); + std::fs::remove_file(temp.path().join("c")).unwrap(); + File::open(&original).unwrap().sync_all().unwrap(); + let tree = + crate::scan::scan(temp.path(), crate::scan::ScanOptions::default()) + .unwrap(); + assert_eq!(tree.bytes, 1024 * 1024); + assert_eq!(tree.reclaimable(), Some(1024 * 1024)); + let target = crate::removal::Target { + path: original, + bytes: tree.bytes, + is_dir: false, + hidden: false, + }; + let mut plan = crate::removal::plan(&[target], temp.path()); + plan.account_storage(&tree, crate::removal::RemovalMode::Trash); + assert_eq!(plan.reclaim(), 0); + assert_eq!(plan.unattributed(), 0); + assert!(plan.deferred_to_trash); + } +} diff --git a/crates/disktree-core/src/lib.rs b/crates/disktree-core/src/lib.rs index 0f58c7ba..f03fa7da 100644 --- a/crates/disktree-core/src/lib.rs +++ b/crates/disktree-core/src/lib.rs @@ -6,6 +6,8 @@ //! be built and tested without GPUI, a display, or a GPU. pub mod access; +#[cfg(target_os = "linux")] +mod btrfs; pub mod classify; pub mod deployment; pub mod export; diff --git a/crates/disktree-core/src/removal.rs b/crates/disktree-core/src/removal.rs index 60277529..71709ed7 100644 --- a/crates/disktree-core/src/removal.rs +++ b/crates/disktree-core/src/removal.rs @@ -82,6 +82,11 @@ pub struct Plan { reclaim: u64, /// Bytes the targets free on other volumes; see [`Self::foreign`]. foreign: u64, + /// Btrfs uses a conservative data-only estimate, not the tile weights. + pub btrfs_estimate: bool, + /// Moving files to trash does not immediately release their blocks. + pub deferred_to_trash: bool, + omitted: u64, } impl Plan { @@ -109,7 +114,69 @@ impl Plan { /// either, since a projection the meter cannot stand behind is worse /// than a smaller one. pub fn unattributed(&self) -> u64 { - self.bytes().saturating_sub(self.reclaim + self.foreign) + self.bytes().saturating_sub( + self.reclaim + .saturating_add(self.foreign) + .saturating_add(self.omitted), + ) + } + + /// Replace tile-based projections with the scan's independent estimates. + /// This is a lookup over the accepted targets, never an I/O scan per frame. + pub fn account_storage( + &mut self, + tree: &crate::tree::Node, + mode: RemovalMode, + ) { + let mounts = crate::space::mount_table(); + self.reclaim = 0; + self.foreign = 0; + self.omitted = 0; + self.btrfs_estimate = false; + self.deferred_to_trash = false; + for target in &self.targets { + let node = target.path.strip_prefix(&self.root).ok().and_then( + |relative| { + relative.components().try_fold(tree, |node, component| { + node.child_named( + &component.as_os_str().to_string_lossy(), + ) + }) + }, + ); + let estimate = + node.and_then(crate::tree::Node::reclaimable).or_else(|| { + (node.is_none() && tree.reclaimable().is_some()) + .then_some(0) + }); + self.btrfs_estimate |= estimate.is_some(); + let bytes = estimate.unwrap_or(target.bytes); + let omitted = target.bytes.saturating_sub(bytes); + match crate::space::attribution( + mounts.as_deref(), + &self.root, + &target.path, + ) { + crate::space::Attribution::Scanned => { + self.reclaim = self.reclaim.saturating_add(bytes); + self.omitted = self.omitted.saturating_add(omitted); + } + crate::space::Attribution::Other => { + self.foreign = self.foreign.saturating_add(bytes); + self.omitted = self.omitted.saturating_add(omitted); + } + crate::space::Attribution::Unknown => {} + } + } + if mode == RemovalMode::Trash { + self.deferred_to_trash = true; + self.omitted = self + .omitted + .saturating_add(self.reclaim) + .saturating_add(self.foreign); + self.reclaim = 0; + self.foreign = 0; + } } pub const fn is_empty(&self) -> bool { diff --git a/crates/disktree-core/src/scan.rs b/crates/disktree-core/src/scan.rs index 5ffe81a0..3a0a8109 100644 --- a/crates/disktree-core/src/scan.rs +++ b/crates/disktree-core/src/scan.rs @@ -39,8 +39,8 @@ const MAX_ERROR_DETAIL: usize = 50; #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ScanOptions { /// Measure apparent length instead of allocated blocks. Apparent size is - /// what `ls -l` shows; blocks are what the volume actually spends, which is - /// what a disk-space tool normally wants. + /// what `ls -l` shows. Local Btrfs scans separately measure extents and + /// conservative reclaim estimates; other Unix scans use allocated blocks. pub apparent_size: bool, /// Follow symlinks. Off by default: a home directory is full of links, and /// following them double-counts. @@ -93,6 +93,7 @@ pub struct ScanProgress { bytes: AtomicU64, errors: AtomicU64, finished: AtomicBool, + pub(crate) accounting: AtomicBool, cancelled: AtomicBool, messages: Mutex>, preview_root: OnceLock, @@ -112,6 +113,8 @@ pub struct ScanSnapshot { pub bytes: u64, pub errors: u64, pub finished: bool, + /// The local walk is complete and Btrfs extent accounting is running. + pub accounting: bool, pub cancelled: bool, /// Admitted directory jobs, not a machine-wide thread limit. pub threads: usize, @@ -197,6 +200,7 @@ impl ScanProgress { bytes: self.bytes.load(Ordering::Relaxed), errors: self.errors.load(Ordering::Relaxed), finished: self.finished.load(Ordering::Relaxed), + accounting: self.accounting.load(Ordering::Relaxed), cancelled: self.is_cancelled(), threads: self.threads.load(Ordering::Relaxed), threads_settled: self.threads_settled.load(Ordering::Relaxed), @@ -241,7 +245,7 @@ impl ScanHandle { ) -> Self { let progress = Arc::new(ScanProgress::default()); let context = Arc::new(WalkContext { - known, + known: known.filter(|known| known.tree.storage.is_none()), options, progress: Arc::clone(&progress), root_device: Mutex::new(None), @@ -834,6 +838,8 @@ impl PendingDir { name: partial.name, kind: NodeKind::Directory, bytes: 0, + storage: None, + size_estimated: false, files: 0, dirs: 1, inode: None, @@ -992,6 +998,17 @@ fn scan_on_pool( let node = node.ok_or_else(|| { io::Error::other(format!("{} produced no tree", root.display())) })?; + #[cfg(target_os = "linux")] + let node = { + let mut node = node; + crate::btrfs::measure( + root, + &mut node, + &context.progress, + &context.options, + )?; + node + }; Ok(finish_tree(node, &context.options, pool)) } diff --git a/crates/disktree-core/src/tree.rs b/crates/disktree-core/src/tree.rs index 4bbeda44..33e3b279 100644 --- a/crates/disktree-core/src/tree.rs +++ b/crates/disktree-core/src/tree.rs @@ -76,6 +76,11 @@ pub struct Node { /// Subtree total: direct contents plus every descendant. A file's own /// size, or nothing for a second name of a file already charged. pub bytes: u64, + /// Conservative Btrfs reclaim estimate. None uses ordinary accounting. + /// Shared, compressed, multiply linked and unreadable data is excluded. + pub(crate) storage: Option, + /// Some sizes are logical estimates because physical lengths are unknown. + pub size_estimated: bool, /// Files at or beneath this node; `1` for a file. pub files: u64, /// Directories at or beneath this node; `1` for a directory. Four @@ -109,6 +114,8 @@ impl Node { name: name.into(), kind: NodeKind::Directory, bytes: 0, + storage: None, + size_estimated: false, files: 0, dirs: 1, inode: None, @@ -130,6 +137,8 @@ impl Node { name: name.into(), kind, bytes, + storage: None, + size_estimated: false, files: u64::from(kind == NodeKind::File), dirs: 0, inode: None, @@ -141,6 +150,16 @@ impl Node { } } + /// Conservative bytes recoverable from Btrfs data at scan time. + pub fn reclaimable(&self) -> Option { + self.storage.map(|bytes| bytes.get() - 1) + } + + /// Encode zero in the nonzero niche to store the estimate in one word. + pub(crate) const fn set_reclaimable(&mut self, bytes: u64) { + self.storage = NonZeroU64::new(bytes.saturating_add(1)); + } + pub const fn is_dir(&self) -> bool { self.kind.is_dir() } @@ -394,6 +413,15 @@ fn aggregate_at( files += child.files; dirs += child.dirs; } + if node.storage.is_some() + || node.children.iter().any(|child| child.storage.is_some()) + { + node.set_reclaimable(node.children.iter().fold(0_u64, |sum, child| { + sum.saturating_add(child.reclaimable().unwrap_or(child.bytes)) + })); + } + node.size_estimated = node.read_error + || node.children.iter().any(|child| child.size_estimated); node.bytes = bytes; node.files = files; node.dirs = dirs; @@ -441,7 +469,7 @@ mod tests { #[test] #[cfg(target_pointer_width = "64")] fn a_node_stays_small() { - assert_eq!(size_of::(), 88); + assert_eq!(size_of::(), 104); } #[test]