From f55258476908a1c61ddc9dd488d08138a27dfa1c Mon Sep 17 00:00:00 2001 From: Adam Harris Date: Tue, 27 Oct 2026 23:23:52 -0600 Subject: [PATCH] fix(fixity): size the hashing budget to the threads that drain it A fixed 256 MB of read-but-unhashed bytes is right for a laptop and too much for a two-core NAS, whose hash threads also lose the CPU to the filesystem and everything else it runs. Covering an 80,857-file photo share there peaked at 407 MB on 2 GB of RAM. The budget is now 32 MiB per rayon thread, still capped at 256 MiB. With two threads and 60 MB files read from cache, peak memory falls from 294 MB to 122 MB at the same speed; eight threads are unchanged. --- prov-graph/src/fixity.rs | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/prov-graph/src/fixity.rs b/prov-graph/src/fixity.rs index 05c30f1..ee7b1c9 100644 --- a/prov-graph/src/fixity.rs +++ b/prov-graph/src/fixity.rs @@ -272,11 +272,24 @@ mod pool { use std::sync::mpsc::{Receiver, Sender, channel}; use std::sync::{Arc, Condvar, Mutex}; - /// How many bytes may be read and waiting for a hash at once. Enough to - /// keep every core busy on files of any ordinary size; small enough that - /// verifying an archive never holds more than a sliver of it in memory. + /// How many bytes may be read and waiting for a hash at once, at most. + /// Enough to keep every core busy on files of any ordinary size; small + /// enough that verifying an archive never holds more than a sliver of it + /// in memory. const BUDGET: usize = 256 * 1024 * 1024; + /// The bytes each hashing thread may have waiting. Bytes queued past what + /// the threads can take on only sit in memory, so a machine with few cores + /// gets a smaller budget: a two-core NAS, whose threads also lose the CPU + /// to everything else it runs, filled the whole of [`BUDGET`] while + /// covering a photo share, on 2 GB of RAM. + const PER_THREAD: usize = 32 * 1024 * 1024; + + /// The budget for a pool of `threads`. + fn budget(threads: usize) -> usize { + threads.max(1).saturating_mul(PER_THREAD).min(BUDGET) + } + /// The bytes in flight, and a signal when some are done. struct InFlight { budget: usize, @@ -313,7 +326,7 @@ mod pool { impl Pool { pub(super) fn new() -> Self { - Self::with_budget(BUDGET) + Self::with_budget(budget(rayon::current_num_threads())) } fn with_budget(budget: usize) -> Self { @@ -381,6 +394,13 @@ mod pool { let expected: Vec = buffers.iter().map(|b| super::super::digest(b)).collect(); assert_eq!(pool.finish(), expected); } + + #[test] + fn the_budget_grows_with_the_threads_up_to_its_ceiling() { + assert_eq!(budget(0), PER_THREAD); + assert_eq!(budget(2), 2 * PER_THREAD); + assert_eq!(budget(64), BUDGET); + } } }