From 75c1e90d88eff0be7305efc2cf468b821f883ebd Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Sat, 20 Jun 2026 11:15:26 -0700 Subject: [PATCH] fix(search): survive the transient I/O abort (os 995) during a parked-drive re-warm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A search landing while the daemon re-warms parked drives could fail with `Daemon search_cli failed → I/O error … (os error 995)`. 995 is `ERROR_OPERATION_ABORTED`: the OS cancels a *pending* overlapped read when the thread that issued it is reaped (tokio `spawn_blocking` / rayon worker recycling during the re-warm, whose USN-refresh reads MFT extents over the broker's overlapped handle). The read is valid — it just needs reissuing — but it propagated straight out as a hard search failure (retrying by hand worked). Two layers, root + safety net: 1. **Root (uffs-mft `read_handle_at`):** wrap the overlapped read in a bounded retry on `ERROR_OPERATION_ABORTED`. Every broker-handle read goes through this primitive, so the whole re-warm read path becomes abort-resilient. Split into a `read_handle_at` retry wrapper + `read_handle_at_once` attempt; `is_operation_aborted` classifies the 995. 2. **Safety net (uffs-cli `search_retry`):** a new module wraps `search_cli_raw` in `search_cli_with_warm_retry` — bounded retries with back-off when it sees the os-995 transient, printing "UFFS index is warming up — retrying…" on stderr (never stdout) instead of a raw I/O error. Catches the transient regardless of which read produced it; real errors still fail fast. Verified: macOS + windows-msvc builds, macOS + Windows clippy clean, and a `warming_abort_matches_only_995` unit test (retry only on 995, not real errors). Co-Authored-By: Claude Opus 4.8 --- crates/uffs-cli/src/main.rs | 4 +- crates/uffs-cli/src/search_retry.rs | 101 +++++++++++++++++++++++++ crates/uffs-mft/src/platform/volume.rs | 42 +++++++++- 3 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 crates/uffs-cli/src/search_retry.rs diff --git a/crates/uffs-cli/src/main.rs b/crates/uffs-cli/src/main.rs index 48a93aaa1..105cc202e 100644 --- a/crates/uffs-cli/src/main.rs +++ b/crates/uffs-cli/src/main.rs @@ -55,6 +55,7 @@ use assert_cmd as _; pub mod args; pub mod commands; mod dispatch; +mod search_retry; /// Run the CLI and return a result. fn run() -> Result<()> { @@ -313,8 +314,7 @@ pub(crate) fn run_search(args: &[String]) -> Result<()> { let args_owned: Vec = commands::search::args::inject_no_output_for_null_stdout( commands::search::args::resolve_out_path(args), ); - let raw_response = client - .search_cli_raw(&args_owned) + let raw_response = search_retry::search_cli_with_warm_retry(&mut client, &args_owned) .with_context(|| "Daemon search_cli failed")?; let ipc_ms = t_search.elapsed().as_millis(); diff --git a/crates/uffs-cli/src/search_retry.rs b/crates/uffs-cli/src/search_retry.rs new file mode 100644 index 000000000..c4857fc4b --- /dev/null +++ b/crates/uffs-cli/src/search_retry.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Client-side resilience for a search that races a parked-drive re-warm. +//! +//! When the daemon has idled long enough to **park** drives (bodies evicted, +//! only the bloom + path trie resident), the first search re-warms them. On +//! Windows that re-warm reads MFT extents over the broker's overlapped handle, +//! and a pending read can be cancelled with `ERROR_OPERATION_ABORTED` (os error +//! 995) when the worker thread that issued it is reaped. The daemon's own read +//! primitive ([`uffs-mft`'s `read_handle_at`]) already retries that transient; +//! this is the client-side belt-and-suspenders so a user never sees a raw I/O +//! error for a recoverable warm-up hiccup — and gets a "warming" note instead. + +// Each helper is used exactly once on the search path; this is cohesion, not a +// smell — the `single_call_fn` restriction lint is relaxed for the module. +#![expect( + clippy::single_call_fn, + reason = "search-retry helpers, each called once on the search hot path" +)] + +use uffs_client::connect_sync::UffsClientSync; +use uffs_client::error::ClientError; + +/// Max client-side retries when a search aborts mid-index-warm. +const WARM_RETRY_MAX: u32 = 5; +/// Backoff between warm-retry attempts. +const WARM_RETRY_BACKOFF: core::time::Duration = core::time::Duration::from_millis(400); + +/// Run `search_cli_raw`, transparently retrying the one transient a search can +/// hit while the daemon re-warms parked drives: Windows +/// `ERROR_OPERATION_ABORTED` (os error 995). +/// +/// Bounded + back-off so a genuine, persistent failure still fails fast. +pub(crate) fn search_cli_with_warm_retry( + client: &mut UffsClientSync, + args: &[String], +) -> Result { + let mut attempt = 0_u32; + loop { + match client.search_cli_raw(args) { + Ok(response) => return Ok(response), + Err(err) if attempt < WARM_RETRY_MAX && is_index_warming_abort(&err) => { + attempt += 1; + report_index_warming(attempt); + std::thread::sleep(WARM_RETRY_BACKOFF); + } + Err(err) => return Err(err), + } + } +} + +/// `true` if `err` is the transient `ERROR_OPERATION_ABORTED` (os error 995) a +/// search hits when it races a parked-drive re-warm. Matched on the stable OS +/// error code in the rendered message — the daemon's typed I/O error is +/// flattened to a string across the JSON-RPC boundary, so the code is the only +/// portable signal left. +fn is_index_warming_abort(err: &ClientError) -> bool { + let message = err.to_string(); + message.contains("os error 995") || message.contains("operation has been aborted") +} + +/// Tell the user (on stderr, so it never pollutes stdout/CSV) that the index is +/// warming and the search is being retried. +#[expect(clippy::print_stderr, reason = "user-facing progress note, off stdout")] +fn report_index_warming(attempt: u32) { + eprintln!("UFFS index is warming up — retrying search ({attempt}/{WARM_RETRY_MAX})…"); +} + +#[cfg(test)] +mod tests { + #![expect( + clippy::default_numeric_fallback, + reason = "test module — relaxed linting" + )] + + use uffs_client::error::ClientError; + + use super::is_index_warming_abort; + + /// Only a transient `ERROR_OPERATION_ABORTED` (os error 995) — what a + /// search hits while racing a parked-drive re-warm — should be retried; + /// real errors must surface immediately. + #[test] + fn warming_abort_matches_only_995() { + assert!(is_index_warming_abort(&ClientError::Io( + "The I/O operation has been aborted because of either a thread exit \ + or an application request. (os error 995)" + .to_owned() + ))); + assert!(is_index_warming_abort(&ClientError::DaemonError { + code: -32000, + message: "I/O error: ... (os error 995)".to_owned(), + })); + // Real failures are NOT retried. + assert!(!is_index_warming_abort(&ClientError::Io( + "permission denied (os error 5)".to_owned() + ))); + assert!(!is_index_warming_abort(&ClientError::ConnectionClosed)); + } +} diff --git a/crates/uffs-mft/src/platform/volume.rs b/crates/uffs-mft/src/platform/volume.rs index 57aeb71a0..eae13dce3 100644 --- a/crates/uffs-mft/src/platform/volume.rs +++ b/crates/uffs-mft/src/platform/volume.rs @@ -173,8 +173,48 @@ pub(crate) fn try_adopt_broker_handle(drive: super::DriveLetter) -> Result Result<()> { + // `ERROR_OPERATION_ABORTED` (995) is transient: the OS cancels a *pending* + // overlapped read when the thread that issued it is reaped — e.g. a tokio + // `spawn_blocking` / rayon worker recycling during a parked-drive re-warm + // (the USN-refresh reads MFT extents over the broker's overlapped handle). + // The read is valid, so reissue it; bounded so a genuinely wedged handle + // still fails fast instead of spinning forever. + const MAX_ABORT_RETRIES: u32 = 8; + + let mut attempt = 0_u32; + loop { + match read_handle_at_once(handle, offset, buf) { + Ok(()) => return Ok(()), + Err(err) if attempt < MAX_ABORT_RETRIES && is_operation_aborted(&err) => { + attempt += 1; + // Yield so the recycled thread / overlapped slot can settle + // before the read is reissued. + std::thread::yield_now(); + } + Err(err) => return Err(err), + } + } +} + +/// `true` when `err` is a Windows `ERROR_OPERATION_ABORTED` (995) I/O failure — +/// the transient cancellation [`read_handle_at`] retries. +#[cfg(windows)] +fn is_operation_aborted(err: &MftError) -> bool { + let aborted = i32::try_from(ERROR_OPERATION_ABORTED_CODE).ok(); + matches!(err, MftError::Io(io) if io.raw_os_error() == aborted) +} + +/// One overlapped read attempt; the [`read_handle_at`] wrapper adds the +/// transient-abort (995) retry. +/// +/// # Errors +/// +/// [`MftError::Io`] if `ReadFile` / `GetOverlappedResult` fail, or +/// [`MftError::InvalidData`] on a short read. +#[cfg(windows)] +#[expect(unsafe_code, reason = "FFI: overlapped ReadFile + GetOverlappedResult")] +fn read_handle_at_once(handle: HANDLE, offset: u64, buf: &mut [u8]) -> Result<()> { use windows::Win32::Foundation::ERROR_IO_PENDING; use windows::Win32::Storage::FileSystem::ReadFile; use windows::Win32::System::IO::{GetOverlappedResult, OVERLAPPED};