diff --git a/Cargo.lock b/Cargo.lock index a8dc02235..67d0eb5a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4480,65 +4480,6 @@ dependencies = [ "windows 0.62.2", ] -[[package]] -name = "uffs-content" -version = "0.6.28" -dependencies = [ - "anyhow", - "blake3", - "crossbeam-channel", - "serde", - "serde_json", - "tempfile", - "tokio", - "tracing", - "tracing-subscriber", - "uffs-broker-protocol", - "uffs-client", - "uffs-content-protocol", - "uffs-content-reader-protocol", - "uffs-mft", - "uffs-security", - "uffs-version", - "uuid", - "winresource", -] - -[[package]] -name = "uffs-content-protocol" -version = "0.6.28" -dependencies = [ - "bitflags", - "blake3", - "proptest", - "thiserror 2.0.18", -] - -[[package]] -name = "uffs-content-reader" -version = "0.6.28" -dependencies = [ - "anyhow", - "proptest", - "thiserror 2.0.18", - "tokio", - "tracing", - "tracing-subscriber", - "uffs-content-reader-protocol", - "uffs-security", - "uffs-version", - "windows 0.62.2", - "winresource", -] - -[[package]] -name = "uffs-content-reader-protocol" -version = "0.6.28" -dependencies = [ - "proptest", - "thiserror 2.0.18", -] - [[package]] name = "uffs-core" version = "0.6.28" diff --git a/Cargo.toml b/Cargo.toml index 23a1c4c1d..164a9c9f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,27 +24,23 @@ cargo-features = [ resolver = "3" members = [ # โ”€โ”€ Foundation โ”€โ”€ - "crates/uffs-polars", # ๐Ÿš€ Polars facade (compilation isolation) - "crates/uffs-security", # ๐Ÿ”’ Crypto, key storage, secure FS ops - "crates/uffs-text", # ๐Ÿ“ Unicode text processing, i18n foundation - "crates/uffs-time", # โฑ๏ธ NTFS FILETIME arithmetic (pure, zero deps) - "crates/uffs-version", # ๐Ÿท๏ธ Shared --version strings + build-metadata stamp (leaf) - "crates/uffs-statusfmt", # ๐ŸŽจ Shared operator-status styling (color, glyphs, fields) (leaf) - "crates/uffs-broker-protocol", # ๐Ÿ“Ÿ Cross-platform broker wire-protocol types (F5) - "crates/uffs-content-protocol", # ๐Ÿ“จ Cross-platform Content Service wire-protocol types - "crates/uffs-content-reader-protocol", # ๐Ÿ“ก Private Coordinator<->Snapshot Reader wire-protocol types - "crates/uffs-winsvc", # ๐ŸชŸ Native Windows service control + broker-pipe probe (leaf) - "crates/uffs-mft", # ๐Ÿ“ฆ MFT reading โ†’ Polars DataFrame - "crates/uffs-format", # ๐Ÿงพ Shared CSV formatter (daemon + thin CLI) - "crates/uffs-core", # ๐ŸŽฏ Query engine + compact search engine + "crates/uffs-polars", # ๐Ÿš€ Polars facade (compilation isolation) + "crates/uffs-security", # ๐Ÿ”’ Crypto, key storage, secure FS ops + "crates/uffs-text", # ๐Ÿ“ Unicode text processing, i18n foundation + "crates/uffs-time", # โฑ๏ธ NTFS FILETIME arithmetic (pure, zero deps) + "crates/uffs-version", # ๐Ÿท๏ธ Shared --version strings + build-metadata stamp (leaf) + "crates/uffs-statusfmt", # ๐ŸŽจ Shared operator-status styling (color, glyphs, fields) (leaf) + "crates/uffs-broker-protocol", # ๐Ÿ“Ÿ Cross-platform broker wire-protocol types (F5) + "crates/uffs-winsvc", # ๐ŸชŸ Native Windows service control + broker-pipe probe (leaf) + "crates/uffs-mft", # ๐Ÿ“ฆ MFT reading โ†’ Polars DataFrame + "crates/uffs-format", # ๐Ÿงพ Shared CSV formatter (daemon + thin CLI) + "crates/uffs-core", # ๐ŸŽฏ Query engine + compact search engine # โ”€โ”€ Daemon Architecture โ”€โ”€ - "crates/uffs-daemon", # ๐Ÿ›ก๏ธ Background service process - "crates/uffs-client", # ๐Ÿ“ก Thin client library - "crates/uffs-mcp", # ๐Ÿค– MCP stdio adapter for AI agents - "crates/uffs-broker", # ๐Ÿ”‘ Windows elevated handle broker (optional) - "crates/uffs-vss-requestor", # ๐Ÿฉน Per-run native VSS snapshot helper, spawned by uffs-broker (optional) - "crates/uffs-content", # ๐Ÿ“ฆ Content Service โ€” VSS-snapshot-scoped file content export (optional) - "crates/uffs-content-reader", # ๐Ÿ“– Privileged narrow Snapshot Reader, spawned by uffs-content (optional) + "crates/uffs-daemon", # ๐Ÿ›ก๏ธ Background service process + "crates/uffs-client", # ๐Ÿ“ก Thin client library + "crates/uffs-mcp", # ๐Ÿค– MCP stdio adapter for AI agents + "crates/uffs-broker", # ๐Ÿ”‘ Windows elevated handle broker (optional) + "crates/uffs-vss-requestor", # ๐Ÿฉน Per-run native VSS snapshot helper, spawned by uffs-broker (optional) # โ”€โ”€ Surfaces โ”€โ”€ "crates/uffs-cli", # ๐Ÿ–ฅ๏ธ Command-line interface "crates/uffs-update", # โฌ†๏ธ Self-update acquire helper (HTTP/TLS isolated from the CLI) @@ -154,23 +150,6 @@ uffs-client = { path = "crates/uffs-client", version = "0.6.28" } # F5 (issue #205) so neither side duplicates `BROKER_PIPE_NAME` / # wire-format byte literals. uffs-broker-protocol = { path = "crates/uffs-broker-protocol", version = "0.6.28" } -# `uffs-content-protocol` carries the wire-protocol types shared between -# `uffs-content` (the unprivileged content-coordinator producer, -# Windows-only binary) and any downstream consumer (e.g. Docenta). -# Pure-logic Layer-0 lib โ€” cross-platform tests run on every CI lane, -# matching the `uffs-broker-protocol` pattern above. Design references -# (docs/dev/architecture/, local-only, not tracked): the original -# `content-stream-tool-design.md` sketch, its -# `uffs-content-stream-enterprise-design-review.md` replacement-design -# review, and Docenta's `uffs-ingest-protocol-v2-vss.md`. -uffs-content-protocol = { path = "crates/uffs-content-protocol", version = "0.6.28" } -# `uffs-content-reader-protocol` โ€” the private wire format between -# `uffs-content` (Coordinator) and the privileged Snapshot Reader process -# (addendum ยง2.1-ยง2.4). Deliberately does NOT depend on -# `uffs-content-protocol` โ€” both are Layer 0, and Layer-0-to-Layer-0 -# internal deps are disallowed (see crate-graph.md); see this crate's -# Cargo.toml for the full rationale. -uffs-content-reader-protocol = { path = "crates/uffs-content-reader-protocol", version = "0.6.28" } # `uffs-winsvc` โ€” native Windows service control (SCM query/start/stop) + # the non-connecting broker-pipe readiness probe. Layer-0 leaf: its only # dependency is the `windows` crate (windows-target), with non-Windows @@ -244,10 +223,11 @@ windows = { version = "0.62.2", features = [ # โ”€โ”€โ”€โ”€โ”€ Windows Build Tooling (build-dependency only) โ”€โ”€โ”€โ”€โ”€ # Embeds the UFFS icon + `app.manifest` (asInvoker, PerMonitorV2 DPI, long-path -# aware) into the `uffs.exe` PE on MSVC targets. Used exclusively in -# `crates/uffs-cli/build.rs`; inert on non-Windows and on non-MSVC Windows -# toolchains. Kept at workspace scope for version consistency even though -# only one crate currently imports it. +# aware) into each binary's PE on MSVC targets; inert on non-Windows and on +# non-MSVC Windows toolchains. Used by every binary crate's `build.rs` +# (uffs-cli, uffs-daemon, uffs-broker, uffs-mcp, uffs-update, uffs-vss-requestor, +# uffs-bench, uffs-diag, uffs-mft) โ€” kept at workspace scope for version +# consistency across all of them. winresource = "0.1.31" # โ”€โ”€โ”€โ”€โ”€ Serialization โ”€โ”€โ”€โ”€โ”€ @@ -331,11 +311,6 @@ rustc-hash = "2.1.3" itoa = "1.0.18" sha2 = "0.11.0" hex = "0.4.3" -# BLAKE3 is the content-integrity digest mandated by the UFFS Content-Ingest -# protocol (uffs-content-protocol's manifest/frame checksums and the -# FILE_END content digest) โ€” chosen there for speed over sha2, which stays -# in place for its existing unrelated consumers. -blake3 = "1.8.5" # โ”€โ”€โ”€โ”€โ”€ Network (self-update acquire helper only) โ”€โ”€โ”€โ”€โ”€ # Blocking HTTP with rustls + the system trust store. `rustls-tls-native-roots` diff --git a/crates/uffs-content-protocol/Cargo.toml b/crates/uffs-content-protocol/Cargo.toml deleted file mode 100644 index 6441f77cf..000000000 --- a/crates/uffs-content-protocol/Cargo.toml +++ /dev/null @@ -1,65 +0,0 @@ -# ============================================================================ -# uffs-content-protocol: Content Service wire-protocol types -# ============================================================================ -# Layer 0 Foundation crate. Tiny dedicated lib, mirrors the shape of -# `uffs-broker-protocol`. Pure-logic byte-shuffling and enum/struct -# definitions โ€” no Windows FFI, no I/O, no VSS/MFT access. -# -# Shared between `uffs-content` (the privileged VSS-snapshot-scoped Content -# Service producer, Windows-only binary) and any downstream consumer (e.g. -# Docenta) that speaks the framed content protocol. Keeping the contract in -# its own crate means the manifest layout, frame envelope, job/candidate -# state machine, and error taxonomy have one source of truth instead of -# being duplicated by hand on both sides of the wire. -# -# Cross-platform by design: tests run on every CI lane (Linux + macOS + -# Windows). The Windows-only VSS/MFT machinery stays in `uffs-content`'s -# `[[bin]]` where it belongs. -# -# Design references (docs/dev/architecture/, local-only, not tracked): -# the original `content-stream-tool-design.md` sketch, superseded by its -# `uffs-content-stream-enterprise-design-review.md` replacement-design -# review, and Docenta's `uffs-ingest-protocol-v2-vss.md` spec โ€” the -# authoritative sources this crate's types are scaffolded from. -# ============================================================================ - -[package] -name = "uffs-content-protocol" -description = "UFFS Content Service wire-protocol types (cross-platform, shared by uffs-content + downstream consumers)" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -authors.workspace = true -readme.workspace = true -keywords.workspace = true -categories.workspace = true -# Intentionally NOT published (yet). This crate defines the manifest + -# framed content wire protocol between `uffs-content` (the privileged -# producer) and a downstream consumer such as Docenta. Reserve the name -# on crates.io to prevent squatting, but never carry content until the -# protocol is stable enough to be a public contract. See -# `crates/uffs-broker-protocol/Cargo.toml` for the precedent. -publish.workspace = true - -[package.metadata.docs.rs] -all-features = true -rustdoc-args = ["--cfg", "docsrs"] - -[dependencies] -# Structured error type for `codec::DecodeError`. -thiserror.workspace = true -# Content-integrity digest mandated by the protocol spec (manifest/frame -# checksums + FILE_END content digest) โ€” design-doc ยง15.1. -blake3.workspace = true -# `CandidateFlags` bitfield (design-doc ยง11.4: RESIDENT/SPARSE/COMPRESSED/...). -bitflags.workspace = true - -[dev-dependencies] -# Round-trip property tests for the manifest/frame codec (ยง21.6 of the -# design doc: fuzz manifest lengths, frame lengths, offsets, sequence -# numbers). -proptest.workspace = true - -[lints] -workspace = true diff --git a/crates/uffs-content-protocol/proptest-regressions/manifest.txt b/crates/uffs-content-protocol/proptest-regressions/manifest.txt deleted file mode 100644 index 8d573963f..000000000 --- a/crates/uffs-content-protocol/proptest-regressions/manifest.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -cc 0bf1d53bdbc3bbaceb94f4f23c1622fc6cf81da625de5a026488fd4fb18e56ac # shrinks to candidate_id = 0, file_reference = 0, logical_size = 0, valid_data_length = 0, mtime_unix_ms = 0, flags_bits = 0, path_str = "" diff --git a/crates/uffs-content-protocol/src/codec.rs b/crates/uffs-content-protocol/src/codec.rs deleted file mode 100644 index 0e198cc93..000000000 --- a/crates/uffs-content-protocol/src/codec.rs +++ /dev/null @@ -1,627 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Shared little-endian encode/decode primitives for the manifest and -//! frame codecs. -//! -//! Design-doc ยง11/ยง12; addendum ยง5.4: "an explicit deterministic binary -//! codec... not a language-native memory layout or a compatibility-unstable -//! serializer." -//! -//! [`Reader`] is the single chokepoint every length-prefixed field passes -//! through. Its job is to make the bug class from the enterprise review's -//! Finding H10 structurally hard to reintroduce: every bounds check -//! happens *before* any allocation or slice indexing, never after. - -/// Errors produced while decoding wire bytes. -/// -/// Distinct from [`crate::error::ErrorCode`] (design-doc ยง16), which is -/// the *wire-visible* status carried inside frames like `FILE_FAILED`. -/// [`DecodeError`] is a local, Rust-side parsing failure: malformed or -/// truncated bytes handed to a decoder. A [`DecodeError::Truncated`] while -/// parsing a frame is exactly the situation that becomes a `FrameCorrupt` -/// [`crate::error::ErrorCode`] one layer up, once the caller decides how -/// to report it. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[non_exhaustive] -pub enum DecodeError { - /// Fewer bytes remained than the field being read requires. - #[error("truncated input: needed {needed} bytes, only {available} remained")] - Truncated { - /// Bytes required to satisfy the read. - needed: usize, - /// Bytes actually remaining in the input. - available: usize, - }, - /// A length-prefixed field declared more bytes than the caller's - /// configured maximum allows. Checked *before* allocation โ€” this is - /// the direct fix for Finding H10 ("no physical read occurs solely - /// because an unvalidated parser returned an offset"). - #[error("field '{field}' declared length {declared} exceeds maximum {max}")] - LengthOutOfBounds { - /// Name of the offending field, for diagnostics. - field: &'static str, - /// Length the wire bytes claimed. - declared: u64, - /// Maximum length the caller configured. - max: u64, - }, - /// A checksum recomputed over decoded bytes did not match the - /// checksum carried on the wire. - #[error("checksum mismatch: expected 0x{expected:08x}, computed 0x{computed:08x}")] - ChecksumMismatch { - /// Checksum read from the wire. - expected: u32, - /// Checksum recomputed locally. - computed: u32, - }, - /// A discriminant byte/word did not match any known variant of the - /// field it was decoded into (e.g. an unrecognized `frame_type`). - #[error("unknown discriminant for '{field}': {value}")] - UnknownDiscriminant { - /// Name of the field being decoded, for diagnostics. - field: &'static str, - /// The unrecognized value. - value: u64, - }, -} - -/// Bounds-checked little-endian cursor over a decode input buffer. -/// -/// Every `read_*` method checks `self.remaining()` against the field width -/// (or an explicit `max_len`, for length-prefixed data) before touching -/// the slice. There is no path from malformed input to a panic or an -/// over-large allocation. -#[derive(Debug, Clone, Copy)] -pub struct Reader<'a> { - /// Backing bytes being decoded. - buf: &'a [u8], - /// Read offset into `buf`; always `<= buf.len()`. - pos: usize, -} - -impl<'a> Reader<'a> { - /// Wrap `buf` for bounds-checked reading, starting at offset 0. - #[must_use] - pub const fn new(buf: &'a [u8]) -> Self { - Self { buf, pos: 0 } - } - - /// Bytes not yet consumed. - #[must_use] - pub const fn remaining(&self) -> usize { - self.buf.len() - self.pos - } - - /// Current read offset from the start of the buffer. - #[must_use] - pub const fn position(&self) -> usize { - self.pos - } - - /// The full backing buffer, unaffected by how much has been consumed. - /// - /// Used by checksum verification, which recomputes over a byte range - /// of the *original* input rather than the remaining tail. - #[must_use] - pub const fn full_buffer(&self) -> &'a [u8] { - self.buf - } - - /// Consume and return exactly `len` bytes, or a [`DecodeError::Truncated`] - /// if fewer remain. The bounds check happens before the slice is ever - /// touched โ€” this is the single place that guarantee is enforced for - /// every other method in this type. - fn take(&mut self, len: usize) -> Result<&'a [u8], DecodeError> { - let available = self.remaining(); - if available < len { - return Err(DecodeError::Truncated { - needed: len, - available, - }); - } - let start = self.pos; - // `.get(start..start + len)` cannot return `None` here: `len <= - // available == self.buf.len() - start` was just checked above. - // Using `.get()` instead of direct range indexing keeps this - // function panic-free by construction rather than "panic-free - // because a check happens to precede it." - let slice = self - .buf - .get(start..start + len) - .ok_or(DecodeError::Truncated { - needed: len, - available, - })?; - self.pos += len; - Ok(slice) - } - - /// Read a single byte. - /// - /// # Errors - /// - /// [`DecodeError::Truncated`] if no bytes remain. - pub fn read_u8(&mut self) -> Result { - let bytes = self.take(1)?; - // `take(1)` guarantees exactly one byte. - bytes.first().copied().ok_or(DecodeError::Truncated { - needed: 1, - available: 0, - }) - } - - /// Read a little-endian `u16`. - /// - /// # Errors - /// - /// [`DecodeError::Truncated`] if fewer than 2 bytes remain. - pub fn read_u16_le(&mut self) -> Result { - Ok(u16::from_le_bytes(self.read_array()?)) - } - - /// Read a little-endian `u32`. - /// - /// # Errors - /// - /// [`DecodeError::Truncated`] if fewer than 4 bytes remain. - pub fn read_u32_le(&mut self) -> Result { - Ok(u32::from_le_bytes(self.read_array()?)) - } - - /// Read a little-endian `u64`. - /// - /// # Errors - /// - /// [`DecodeError::Truncated`] if fewer than 8 bytes remain. - pub fn read_u64_le(&mut self) -> Result { - Ok(u64::from_le_bytes(self.read_array()?)) - } - - /// Read a little-endian `i64`. - /// - /// # Errors - /// - /// [`DecodeError::Truncated`] if fewer than 8 bytes remain. - pub fn read_i64_le(&mut self) -> Result { - Ok(self.read_u64_le()?.cast_signed()) - } - - /// Read exactly `N` raw bytes. - /// - /// # Errors - /// - /// [`DecodeError::Truncated`] if fewer than `N` bytes remain. - pub fn read_array(&mut self) -> Result<[u8; N], DecodeError> { - let bytes = self.take(N)?; - let mut out = [0_u8; N]; - out.copy_from_slice(bytes); - Ok(out) - } - - /// Read a `u32`-length-prefixed byte string, rejecting (before any - /// allocation) a declared length that exceeds `max_len` or the bytes - /// actually remaining. - /// - /// `field` is only used for the error message. - /// - /// # Errors - /// - /// - [`DecodeError::LengthOutOfBounds`] if the declared length exceeds - /// `max_len`. - /// - [`DecodeError::Truncated`] if the declared length exceeds the bytes - /// remaining. - pub fn read_bytes_u32_prefixed( - &mut self, - field: &'static str, - max_len: u32, - ) -> Result, DecodeError> { - let len = self.read_u32_le()?; - if len > max_len { - return Err(DecodeError::LengthOutOfBounds { - field, - declared: u64::from(len), - max: u64::from(max_len), - }); - } - // `len` is already bounds-checked against `max_len`; `take` - // additionally checks it against bytes actually remaining before - // any copy happens. - let bytes = self.take(len as usize)?; - Ok(bytes.to_vec()) - } - - /// Read a `u16`-length-prefixed byte string, same bounds discipline as - /// [`read_bytes_u32_prefixed`](Self::read_bytes_u32_prefixed). - /// - /// # Errors - /// - /// Same as [`read_bytes_u32_prefixed`](Self::read_bytes_u32_prefixed). - pub fn read_bytes_u16_prefixed( - &mut self, - field: &'static str, - max_len: u16, - ) -> Result, DecodeError> { - let len = self.read_u16_le()?; - if len > max_len { - return Err(DecodeError::LengthOutOfBounds { - field, - declared: u64::from(len), - max: u64::from(max_len), - }); - } - let bytes = self.take(len as usize)?; - Ok(bytes.to_vec()) - } - - /// Read exactly `len` raw bytes with **no** length prefix on the - /// wire โ€” for callers whose length already came from elsewhere (e.g. - /// [`crate::frame::FrameEnvelope`]'s separately-encoded - /// `payload_length` field). The caller is responsible for having - /// already bounds-checked `len` against its own maximum; this method - /// only guarantees `len` does not exceed the bytes actually - /// remaining. - /// - /// # Errors - /// - /// [`DecodeError::Truncated`] if fewer than `len` bytes remain. - pub fn read_bytes_exact(&mut self, len: usize) -> Result, DecodeError> { - let bytes = self.take(len)?; - Ok(bytes.to_vec()) - } -} - -/// Append a little-endian `u16` to `out`. -pub fn write_u16_le(out: &mut Vec, value: u16) { - out.extend_from_slice(&value.to_le_bytes()); -} - -/// Append a little-endian `u32` to `out`. -pub fn write_u32_le(out: &mut Vec, value: u32) { - out.extend_from_slice(&value.to_le_bytes()); -} - -/// Append a little-endian `u64` to `out`. -pub fn write_u64_le(out: &mut Vec, value: u64) { - out.extend_from_slice(&value.to_le_bytes()); -} - -/// Append a little-endian `i64` to `out`. -pub fn write_i64_le(out: &mut Vec, value: i64) { - out.extend_from_slice(&value.cast_unsigned().to_le_bytes()); -} - -/// Append a `u32`-length-prefixed byte string to `out`. -/// -/// # Panics -/// -/// Never panics on `bytes.len() <= u32::MAX`; callers constructing an -/// encoder are expected to keep byte strings within that bound (the -/// decoder side enforces this as a real, non-panicking rejection via -/// [`Reader::read_bytes_u32_prefixed`] โ€” this is the encode side, which -/// only ever runs over data this process already validated on the way -/// in). -pub fn write_bytes_u32_prefixed(out: &mut Vec, bytes: &[u8]) { - #[expect( - clippy::cast_possible_truncation, - reason = "encode-side only; `bytes.len()` is expected to already be \ - bounds-checked by the caller before reaching this helper. \ - A value exceeding u32::MAX here indicates a caller bug, \ - not malformed wire input โ€” there is no untrusted-input \ - path through this function." - )] - let len = bytes.len() as u32; - write_u32_le(out, len); - out.extend_from_slice(bytes); -} - -/// Append a `u16`-length-prefixed byte string to `out`. See -/// [`write_bytes_u32_prefixed`] for the truncation-safety note. -pub fn write_bytes_u16_prefixed(out: &mut Vec, bytes: &[u8]) { - #[expect( - clippy::cast_possible_truncation, - reason = "encode-side only; see write_bytes_u32_prefixed." - )] - let len = bytes.len() as u16; - write_u16_le(out, len); - out.extend_from_slice(bytes); -} - -/// Truncated-BLAKE3 checksum used for manifest/frame header and record -/// checksums (design-doc ยง11/ยง12 call for "u32 or stronger"). -/// -/// Rationale for reusing BLAKE3 here instead of adding a second checksum -/// crate (e.g. CRC-32): the protocol already requires BLAKE3 as its -/// content-integrity digest (ยง15.1), so truncating it to 32 bits for the -/// cheaper structural checksums keeps this crate to one hash primitive. -/// This is explicitly *not* used for content integrity โ€” see -/// [`digest`] for the full 256-bit digest used there. -#[must_use] -pub fn checksum32(bytes: &[u8]) -> u32 { - let hash = blake3::hash(bytes); - let first4: [u8; 4] = hash.as_bytes()[0..4] - .try_into() - .unwrap_or_else(|_| unreachable_checksum_slice()); - u32::from_le_bytes(first4) -} - -/// `blake3::Hash` is always exactly 32 bytes, so slicing its first 4 bytes -/// always succeeds; this helper exists only so `checksum32` has no -/// `unwrap`/`expect` call site, per the workspace's panic policy. -const fn unreachable_checksum_slice() -> [u8; 4] { - [0, 0, 0, 0] -} - -/// Full 256-bit BLAKE3 content digest, as required by design-doc ยง15.1 -/// ("Version 2 uses full-length BLAKE3 over the exact logical bytes -/// emitted for the file"). -/// -/// # Consumer contract (locked, do not change casually) -/// -/// This is **plain, unkeyed BLAKE3-256** over the exact logical bytes โ€” -/// `blake3::hash(bytes)`, no key, no context string, no XOF, standard -/// 32-byte output. This is a deliberate cross-product contract: Docenta's -/// `ContentId` is `blake3:` computed the same way over the same -/// logical bytes, so as long as this stays plain unkeyed BLAKE3-256, -/// Docenta can use `FILE_END.content_digest` directly as its content ID -/// and skip re-hashing entirely. If this ever needs to become keyed or -/// use a different output length, that is a wire-breaking change for -/// Docenta's content-addressing, not just an internal UFFS detail โ€” it -/// needs sign-off from the consumer side, not just a version bump here. -pub type Digest = [u8; 32]; - -/// Compute the full BLAKE3 digest of `bytes`. -#[must_use] -pub fn digest(bytes: &[u8]) -> Digest { - *blake3::hash(bytes).as_bytes() -} - -/// Incremental variant of [`digest`]: the same plain, unkeyed BLAKE3-256 -/// contract, computed over bytes fed in one or more calls to -/// [`IncrementalDigest::update`] instead of one contiguous buffer. -/// -/// Exists so a caller streaming a file in bounded chunks (e.g. this -/// crate's own `CONTENT_CHUNK` producer) can compute `FILE_END`'s -/// `content_digest` without buffering the whole file's bytes just to -/// call [`digest`] once at the end โ€” for a large file, that buffering -/// is the difference between bounded, chunk-sized memory use and memory -/// proportional to the file's full size. -#[derive(Debug, Default)] -pub struct IncrementalDigest { - /// The running BLAKE3 state. - hasher: blake3::Hasher, -} - -impl IncrementalDigest { - /// A fresh hasher with no bytes fed yet. - #[must_use] - pub fn new() -> Self { - Self { - hasher: blake3::Hasher::new(), - } - } - - /// Feed more bytes into the running hash, in order. - pub fn update(&mut self, bytes: &[u8]) { - self.hasher.update(bytes); - } - - /// Finalize and return the digest over every byte fed so far. - /// - /// Takes `&self`, not `self`, matching `blake3::Hasher::finalize`'s - /// own shape โ€” finalizing does not consume the hasher, though this - /// crate's callers only ever finalize once per instance in practice. - #[must_use] - pub fn finalize(&self) -> Digest { - *self.hasher.finalize().as_bytes() - } -} - -#[cfg(test)] -mod tests { - use super::{ - DecodeError, IncrementalDigest, Reader, checksum32, digest, write_bytes_u16_prefixed, - write_bytes_u32_prefixed, write_i64_le, write_u16_le, write_u32_le, write_u64_le, - }; - - #[test] - fn read_u8_consumes_one_byte() { - let mut reader = Reader::new(&[0x42, 0x99]); - assert_eq!(reader.read_u8().unwrap(), 0x42); - assert_eq!(reader.remaining(), 1); - } - - #[test] - fn read_u8_truncated_on_empty() { - let mut reader = Reader::new(&[]); - assert_eq!(reader.read_u8().unwrap_err(), DecodeError::Truncated { - needed: 1, - available: 0 - }); - } - - #[test] - fn read_u16_le_matches_manual_bytes() { - let mut buf = Vec::new(); - write_u16_le(&mut buf, 0x1234); - assert_eq!(buf, [0x34, 0x12]); - let mut reader = Reader::new(&buf); - assert_eq!(reader.read_u16_le().unwrap(), 0x1234); - } - - #[test] - fn read_u32_le_round_trip_boundaries() { - for value in [0_u32, 1, 0xFF, 0x1234_5678, u32::MAX] { - let mut buf = Vec::new(); - write_u32_le(&mut buf, value); - let mut reader = Reader::new(&buf); - assert_eq!(reader.read_u32_le().unwrap(), value); - } - } - - #[test] - fn read_u64_le_round_trip_boundaries() { - for value in [0_u64, 1, 0xFF, 0x0123_4567_89AB_CDEF, u64::MAX] { - let mut buf = Vec::new(); - write_u64_le(&mut buf, value); - let mut reader = Reader::new(&buf); - assert_eq!(reader.read_u64_le().unwrap(), value); - } - } - - #[test] - fn read_i64_le_round_trip_negative() { - for value in [i64::MIN, -1_i64, 0, 1, i64::MAX] { - let mut buf = Vec::new(); - write_i64_le(&mut buf, value); - let mut reader = Reader::new(&buf); - assert_eq!(reader.read_i64_le().unwrap(), value); - } - } - - #[test] - fn read_array_exact_width() { - let mut reader = Reader::new(&[1, 2, 3, 4, 5]); - let arr: [u8; 3] = reader.read_array().unwrap(); - assert_eq!(arr, [1, 2, 3]); - assert_eq!(reader.remaining(), 2); - } - - #[test] - fn length_prefixed_u32_round_trip() { - let mut buf = Vec::new(); - write_bytes_u32_prefixed(&mut buf, b"hello world"); - let mut reader = Reader::new(&buf); - let decoded = reader.read_bytes_u32_prefixed("test_field", 1024).unwrap(); - assert_eq!(decoded, b"hello world"); - } - - #[test] - fn length_prefixed_u32_rejects_over_max_before_truncation_check() { - // Declared length (1000) exceeds max_len (10) even though the - // buffer doesn't actually contain 1000 bytes โ€” this must be - // rejected as LengthOutOfBounds, not Truncated, proving the - // max_len check runs before any attempt to read the payload. - let mut buf = Vec::new(); - write_u32_le(&mut buf, 1000); - let mut reader = Reader::new(&buf); - let err = reader - .read_bytes_u32_prefixed("test_field", 10) - .unwrap_err(); - assert_eq!(err, DecodeError::LengthOutOfBounds { - field: "test_field", - declared: 1000, - max: 10, - }); - } - - #[test] - fn length_prefixed_u32_rejects_declared_length_exceeding_remaining_bytes() { - // Declared length (5) is within max_len (1024) but exceeds what's - // actually left in the buffer (2 bytes) โ€” must be Truncated. - let mut buf = Vec::new(); - write_u32_le(&mut buf, 5); - buf.extend_from_slice(&[1, 2]); - let mut reader = Reader::new(&buf); - let err = reader - .read_bytes_u32_prefixed("test_field", 1024) - .unwrap_err(); - assert_eq!(err, DecodeError::Truncated { - needed: 5, - available: 2, - }); - } - - #[test] - fn length_prefixed_u16_round_trip_and_bounds() { - let mut buf = Vec::new(); - write_bytes_u16_prefixed(&mut buf, b"abc"); - let mut reader = Reader::new(&buf); - assert_eq!(reader.read_bytes_u16_prefixed("f", 10).unwrap(), b"abc"); - - let mut buf2 = Vec::new(); - write_u16_le(&mut buf2, 500); - let mut reader2 = Reader::new(&buf2); - assert_eq!( - reader2.read_bytes_u16_prefixed("f", 10).unwrap_err(), - DecodeError::LengthOutOfBounds { - field: "f", - declared: 500, - max: 10, - } - ); - } - - #[test] - fn checksum32_is_deterministic_and_sensitive_to_content() { - let checksum_hello_1 = checksum32(b"hello"); - let checksum_hello_2 = checksum32(b"hello"); - let checksum_hellp = checksum32(b"hellp"); - assert_eq!(checksum_hello_1, checksum_hello_2); - assert_ne!( - checksum_hello_1, checksum_hellp, - "single-byte change must change the checksum" - ); - } - - #[test] - fn checksum32_empty_input_is_stable() { - // Anchor test: if this ever changes, every existing manifest - // fixture's header checksum silently breaks. - assert_eq!(checksum32(b""), checksum32(b"")); - } - - #[test] - fn digest_is_32_bytes_and_deterministic() { - let d1 = digest(b"some file content"); - let d2 = digest(b"some file content"); - assert_eq!(d1, d2); - assert_eq!(d1.len(), 32); - } - - #[test] - fn digest_differs_for_different_content() { - assert_ne!(digest(b"a"), digest(b"b")); - } - - #[test] - fn digest_matches_plain_unkeyed_blake3_hex_form() { - // Locks the consumer contract documented on `Digest`: this MUST - // be identical to calling `blake3::hash` directly (unkeyed, - // standard output) so `format!("blake3:{}", hex::encode(digest))` - // is byte-for-byte what a Docenta-side `blake3:` content ID - // would compute independently over the same bytes. - let content = b"some file content"; - let via_this_crate = digest(content); - let via_plain_blake3 = blake3::hash(content); - assert_eq!(&via_this_crate, via_plain_blake3.as_bytes()); - } - - #[test] - fn incremental_digest_matches_one_shot_digest_over_the_same_bytes() { - let content = b"some file content, split across several chunks"; - let one_shot = digest(content); - - let mut incremental = IncrementalDigest::new(); - for chunk in content.chunks(7) { - incremental.update(chunk); - } - assert_eq!(incremental.finalize(), one_shot); - } - - #[test] - fn incremental_digest_of_no_bytes_matches_digest_of_empty_slice() { - let incremental = IncrementalDigest::new(); - assert_eq!(incremental.finalize(), digest(b"")); - } - - #[test] - fn reader_position_and_remaining_track_consumption() { - let mut reader = Reader::new(&[0_u8; 10]); - assert_eq!(reader.position(), 0); - assert_eq!(reader.remaining(), 10); - let _consumed: u32 = reader.read_u32_le().unwrap(); - assert_eq!(reader.position(), 4); - assert_eq!(reader.remaining(), 6); - } -} diff --git a/crates/uffs-content-protocol/src/error.rs b/crates/uffs-content-protocol/src/error.rs deleted file mode 100644 index 643275e1d..000000000 --- a/crates/uffs-content-protocol/src/error.rs +++ /dev/null @@ -1,299 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Stable, machine-readable error taxonomy (design-doc ยง16). - -/// Stable, machine-readable protocol/content error code. -/// -/// Every variant maps to exactly one `SCREAMING_SNAKE_CASE` stable string -/// via [`ErrorCode::as_str`] โ€” this is the wire-visible form, and it's -/// what a non-Rust consumer implementing the spec from the Markdown -/// document alone would match on. [`ErrorCode::as_str`] and the -/// [`FromStr`](core::str::FromStr) impl round-trip for every variant; -/// this is asserted by an exhaustive test in this module specifically so -/// a future contributor cannot silently rename a code in one language and -/// not the other (that exact risk is called out in -/// `uffs-ingest-implementation-plan.md` ยง2.2). -/// -/// `#[non_exhaustive]`: a future protocol revision adding a code is an -/// additive, non-breaking change for consumers whose `match` has a -/// wildcard arm. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[non_exhaustive] -pub enum ErrorCode { - // โ”€โ”€ Snapshot lifecycle โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - /// VSS snapshot creation failed. - SnapshotCreateFailed, - /// VSS snapshot device could not be opened. - SnapshotOpenFailed, - /// The snapshot became unavailable after candidates began processing. - SnapshotLost, - /// Copy-on-write storage backing the snapshot was exhausted. - SnapshotStorageExhausted, - - // โ”€โ”€ Manifest / identity โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - /// The manifest failed structural or checksum validation. - ManifestCorrupt, - /// A referenced `candidate_id` does not exist in the finalized manifest. - CandidateIdInvalid, - /// The object opened at read time does not match the manifest identity. - IdentityMismatch, - /// The full file reference's sequence number indicates the MFT record - /// was reused since manifest finalization. - FileReferenceReused, - /// The candidate's path could not be resolved/opened. - PathUnresolvable, - - // โ”€โ”€ Stream resolution โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - /// The requested stream (unnamed `$DATA`) was not found on the object. - StreamNotFound, - /// The stream's on-disk layout is not one this version supports. - StreamLayoutUnsupported, - /// The object's attribute-list layout is not one this version supports. - AttributeListUnsupported, - /// The nonresident data run list failed validation. - RunlistCorrupt, - /// A resolved extent falls outside validated volume bounds. - ExtentOutOfBounds, - /// The EOF/valid-data-length relationship failed validation. - VdlEofInvalid, - - // โ”€โ”€ Deferred-to-manual reasons โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - /// Candidate is NTFS-compressed; deferred to manual handling. - CompressedManual, - /// Candidate is EFS-encrypted; deferred to manual handling. - EncryptedManual, - /// Candidate has an unsupported sparse layout; deferred to manual handling. - SparseManual, - /// Candidate is reparse-point-backed; deferred to manual handling. - ReparseManual, - /// Candidate is Data-Dedup-optimized or otherwise provider-backed; - /// deferred to manual handling. - DedupProviderManual, - /// Candidate is a cloud placeholder; deferred to manual handling. - CloudPlaceholderManual, - /// Candidate has other special semantics not yet supported; deferred - /// to manual handling. - SpecialSemanticsManual, - - // โ”€โ”€ Read / integrity โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - /// A transient I/O error occurred while reading. - ReadIoTransient, - /// A permanent I/O error occurred while reading. - ReadIoPermanent, - /// A read returned fewer bytes than the validated plan required. - ReadShort, - /// Incremental hashing failed internally. - HashFailed, - /// The final digest did not match the expected/reported value. - DigestMismatch, - - // โ”€โ”€ Transport / job โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - /// The consumer disconnected before the operation completed. - ConsumerDisconnected, - /// The consumer explicitly rejected a delivered frame (e.g. digest - /// mismatch on its side). - ConsumerRejected, - /// A protocol-level violation occurred (version skew, invalid frame - /// sequence, etc.) distinct from a single frame's bytes being corrupt. - /// - /// Wire string is `PROTOCOL_ERROR` (design-doc ยง16 literal token); - /// the variant is named `ProtocolViolation` to avoid colliding with - /// this crate's [`crate::codec::DecodeError`] naming. - ProtocolViolation, - /// A frame failed header/payload checksum validation. - FrameCorrupt, - /// The job was cancelled. - JobCancelled, - /// A configured resource limit (byte/file/time/memory/concurrency) - /// was reached. - ResourceLimit, - /// An internal error occurred that does not fit another category. - InternalError, -} - -impl ErrorCode { - /// The stable `SCREAMING_SNAKE_CASE` wire string for this code. - #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::SnapshotCreateFailed => "SNAPSHOT_CREATE_FAILED", - Self::SnapshotOpenFailed => "SNAPSHOT_OPEN_FAILED", - Self::SnapshotLost => "SNAPSHOT_LOST", - Self::SnapshotStorageExhausted => "SNAPSHOT_STORAGE_EXHAUSTED", - Self::ManifestCorrupt => "MANIFEST_CORRUPT", - Self::CandidateIdInvalid => "CANDIDATE_ID_INVALID", - Self::IdentityMismatch => "IDENTITY_MISMATCH", - Self::FileReferenceReused => "FILE_REFERENCE_REUSED", - Self::PathUnresolvable => "PATH_UNRESOLVABLE", - Self::StreamNotFound => "STREAM_NOT_FOUND", - Self::StreamLayoutUnsupported => "STREAM_LAYOUT_UNSUPPORTED", - Self::AttributeListUnsupported => "ATTRIBUTE_LIST_UNSUPPORTED", - Self::RunlistCorrupt => "RUNLIST_CORRUPT", - Self::ExtentOutOfBounds => "EXTENT_OUT_OF_BOUNDS", - Self::VdlEofInvalid => "VDL_EOF_INVALID", - Self::CompressedManual => "COMPRESSED_MANUAL", - Self::EncryptedManual => "ENCRYPTED_MANUAL", - Self::SparseManual => "SPARSE_MANUAL", - Self::ReparseManual => "REPARSE_MANUAL", - Self::DedupProviderManual => "DEDUP_PROVIDER_MANUAL", - Self::CloudPlaceholderManual => "CLOUD_PLACEHOLDER_MANUAL", - Self::SpecialSemanticsManual => "SPECIAL_SEMANTICS_MANUAL", - Self::ReadIoTransient => "READ_IO_TRANSIENT", - Self::ReadIoPermanent => "READ_IO_PERMANENT", - Self::ReadShort => "READ_SHORT", - Self::HashFailed => "HASH_FAILED", - Self::DigestMismatch => "DIGEST_MISMATCH", - Self::ConsumerDisconnected => "CONSUMER_DISCONNECTED", - Self::ConsumerRejected => "CONSUMER_REJECTED", - Self::ProtocolViolation => "PROTOCOL_ERROR", - Self::FrameCorrupt => "FRAME_CORRUPT", - Self::JobCancelled => "JOB_CANCELLED", - Self::ResourceLimit => "RESOURCE_LIMIT", - Self::InternalError => "INTERNAL_ERROR", - } - } - - /// All variants, for exhaustive round-trip testing. - #[cfg(test)] - const ALL: &'static [Self] = &[ - Self::SnapshotCreateFailed, - Self::SnapshotOpenFailed, - Self::SnapshotLost, - Self::SnapshotStorageExhausted, - Self::ManifestCorrupt, - Self::CandidateIdInvalid, - Self::IdentityMismatch, - Self::FileReferenceReused, - Self::PathUnresolvable, - Self::StreamNotFound, - Self::StreamLayoutUnsupported, - Self::AttributeListUnsupported, - Self::RunlistCorrupt, - Self::ExtentOutOfBounds, - Self::VdlEofInvalid, - Self::CompressedManual, - Self::EncryptedManual, - Self::SparseManual, - Self::ReparseManual, - Self::DedupProviderManual, - Self::CloudPlaceholderManual, - Self::SpecialSemanticsManual, - Self::ReadIoTransient, - Self::ReadIoPermanent, - Self::ReadShort, - Self::HashFailed, - Self::DigestMismatch, - Self::ConsumerDisconnected, - Self::ConsumerRejected, - Self::ProtocolViolation, - Self::FrameCorrupt, - Self::JobCancelled, - Self::ResourceLimit, - Self::InternalError, - ]; -} - -/// Error returned by [`ErrorCode`]'s [`FromStr`](core::str::FromStr) impl -/// when the input does not match any known stable wire string. -/// -/// A caller that needs to tolerate codes from a newer producer build -/// should treat this as "unrecognized/future code," not a hard protocol -/// error โ€” that's this crate's forward-compatibility stance for -/// `#[non_exhaustive]` enums in general. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[error("unrecognized ErrorCode wire string")] -pub struct UnknownErrorCode; - -impl core::str::FromStr for ErrorCode { - type Err = UnknownErrorCode; - - fn from_str(value: &str) -> Result { - Ok(match value { - "SNAPSHOT_CREATE_FAILED" => Self::SnapshotCreateFailed, - "SNAPSHOT_OPEN_FAILED" => Self::SnapshotOpenFailed, - "SNAPSHOT_LOST" => Self::SnapshotLost, - "SNAPSHOT_STORAGE_EXHAUSTED" => Self::SnapshotStorageExhausted, - "MANIFEST_CORRUPT" => Self::ManifestCorrupt, - "CANDIDATE_ID_INVALID" => Self::CandidateIdInvalid, - "IDENTITY_MISMATCH" => Self::IdentityMismatch, - "FILE_REFERENCE_REUSED" => Self::FileReferenceReused, - "PATH_UNRESOLVABLE" => Self::PathUnresolvable, - "STREAM_NOT_FOUND" => Self::StreamNotFound, - "STREAM_LAYOUT_UNSUPPORTED" => Self::StreamLayoutUnsupported, - "ATTRIBUTE_LIST_UNSUPPORTED" => Self::AttributeListUnsupported, - "RUNLIST_CORRUPT" => Self::RunlistCorrupt, - "EXTENT_OUT_OF_BOUNDS" => Self::ExtentOutOfBounds, - "VDL_EOF_INVALID" => Self::VdlEofInvalid, - "COMPRESSED_MANUAL" => Self::CompressedManual, - "ENCRYPTED_MANUAL" => Self::EncryptedManual, - "SPARSE_MANUAL" => Self::SparseManual, - "REPARSE_MANUAL" => Self::ReparseManual, - "DEDUP_PROVIDER_MANUAL" => Self::DedupProviderManual, - "CLOUD_PLACEHOLDER_MANUAL" => Self::CloudPlaceholderManual, - "SPECIAL_SEMANTICS_MANUAL" => Self::SpecialSemanticsManual, - "READ_IO_TRANSIENT" => Self::ReadIoTransient, - "READ_IO_PERMANENT" => Self::ReadIoPermanent, - "READ_SHORT" => Self::ReadShort, - "HASH_FAILED" => Self::HashFailed, - "DIGEST_MISMATCH" => Self::DigestMismatch, - "CONSUMER_DISCONNECTED" => Self::ConsumerDisconnected, - "CONSUMER_REJECTED" => Self::ConsumerRejected, - "PROTOCOL_ERROR" => Self::ProtocolViolation, - "FRAME_CORRUPT" => Self::FrameCorrupt, - "JOB_CANCELLED" => Self::JobCancelled, - "RESOURCE_LIMIT" => Self::ResourceLimit, - "INTERNAL_ERROR" => Self::InternalError, - _ => return Err(UnknownErrorCode), - }) - } -} - -#[cfg(test)] -mod tests { - use core::str::FromStr as _; - - use super::ErrorCode; - - #[test] - fn every_variant_round_trips_through_as_str_and_from_str() { - for &code in ErrorCode::ALL { - let wire_str = code.as_str(); - let parsed = ErrorCode::from_str(wire_str) - .unwrap_or_else(|_| panic!("as_str() output {wire_str:?} must parse back")); - assert_eq!(parsed, code, "round-trip mismatch for {wire_str:?}"); - } - } - - #[test] - fn every_variant_has_a_distinct_wire_string() { - let mut seen = std::collections::HashSet::new(); - for &code in ErrorCode::ALL { - assert!( - seen.insert(code.as_str()), - "duplicate wire string: {:?}", - code.as_str() - ); - } - } - - #[test] - fn every_wire_string_is_screaming_snake_case() { - for &code in ErrorCode::ALL { - let wire_str = code.as_str(); - assert!( - wire_str - .chars() - .all(|ch| ch.is_ascii_uppercase() || ch == '_' || ch.is_ascii_digit()), - "{wire_str:?} is not SCREAMING_SNAKE_CASE" - ); - } - } - - #[test] - fn from_str_rejects_unknown_code() { - ErrorCode::from_str("NOT_A_REAL_CODE").unwrap_err(); - ErrorCode::from_str("").unwrap_err(); - ErrorCode::from_str("snapshot_lost").unwrap_err(); // wrong case - } -} diff --git a/crates/uffs-content-protocol/src/frame/content_chunk.rs b/crates/uffs-content-protocol/src/frame/content_chunk.rs deleted file mode 100644 index 779726bef..000000000 --- a/crates/uffs-content-protocol/src/frame/content_chunk.rs +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! `CONTENT_CHUNK` payload (design-doc ยง12.5). - -use super::FrameError; -use crate::codec::{Reader, write_bytes_u32_prefixed, write_u64_le}; - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ CONTENT_CHUNK (ยง12.5) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// `CONTENT_CHUNK` payload. -/// -/// Rules (design-doc ยง12.5): chunks are bounded; `logical_offset` for one -/// file increases monotonically; `payload` is raw logical file bytes in -/// v2; no whole-file buffering is implied. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ContentChunk { - /// Candidate this chunk belongs to. - pub candidate_id: u64, - /// File-local chunk sequence number. - pub chunk_sequence: u64, - /// Logical byte offset of this chunk within the file. - pub logical_offset: u64, - /// Logical length of this chunk (matches `payload.len()`). - pub logical_length: u64, - /// Raw logical file bytes for this chunk. - pub payload: Vec, -} - -impl ContentChunk { - /// Encode this payload. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - write_u64_le(&mut out, self.candidate_id); - write_u64_le(&mut out, self.chunk_sequence); - write_u64_le(&mut out, self.logical_offset); - write_u64_le(&mut out, self.logical_length); - write_bytes_u32_prefixed(&mut out, &self.payload); - out - } - - /// Decode this payload. - /// - /// `max_payload_bytes` bounds the chunk payload before allocation. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>, max_payload_bytes: u32) -> Result { - let candidate_id = reader.read_u64_le()?; - let chunk_sequence = reader.read_u64_le()?; - let logical_offset = reader.read_u64_le()?; - let logical_length = reader.read_u64_le()?; - let payload = reader.read_bytes_u32_prefixed("chunk_payload", max_payload_bytes)?; - Ok(Self { - candidate_id, - chunk_sequence, - logical_offset, - logical_length, - payload, - }) - } -} diff --git a/crates/uffs-content-protocol/src/frame/control.rs b/crates/uffs-content-protocol/src/frame/control.rs deleted file mode 100644 index 3e0230787..000000000 --- a/crates/uffs-content-protocol/src/frame/control.rs +++ /dev/null @@ -1,212 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! `PROGRESS`, `HEARTBEAT`, `JOB_CANCEL`, `WINDOW_UPDATE`, `JOB_RESUME`, -//! and `JOB_SUBMIT` payloads (design-doc ยง12.2, plus this crate's own -//! `JOB_RESUME`/`JOB_SUBMIT` additions โ€” see [`super::FrameType`]'s doc -//! comment). - -use super::{FrameError, read_message, write_message}; -use crate::codec::{Reader, write_u64_le}; - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ PROGRESS / HEARTBEAT / control -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// `PROGRESS` payload (design-doc ยง20.1 job/throughput metrics). Field -/// set is this crate's own choice โ€” the spec names the metric categories -/// but not a fixed wire layout for this frame. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Progress { - /// Candidates discovered so far (manifest may still be enumerating). - pub candidates_discovered: u64, - /// Candidates that have reached a terminal outcome. - pub candidates_completed: u64, - /// Logical bytes successfully emitted so far. - pub logical_bytes_emitted: u64, - /// Total error count (failed + deferred) so far. - pub error_count: u64, -} - -impl Progress { - /// Encode this payload. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - write_u64_le(&mut out, self.candidates_discovered); - write_u64_le(&mut out, self.candidates_completed); - write_u64_le(&mut out, self.logical_bytes_emitted); - write_u64_le(&mut out, self.error_count); - out - } - - /// Decode this payload. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - Ok(Self { - candidates_discovered: reader.read_u64_le()?, - candidates_completed: reader.read_u64_le()?, - logical_bytes_emitted: reader.read_u64_le()?, - error_count: reader.read_u64_le()?, - }) - } -} - -/// `HEARTBEAT` payload. -/// -/// Primarily exists so the frame envelope arriving at all proves -/// liveness (design-doc ยง12.2 "prevents an idle long-file operation from -/// looking dead"), but also carries a cheap resume marker โ€” the -/// producer's own idea of the last candidate it completed โ€” so a -/// reconnecting consumer (or the producer itself, after a transport blip -/// that didn't kill the process) has a recent, no-cost checkpoint -/// without needing a durable ledger. Superseded by the authoritative -/// `FILE_ACK`-driven state in `crate::job::registry` (UFFS-side, not -/// part of this wire crate) whenever the two disagree โ€” this is a hint, -/// not a source of truth. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Heartbeat { - /// The most recent candidate id the producer finished streaming - /// (`FILE_END`/`FILE_FAILED`/`FILE_DEFERRED` already sent for it), or - /// `0` if none yet โ€” `0` is never a real candidate id (candidate ids - /// are 1-based; see `manifest_builder::index_to_candidate_id`'s own - /// reserved-sentinel rationale). - pub last_completed_candidate_id: u64, -} - -impl Heartbeat { - /// Encode this payload. - #[must_use] - pub fn encode(self) -> Vec { - let mut out = Vec::new(); - write_u64_le(&mut out, self.last_completed_candidate_id); - out - } - - /// Decode this payload. A short/empty buffer (an old peer's empty - /// `HEARTBEAT`) decodes as `last_completed_candidate_id: 0` rather - /// than erroring โ€” liveness-only heartbeats from a peer that - /// predates this marker are still valid heartbeats. - #[must_use] - pub fn decode(reader: &mut Reader<'_>) -> Self { - Self { - last_completed_candidate_id: reader.read_u64_le().unwrap_or(0), - } - } -} - -/// `JOB_RESUME` payload: empty. -/// -/// Sent by a reconnecting consumer to resume the job named by this -/// frame's own `FrameEnvelope::job_id` โ€” nothing else needs to travel in -/// the payload, since the envelope already identifies the job. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct JobResume; - -impl JobResume { - /// Encode this payload (always empty). - #[must_use] - #[expect( - clippy::unused_self, - reason = "kept as an instance method for API uniformity with every \ - other frame payload's encode(self/&self) -> Vec shape, \ - even though this particular payload carries no fields" - )] - pub const fn encode(self) -> Vec { - Vec::new() - } - - /// Decode this payload (always succeeds; ignores any bytes present). - #[must_use] - pub const fn decode() -> Self { - Self - } -} - -/// `JOB_SUBMIT` payload: a JSON-encoded job spec. -/// -/// Deliberately opaque bytes rather than a structured wire layout this -/// crate parses field-by-field: the job spec is UFFS-side application -/// data (`uffs_content::job::intake::JobRequest`, outside this crate), -/// not part of the UFFS/Docenta content-stream contract itself. This -/// frame only needs to get those bytes from the consumer to the -/// producer intact; the envelope's own checksums already guard that. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JobSubmit { - /// The job spec, as UTF-8 JSON bytes. - pub job_spec_json: Vec, -} - -impl JobSubmit { - /// Encode this payload (the JSON bytes verbatim). - #[must_use] - pub fn encode(&self) -> Vec { - self.job_spec_json.clone() - } - - /// Decode this payload: the entire frame payload is the JSON bytes, - /// so this takes the raw payload directly rather than a [`Reader`] - /// (there are no further sub-fields to walk). - #[must_use] - pub fn decode(payload: &[u8]) -> Self { - Self { - job_spec_json: payload.to_vec(), - } - } -} - -/// `JOB_CANCEL` payload, sent by the consumer. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JobCancel { - /// Human-readable cancellation reason. - pub reason: String, -} - -impl JobCancel { - /// Encode this payload. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - write_message(&mut out, &self.reason); - out - } - - /// Decode this payload. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - Ok(Self { - reason: read_message(reader)?, - }) - } -} - -/// `WINDOW_UPDATE` payload, sent by the consumer to grant additional -/// backpressure budget (design-doc ยง13.1). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct WindowUpdate { - /// Additional bytes the producer may now have unacknowledged/in-flight. - pub additional_window_bytes: u64, -} - -impl WindowUpdate { - /// Encode this payload. - #[must_use] - pub fn encode(self) -> Vec { - let mut out = Vec::new(); - write_u64_le(&mut out, self.additional_window_bytes); - out - } - - /// Decode this payload. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - Ok(Self { - additional_window_bytes: reader.read_u64_le()?, - }) - } -} diff --git a/crates/uffs-content-protocol/src/frame/file_ack.rs b/crates/uffs-content-protocol/src/frame/file_ack.rs deleted file mode 100644 index 62279d448..000000000 --- a/crates/uffs-content-protocol/src/frame/file_ack.rs +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! `FILE_ACK` payload (design-doc ยง12.9). - -use super::{ConsumerAckStatus, FrameError, read_message, write_message}; -use crate::codec::{Digest, Reader, write_u64_le}; - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ FILE_ACK (ยง12.9) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// `FILE_ACK` payload, sent by the consumer. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileAck { - /// Candidate being acknowledged. - pub candidate_id: u64, - /// Digest the consumer computed for the received content, for - /// `job_id + candidate_id + content_digest` idempotency (design-doc - /// ยง9.4). - pub content_digest: Digest, - /// Whether the consumer accepted or rejected the file. - pub consumer_status: ConsumerAckStatus, - /// Consumer-side error code, if rejected. - pub consumer_error_code: Option, -} - -impl FileAck { - /// Encode this payload. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - write_u64_le(&mut out, self.candidate_id); - out.extend_from_slice(&self.content_digest); - out.push(self.consumer_status.encode()); - match &self.consumer_error_code { - Some(code) => { - out.push(1); - write_message(&mut out, code); - } - None => out.push(0), - } - out - } - - /// Decode this payload. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let candidate_id = reader.read_u64_le()?; - let content_digest: Digest = reader.read_array()?; - let status_byte = reader.read_u8()?; - let consumer_status = ConsumerAckStatus::decode(status_byte).map_err(|byte| { - FrameError::UnknownDiscriminant { - field: "consumer_status", - value: u64::from(byte), - } - })?; - let error_present = reader.read_u8()?; - let consumer_error_code = if error_present == 0 { - None - } else { - Some(read_message(reader)?) - }; - Ok(Self { - candidate_id, - content_digest, - consumer_status, - consumer_error_code, - }) - } -} diff --git a/crates/uffs-content-protocol/src/frame/file_begin.rs b/crates/uffs-content-protocol/src/frame/file_begin.rs deleted file mode 100644 index 79e7eddae..000000000 --- a/crates/uffs-content-protocol/src/frame/file_begin.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! `FILE_BEGIN` payload (design-doc ยง12.4). - -use super::{FrameError, ReadMode, read_optional_u64, write_optional_u64}; -use crate::codec::{Reader, write_i64_le, write_u32_le, write_u64_le}; -use crate::path_encoding::WindowsPath; - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ FILE_BEGIN (ยง12.4) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// `FILE_BEGIN` payload. Does not imply success (design-doc ยง12.4). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileBegin { - /// Candidate identifier. - pub candidate_id: u64, - /// Full NTFS file reference. - pub file_reference: u64, - /// Lossless Windows path. - pub path: WindowsPath, - /// Logical size at snapshot time. - pub logical_size: u64, - /// Modification time, Unix milliseconds. - pub mtime: i64, - /// Read mode selected for this attempt. - pub read_mode: ReadMode, - /// 1-based attempt number for this candidate within the job. - pub attempt_number: u32, - /// Optional shared content-object identifier (design-doc ยง5.3: hard - /// links may share one emitted content body). - pub content_object_id: Option, -} - -impl FileBegin { - /// Encode this payload. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - write_u64_le(&mut out, self.candidate_id); - write_u64_le(&mut out, self.file_reference); - self.path.encode(&mut out); - write_u64_le(&mut out, self.logical_size); - write_i64_le(&mut out, self.mtime); - out.push(self.read_mode.encode()); - write_u32_le(&mut out, self.attempt_number); - write_optional_u64(&mut out, self.content_object_id); - out - } - - /// Decode this payload. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let candidate_id = reader.read_u64_le()?; - let file_reference = reader.read_u64_le()?; - let path = WindowsPath::decode(reader, crate::manifest::MAX_PATH_BYTES)?; - let logical_size = reader.read_u64_le()?; - let mtime = reader.read_i64_le()?; - let read_mode_byte = reader.read_u8()?; - let read_mode = - ReadMode::decode(read_mode_byte).map_err(|byte| FrameError::UnknownDiscriminant { - field: "read_mode", - value: u64::from(byte), - })?; - let attempt_number = reader.read_u32_le()?; - let content_object_id = read_optional_u64(reader)?; - Ok(Self { - candidate_id, - file_reference, - path, - logical_size, - mtime, - read_mode, - attempt_number, - content_object_id, - }) - } -} diff --git a/crates/uffs-content-protocol/src/frame/file_deferred.rs b/crates/uffs-content-protocol/src/frame/file_deferred.rs deleted file mode 100644 index 9cf2d981e..000000000 --- a/crates/uffs-content-protocol/src/frame/file_deferred.rs +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! `FILE_DEFERRED` payload (design-doc ยง12.8). - -use core::str::FromStr as _; - -use super::{FrameError, read_message, write_message}; -use crate::codec::{Reader, write_bytes_u16_prefixed, write_u64_le}; - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ FILE_DEFERRED (ยง12.8) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// `FILE_DEFERRED` payload. No content body is considered successful -/// (design-doc ยง12.8). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileDeferred { - /// Candidate this terminates. - pub candidate_id: u64, - /// Stable machine-readable reason code (one of the `*_MANUAL` - /// [`crate::error::ErrorCode`] variants). - pub reason_code: crate::error::ErrorCode, - /// Optional hint for which manual handler applies. - pub manual_handler_hint: Option, - /// Human-readable diagnostic message. - pub message: String, -} - -impl FileDeferred { - /// Encode this payload. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - write_u64_le(&mut out, self.candidate_id); - write_bytes_u16_prefixed(&mut out, self.reason_code.as_str().as_bytes()); - match &self.manual_handler_hint { - Some(hint) => { - out.push(1); - write_message(&mut out, hint); - } - None => out.push(0), - } - write_message(&mut out, &self.message); - out - } - - /// Decode this payload. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let candidate_id = reader.read_u64_le()?; - let reason_bytes = reader.read_bytes_u16_prefixed("reason_code", 64)?; - let reason_str = String::from_utf8(reason_bytes) - .map_err(|_err| FrameError::InvalidUtf8("reason_code"))?; - let reason_code = crate::error::ErrorCode::from_str(&reason_str).map_err(|_err| { - FrameError::UnknownDiscriminant { - field: "reason_code", - value: 0, - } - })?; - let hint_present = reader.read_u8()?; - let manual_handler_hint = if hint_present == 0 { - None - } else { - Some(read_message(reader)?) - }; - let message = read_message(reader)?; - Ok(Self { - candidate_id, - reason_code, - manual_handler_hint, - message, - }) - } -} diff --git a/crates/uffs-content-protocol/src/frame/file_end.rs b/crates/uffs-content-protocol/src/frame/file_end.rs deleted file mode 100644 index b68021f8c..000000000 --- a/crates/uffs-content-protocol/src/frame/file_end.rs +++ /dev/null @@ -1,93 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! `FILE_END` payload (design-doc ยง12.6). - -use super::{FrameError, ReadMode}; -use crate::codec::{Digest, Reader, write_u32_le, write_u64_le}; - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ FILE_END (ยง12.6) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// `FILE_END` payload: a candidate is successful only after this frame -/// (design-doc ยง12.6). -/// -/// `content_digest` is `None` exactly when `read_mode == -/// ReadMode::MetadataOnly` โ€” the candidate matched the job's query but -/// exceeded its content-delivery ceiling, so no bytes were read and -/// `chunk_count` is `0`. This is still a successful outcome: the -/// candidate is validated and present in the manifest, nothing failed. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileEnd { - /// Candidate this terminates. - pub candidate_id: u64, - /// Total logical bytes emitted. `0` when `content_digest` is `None` - /// (the file's true size is already in `FILE_BEGIN.logical_size`). - pub total_logical_bytes: u64, - /// BLAKE3 digest over the exact emitted logical bytes, or `None` if - /// no content was delivered (see field-level docs above). - pub content_digest: Option, - /// Read mode actually used. - pub read_mode: ReadMode, - /// Number of `CONTENT_CHUNK` frames emitted for this file. - pub chunk_count: u64, - /// Elapsed time for this attempt, milliseconds. - pub elapsed_ms: u64, - /// Reserved warning bitfield; no bits defined in v2. - pub warning_flags: u32, -} - -impl FileEnd { - /// Encode this payload. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - write_u64_le(&mut out, self.candidate_id); - write_u64_le(&mut out, self.total_logical_bytes); - match self.content_digest { - Some(digest) => { - out.push(1); - out.extend_from_slice(&digest); - } - None => out.push(0), - } - out.push(self.read_mode.encode()); - write_u64_le(&mut out, self.chunk_count); - write_u64_le(&mut out, self.elapsed_ms); - write_u32_le(&mut out, self.warning_flags); - out - } - - /// Decode this payload. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let candidate_id = reader.read_u64_le()?; - let total_logical_bytes = reader.read_u64_le()?; - let digest_present = reader.read_u8()?; - let content_digest = if digest_present == 0 { - None - } else { - let digest: Digest = reader.read_array()?; - Some(digest) - }; - let read_mode_byte = reader.read_u8()?; - let read_mode = - ReadMode::decode(read_mode_byte).map_err(|byte| FrameError::UnknownDiscriminant { - field: "read_mode", - value: u64::from(byte), - })?; - let chunk_count = reader.read_u64_le()?; - let elapsed_ms = reader.read_u64_le()?; - let warning_flags = reader.read_u32_le()?; - Ok(Self { - candidate_id, - total_logical_bytes, - content_digest, - read_mode, - chunk_count, - elapsed_ms, - warning_flags, - }) - } -} diff --git a/crates/uffs-content-protocol/src/frame/file_failed.rs b/crates/uffs-content-protocol/src/frame/file_failed.rs deleted file mode 100644 index d40ffec94..000000000 --- a/crates/uffs-content-protocol/src/frame/file_failed.rs +++ /dev/null @@ -1,135 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! `FILE_FAILED` payload (design-doc ยง12.7). - -use core::str::FromStr as _; - -use super::{ - FailureStage, FrameError, RetryClass, read_message, read_optional_i64, write_message, - write_optional_i64, -}; -use crate::codec::{Reader, write_bytes_u16_prefixed, write_u64_le}; - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ FILE_FAILED (ยง12.7) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// `FILE_FAILED` outcome discriminant (design-doc ยง12.7): either of the -/// two failure [`crate::state::CandidateOutcome`] variants. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum FailedOutcome { - /// May be retried in a later job attempt. - Retryable = 0, - /// Will not succeed on retry. - Terminal = 1, -} - -impl FailedOutcome { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::Retryable), - 1 => Ok(Self::Terminal), - other => Err(other), - } - } -} - -/// `FILE_FAILED` payload. Contains no successful content object -/// (design-doc ยง12.7). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileFailed { - /// Candidate this terminates. - pub candidate_id: u64, - /// Retryable vs. terminal. - pub outcome: FailedOutcome, - /// Which stage the failure occurred at. - pub failure_stage: FailureStage, - /// Stable machine-readable error code. - pub error_code: crate::error::ErrorCode, - /// Underlying OS error code, if applicable. - pub os_error_code: Option, - /// How this failure may be retried. - pub retry_class: RetryClass, - /// Bytes emitted before the failure (consumer MUST discard them โ€” - /// design-doc ยง12.7). - pub bytes_emitted_before_failure: u64, - /// Human-readable diagnostic message. - pub message: String, -} - -impl FileFailed { - /// Encode this payload. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - write_u64_le(&mut out, self.candidate_id); - out.push(self.outcome.encode()); - out.push(self.failure_stage.encode()); - write_bytes_u16_prefixed(&mut out, self.error_code.as_str().as_bytes()); - write_optional_i64(&mut out, self.os_error_code); - out.push(self.retry_class.encode()); - write_u64_le(&mut out, self.bytes_emitted_before_failure); - write_message(&mut out, &self.message); - out - } - - /// Decode this payload. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let candidate_id = reader.read_u64_le()?; - let outcome_byte = reader.read_u8()?; - let outcome = FailedOutcome::decode(outcome_byte).map_err(|byte| { - FrameError::UnknownDiscriminant { - field: "outcome", - value: u64::from(byte), - } - })?; - let stage_byte = reader.read_u8()?; - let failure_stage = - FailureStage::decode(stage_byte).map_err(|byte| FrameError::UnknownDiscriminant { - field: "failure_stage", - value: u64::from(byte), - })?; - let error_code_bytes = reader.read_bytes_u16_prefixed("error_code", 64)?; - let error_code_str = String::from_utf8(error_code_bytes) - .map_err(|_err| FrameError::InvalidUtf8("error_code"))?; - let error_code = crate::error::ErrorCode::from_str(&error_code_str).map_err(|_err| { - FrameError::UnknownDiscriminant { - field: "error_code", - value: 0, - } - })?; - let os_error_code = read_optional_i64(reader)?; - let retry_class_byte = reader.read_u8()?; - let retry_class = RetryClass::decode(retry_class_byte).map_err(|byte| { - FrameError::UnknownDiscriminant { - field: "retry_class", - value: u64::from(byte), - } - })?; - let bytes_emitted_before_failure = reader.read_u64_le()?; - let message = read_message(reader)?; - Ok(Self { - candidate_id, - outcome, - failure_stage, - error_code, - os_error_code, - retry_class, - bytes_emitted_before_failure, - message, - }) - } -} diff --git a/crates/uffs-content-protocol/src/frame/job_begin.rs b/crates/uffs-content-protocol/src/frame/job_begin.rs deleted file mode 100644 index f871f820d..000000000 --- a/crates/uffs-content-protocol/src/frame/job_begin.rs +++ /dev/null @@ -1,130 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! `JOB_BEGIN` payload (design-doc ยง12.3). - -use super::{ - ContentSemantics, DigestAlgorithm, FrameError, FrameOrdering, read_optional_u64, - write_optional_u64, -}; -use crate::codec::{ - Digest, Reader, write_bytes_u16_prefixed, write_i64_le, write_u32_le, write_u64_le, -}; - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ JOB_BEGIN (ยง12.3) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// `JOB_BEGIN` payload. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JobBegin { - /// Job identifier. - pub job_id: [u8; 16], - /// Source identifier. - pub source_id: [u8; 16], - /// Opaque VSS snapshot identifier. - pub snapshot_id: Vec, - /// Snapshot creation time, Unix milliseconds. - pub snapshot_created_at: i64, - /// Digest of the finalized candidate manifest. - pub manifest_digest: Digest, - /// Total candidates in the manifest. - pub candidate_count: u64, - /// Authorization model this job was authorized under. - pub authorization_mode: crate::manifest::AuthorizationMode, - /// Cross-file ordering contract (fixed `NONE` in v2). - pub ordering: FrameOrdering, - /// Content semantics (fixed `UNNAMED_LOGICAL_STREAM` in v2). - pub content_semantics: ContentSemantics, - /// Digest algorithm (fixed `BLAKE3` in v2). - pub digest_algorithm: DigestAlgorithm, - /// Negotiated maximum `CONTENT_CHUNK` payload size. - pub max_chunk_bytes: u32, - /// Content-delivery ceiling: candidates whose `logical_size` exceeds - /// this are still enumerated in the manifest (for reap/metadata - /// completeness) but their body is not streamed โ€” - /// `FILE_END.read_mode == ReadMode::MetadataOnly` for those - /// candidates. `None` means no ceiling: every matched candidate gets - /// its content delivered. This is independent of the query's own - /// candidate-match filters (ext/date/etc.) โ€” see - /// [`super::ReadMode::MetadataOnly`]. - pub max_content_delivery_bytes: Option, -} - -impl JobBegin { - /// Encode this payload. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - out.extend_from_slice(&self.job_id); - out.extend_from_slice(&self.source_id); - write_bytes_u16_prefixed(&mut out, &self.snapshot_id); - write_i64_le(&mut out, self.snapshot_created_at); - out.extend_from_slice(&self.manifest_digest); - write_u64_le(&mut out, self.candidate_count); - out.push(self.authorization_mode.encode()); - out.push(self.ordering.encode()); - out.push(self.content_semantics.encode()); - out.push(self.digest_algorithm.encode()); - write_u32_le(&mut out, self.max_chunk_bytes); - write_optional_u64(&mut out, self.max_content_delivery_bytes); - out - } - - /// Decode this payload. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let job_id: [u8; 16] = reader.read_array()?; - let source_id: [u8; 16] = reader.read_array()?; - let snapshot_id = - reader.read_bytes_u16_prefixed("snapshot_id", crate::manifest::MAX_IDENTIFIER_BYTES)?; - let snapshot_created_at = reader.read_i64_le()?; - let manifest_digest: Digest = reader.read_array()?; - let candidate_count = reader.read_u64_le()?; - let auth_byte = reader.read_u8()?; - let authorization_mode = - crate::manifest::AuthorizationMode::decode(auth_byte).map_err(|byte| { - FrameError::UnknownDiscriminant { - field: "authorization_mode", - value: u64::from(byte), - } - })?; - let ordering_byte = reader.read_u8()?; - let ordering = FrameOrdering::decode(ordering_byte).map_err(|byte| { - FrameError::UnknownDiscriminant { - field: "ordering", - value: u64::from(byte), - } - })?; - let semantics_byte = reader.read_u8()?; - let content_semantics = ContentSemantics::decode(semantics_byte).map_err(|byte| { - FrameError::UnknownDiscriminant { - field: "content_semantics", - value: u64::from(byte), - } - })?; - let digest_algo_byte = reader.read_u8()?; - let digest_algorithm = DigestAlgorithm::decode(digest_algo_byte).map_err(|byte| { - FrameError::UnknownDiscriminant { - field: "digest_algorithm", - value: u64::from(byte), - } - })?; - let max_chunk_bytes = reader.read_u32_le()?; - let max_content_delivery_bytes = read_optional_u64(reader)?; - Ok(Self { - job_id, - source_id, - snapshot_id, - snapshot_created_at, - manifest_digest, - candidate_count, - authorization_mode, - ordering, - content_semantics, - digest_algorithm, - max_chunk_bytes, - max_content_delivery_bytes, - }) - } -} diff --git a/crates/uffs-content-protocol/src/frame/job_end.rs b/crates/uffs-content-protocol/src/frame/job_end.rs deleted file mode 100644 index ae4485a1d..000000000 --- a/crates/uffs-content-protocol/src/frame/job_end.rs +++ /dev/null @@ -1,132 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! `JOB_END` payload (design-doc ยง12.10). - -use super::{FrameError, JobStatus}; -use crate::codec::{Digest, Reader, write_bytes_u16_prefixed, write_u64_le}; - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ JOB_END (ยง12.10) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// `JOB_END` payload: the receiver verifies the completeness invariant -/// against these counts (design-doc ยง12.10, ยง2.2, ยง21.7). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JobEnd { - /// Total candidates in the finalized manifest. - pub candidate_count: u64, - /// Candidates with a `SUCCEEDED` outcome. - pub succeeded_count: u64, - /// Candidates with a `FAILED_RETRYABLE` outcome. - pub failed_retryable_count: u64, - /// Candidates with a `FAILED_TERMINAL` outcome. - pub failed_terminal_count: u64, - /// Candidates with a `DEFERRED_MANUAL` outcome. - pub deferred_manual_count: u64, - /// Successful candidates the consumer has durably acknowledged. - pub acknowledged_success_count: u64, - /// Total logical bytes across all successful candidates. - pub logical_bytes_succeeded: u64, - /// Identifier of the durable failure-bucket record set for this job. - pub failure_bucket_id: Vec, - /// Digest of the finalized candidate manifest (must match `JOB_BEGIN`). - pub manifest_digest: Digest, - /// Digest of the durable outcome ledger. - pub outcome_ledger_digest: Digest, - /// Terminal job status. - pub job_status: JobStatus, -} - -impl JobEnd { - /// Encode this payload. - /// - /// # Errors - /// - /// Returns `Err` if `job_status` is not one of the four terminal - /// [`JobStatus`] variants a `JOB_END` frame may carry. - pub fn encode(&self) -> Result, FrameError> { - let job_status_byte = encode_terminal_job_status(self.job_status)?; - let mut out = Vec::new(); - write_u64_le(&mut out, self.candidate_count); - write_u64_le(&mut out, self.succeeded_count); - write_u64_le(&mut out, self.failed_retryable_count); - write_u64_le(&mut out, self.failed_terminal_count); - write_u64_le(&mut out, self.deferred_manual_count); - write_u64_le(&mut out, self.acknowledged_success_count); - write_u64_le(&mut out, self.logical_bytes_succeeded); - write_bytes_u16_prefixed(&mut out, &self.failure_bucket_id); - out.extend_from_slice(&self.manifest_digest); - out.extend_from_slice(&self.outcome_ledger_digest); - out.push(job_status_byte); - Ok(out) - } - - /// Decode this payload. - /// - /// # Errors - /// See [`FrameError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let candidate_count = reader.read_u64_le()?; - let succeeded_count = reader.read_u64_le()?; - let failed_retryable_count = reader.read_u64_le()?; - let failed_terminal_count = reader.read_u64_le()?; - let deferred_manual_count = reader.read_u64_le()?; - let acknowledged_success_count = reader.read_u64_le()?; - let logical_bytes_succeeded = reader.read_u64_le()?; - let failure_bucket_id = reader - .read_bytes_u16_prefixed("failure_bucket_id", crate::manifest::MAX_IDENTIFIER_BYTES)?; - let manifest_digest: Digest = reader.read_array()?; - let outcome_ledger_digest: Digest = reader.read_array()?; - let job_status_byte = reader.read_u8()?; - let job_status = decode_terminal_job_status(job_status_byte)?; - Ok(Self { - candidate_count, - succeeded_count, - failed_retryable_count, - failed_terminal_count, - deferred_manual_count, - acknowledged_success_count, - logical_bytes_succeeded, - failure_bucket_id, - manifest_digest, - outcome_ledger_digest, - job_status, - }) - } -} - -/// Encodes only the four terminal [`JobStatus`] variants a `JOB_END` -/// frame may legally carry โ€” a non-terminal `JobState` reaching this -/// point would itself be a producer bug, not a wire concern. -const fn encode_terminal_job_status(status: JobStatus) -> Result { - match status { - JobStatus::Completed => Ok(0), - JobStatus::CompletedWithFailures => Ok(1), - JobStatus::Cancelled => Ok(2), - JobStatus::Aborted => Ok(3), - JobStatus::Created - | JobStatus::SnapshotCreating - | JobStatus::SnapshotReady - | JobStatus::Enumerating - | JobStatus::ManifestFinalized - | JobStatus::Streaming - | JobStatus::Completing => Err(FrameError::UnknownDiscriminant { - field: "job_status", - value: 255, - }), - } -} - -/// Decode a terminal [`JobStatus`] byte written by -/// [`encode_terminal_job_status`]. -fn decode_terminal_job_status(byte: u8) -> Result { - match byte { - 0 => Ok(JobStatus::Completed), - 1 => Ok(JobStatus::CompletedWithFailures), - 2 => Ok(JobStatus::Cancelled), - 3 => Ok(JobStatus::Aborted), - other => Err(FrameError::UnknownDiscriminant { - field: "job_status", - value: u64::from(other), - }), - } -} diff --git a/crates/uffs-content-protocol/src/frame/mod.rs b/crates/uffs-content-protocol/src/frame/mod.rs deleted file mode 100644 index 1fc5470c1..000000000 --- a/crates/uffs-content-protocol/src/frame/mod.rs +++ /dev/null @@ -1,730 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Frame envelope and per-frame-type payloads (design-doc ยง12). -//! -//! [`FrameEnvelope`] is deliberately payload-agnostic: it validates and -//! frames an opaque byte blob (checking `payload_length` against a -//! caller-supplied maximum *before* allocating, per Finding H10), and -//! hands the payload bytes back to the caller. Decoding those bytes into -//! a concrete frame (`JobBegin`, `FileEnd`, ...) is a second step, keyed -//! on [`FrameType`]. This mirrors [`crate::manifest`]'s -//! header/record split and keeps the bounds-checking chokepoint in one -//! place regardless of which of the 12 frame types is inside. -//! -//! # Wire layout -//! -//! Every frame is this exact byte sequence, all integers little-endian. -//! There is no separate outer length prefix โ€” `header_length` and -//! `payload_length` below are it โ€” so a consumer reading frames directly -//! off a stream (a named pipe, a socket) reads this sequence in order: -//! -//! | Bytes | Field | Notes | -//! |---|---|---| -//! | 4 | `magic` | [`FRAME_MAGIC`] (`b"UFS2"`) | -//! | 2 | `protocol_version` | must equal [`PROTOCOL_VERSION`] | -//! | 2 | `frame_type` | [`FrameType`] discriminant | -//! | 4 | `flags` | reserved, `0` in v2 | -//! | 4 | `header_length` | bytes from `magic` through `frame_sequence`, inclusive (always `48` in v2) | -//! | 8 | `payload_length` | byte length of `payload` below | -//! | 16 | `job_id` | | -//! | 8 | `frame_sequence` | | -//! | 4 | `header_checksum` | [`crate::codec::checksum32`] over the 48 bytes above | -//! | 4 | `payload_checksum` | `checksum32` over `payload` | -//! | `payload_length` | `payload` | opaque bytes; decode per `frame_type` (e.g. [`JobBegin::decode`]) | -//! -//! So: read 24 bytes to learn `payload_length`, read 56 bytes total -//! (`header_length` + both checksums) before you can validate anything, -//! then read exactly `payload_length` more bytes for the payload โ€” 56 + -//! `payload_length` bytes per frame, back to back, no gaps. Validate -//! `header_checksum` against bytes `0..48` and `payload_checksum` -//! against the payload before trusting either; [`FrameEnvelope::decode`] -//! already does all of this for an in-memory buffer holding a whole -//! frame. For assembling frames out of arbitrary read-sized chunks off a -//! live stream, use [`FrameStreamReader`] instead of reimplementing this -//! table โ€” it performs exactly the above and needs no more wiring than a -//! `feed()` call per read plus a `try_next()` loop. - -use crate::codec::{ - Reader, checksum32, write_bytes_u16_prefixed, write_i64_le, write_u16_le, write_u32_le, - write_u64_le, -}; -use crate::path_encoding::PathDecodeError; - -mod content_chunk; -mod control; -mod file_ack; -mod file_begin; -mod file_deferred; -mod file_end; -mod file_failed; -mod job_begin; -mod job_end; -mod stream_reader; - -pub use content_chunk::ContentChunk; -pub use control::{Heartbeat, JobCancel, JobResume, JobSubmit, Progress, WindowUpdate}; -pub use file_ack::FileAck; -pub use file_begin::FileBegin; -pub use file_deferred::FileDeferred; -pub use file_end::FileEnd; -pub use file_failed::{FailedOutcome, FileFailed}; -pub use job_begin::JobBegin; -pub use job_end::JobEnd; -pub use stream_reader::FrameStreamReader; - -/// Frame envelope magic (design-doc ยง12.1). -pub const FRAME_MAGIC: [u8; 4] = *b"UFS2"; - -/// Wire format version this build produces and requires on decode. -/// -/// Every [`FrameEnvelope`] encoded by this crate sets `protocol_version` -/// to this value, and [`FrameEnvelope::decode`] rejects any other value -/// explicitly (see [`FrameError::ProtocolVersionMismatch`]) rather than -/// attempting to parse a header shape it was never validated against โ€” -/// a future wire-breaking change should bump this constant, not -/// silently reinterpret old bytes under a new layout. -pub const PROTOCOL_VERSION: u16 = 2; - -/// Bytes of the fixed envelope header preceding `header_checksum`: -/// magic(4) + `protocol_version`(2) + `frame_type`(2) + flags(4) + -/// `header_length`(4) + `payload_length`(8) + `job_id`(16) + -/// `frame_sequence`(8) = 48. -const ENVELOPE_HEADER_LEN: usize = 48; - -/// Errors decoding a frame envelope or a typed frame payload. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[non_exhaustive] -pub enum FrameError { - /// Underlying bounds/length-prefix decode failure. - #[error(transparent)] - Decode(#[from] crate::codec::DecodeError), - /// Path field failed to decode. - #[error(transparent)] - Path(#[from] PathDecodeError), - /// Envelope magic did not match [`FRAME_MAGIC`]. - #[error("bad frame magic: {0:?}")] - BadMagic([u8; 4]), - /// `protocol_version` did not match [`PROTOCOL_VERSION`] โ€” a peer - /// speaking a different wire format, not a corrupt frame. Rejected - /// before any version-shape-dependent field is parsed, so a future - /// breaking wire change fails loud instead of misparsing. - #[error("protocol_version mismatch: expected {expected}, got {actual}")] - ProtocolVersionMismatch { - /// This build's [`PROTOCOL_VERSION`]. - expected: u16, - /// The version the peer actually sent. - actual: u16, - }, - /// The declared `header_length` did not match bytes actually consumed. - #[error("header_length mismatch: declared {declared}, actual {actual}")] - HeaderLengthMismatch { - /// Declared length from the wire. - declared: u32, - /// Bytes actually consumed decoding the fixed header. - actual: usize, - }, - /// The header checksum did not match the bytes it covers. - #[error("header checksum mismatch: expected 0x{expected:08x}, computed 0x{computed:08x}")] - HeaderChecksumMismatch { - /// Checksum read from the wire. - expected: u32, - /// Checksum recomputed locally. - computed: u32, - }, - /// The payload checksum did not match the bytes it covers. - #[error("payload checksum mismatch: expected 0x{expected:08x}, computed 0x{computed:08x}")] - PayloadChecksumMismatch { - /// Checksum read from the wire. - expected: u32, - /// Checksum recomputed locally. - computed: u32, - }, - /// `payload_length` exceeded the caller's configured - /// `max_frame_payload_bytes` (design-doc ยง12.1/ยง13.1). - #[error("payload_length {declared} exceeds max_frame_payload_bytes {max}")] - PayloadTooLarge { - /// Declared payload length from the wire. - declared: u64, - /// Caller-configured maximum. - max: u64, - }, - /// `frame_type` did not match a known [`FrameType`] discriminant. - #[error("unknown frame_type: {0}")] - UnknownFrameType(u16), - /// A string field (e.g. `message`) was not valid UTF-8. - #[error("field '{0}' is not valid UTF-8")] - InvalidUtf8(&'static str), - /// A discriminant byte did not match any known enum variant. - #[error("unknown discriminant for '{field}': {value}")] - UnknownDiscriminant { - /// Name of the field being decoded, for diagnostics. - field: &'static str, - /// The unrecognized value. - value: u64, - }, -} - -/// The 12 required frame types (design-doc ยง12.2), plus -/// [`Self::JobResume`] and [`Self::JobSubmit`]. -/// -/// Both additions cover ground the design doc leaves unspecified: how a -/// consumer starts a job and how it reconnects to one after a transport -/// blip (ยง10 "Transport model" names the channels but not a submission/ -/// reconnect handshake). `JobResume` has an empty payload โ€” the frame -/// envelope's own `job_id` already names which job to resume. -/// `JobSubmit`'s payload is a JSON-encoded job spec; the consumer -/// chooses the `job_id` up front (in the envelope) and the producer -/// adopts it for the whole job, including its own `JOB_BEGIN`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum FrameType { - /// First frame of a job: query/manifest identity and negotiated limits. - JobBegin = 1, - /// Announces a candidate is about to stream; does not imply success. - FileBegin = 2, - /// One bounded chunk of a file's logical bytes. - ContentChunk = 3, - /// Terminal success for one candidate. - FileEnd = 4, - /// Terminal failure (retryable or terminal) for one candidate. - FileFailed = 5, - /// Terminal manual deferral for one candidate. - FileDeferred = 6, - /// Consumer acknowledgement of a successful file. - FileAck = 7, - /// Periodic job progress metrics. - Progress = 8, - /// Keeps an idle long-file operation from looking dead. - Heartbeat = 9, - /// Final frame of a job: totals and reconciliation. - JobEnd = 10, - /// Consumer-initiated job cancellation. - JobCancel = 11, - /// Consumer-initiated backpressure window increase. - WindowUpdate = 12, - /// Consumer reconnect: resume streaming the job named by this - /// frame's envelope `job_id`, skipping any candidate already - /// acknowledged before the connection dropped. - JobResume = 13, - /// Consumer-initiated job submission: payload is a JSON job spec; - /// the envelope's `job_id` is the consumer-chosen id for the new job. - JobSubmit = 14, -} - -impl FrameType { - /// Serialize to the two-byte wire representation. - #[must_use] - pub const fn encode(self) -> u16 { - self as u16 - } - - /// Parse the two-byte wire representation. - /// - /// # Errors - /// - /// Returns the offending value if it does not match a known variant. - pub const fn decode(value: u16) -> Result { - match value { - 1 => Ok(Self::JobBegin), - 2 => Ok(Self::FileBegin), - 3 => Ok(Self::ContentChunk), - 4 => Ok(Self::FileEnd), - 5 => Ok(Self::FileFailed), - 6 => Ok(Self::FileDeferred), - 7 => Ok(Self::FileAck), - 8 => Ok(Self::Progress), - 9 => Ok(Self::Heartbeat), - 10 => Ok(Self::JobEnd), - 11 => Ok(Self::JobCancel), - 12 => Ok(Self::WindowUpdate), - 13 => Ok(Self::JobResume), - 14 => Ok(Self::JobSubmit), - other => Err(other), - } - } -} - -/// Frame envelope (design-doc ยง12.1): the fixed header every frame -/// shares, wrapping an opaque payload. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FrameEnvelope { - /// Wire format version. - pub protocol_version: u16, - /// Which of the 12 frame types this is. - pub frame_type: FrameType, - /// Reserved bitfield; no bits defined in v2. - pub flags: u32, - /// Job this frame belongs to. - pub job_id: [u8; 16], - /// Session-global monotonic frame sequence number. - pub frame_sequence: u64, -} - -impl FrameEnvelope { - /// Encode this envelope wrapping `payload`, computing `header_length`, - /// `payload_length`, `header_checksum`, and `payload_checksum` - /// automatically. - #[must_use] - pub fn encode(&self, payload: &[u8]) -> Vec { - // Saturates rather than errors: a >16 EiB payload is not a - // realistic input this crate needs to reject gracefully, and - // saturating keeps this function infallible (no unwrap/expect, - // no manufactured error path for an unreachable case). - let payload_length = u64::try_from(payload.len()).unwrap_or(u64::MAX); - - let mut header = Vec::with_capacity(ENVELOPE_HEADER_LEN); - header.extend_from_slice(&FRAME_MAGIC); - write_u16_le(&mut header, self.protocol_version); - write_u16_le(&mut header, self.frame_type.encode()); - write_u32_le(&mut header, self.flags); - write_u32_le( - &mut header, - u32::try_from(ENVELOPE_HEADER_LEN).unwrap_or(u32::MAX), - ); - write_u64_le(&mut header, payload_length); - header.extend_from_slice(&self.job_id); - write_u64_le(&mut header, self.frame_sequence); - - let header_checksum = checksum32(&header); - let payload_checksum = checksum32(payload); - - let mut out = header; - write_u32_le(&mut out, header_checksum); - write_u32_le(&mut out, payload_checksum); - out.extend_from_slice(payload); - out - } - - /// Decode an envelope and its payload from `reader`, rejecting a - /// `payload_length` exceeding `max_payload_bytes` before allocating - /// the payload buffer (design-doc ยง12.1/ยง13.1). - /// - /// # Errors - /// - /// See [`FrameError`] variants. - pub fn decode( - reader: &mut Reader<'_>, - max_payload_bytes: u64, - ) -> Result<(Self, Vec), FrameError> { - let start = reader.position(); - - let magic: [u8; 4] = reader.read_array()?; - if magic != FRAME_MAGIC { - return Err(FrameError::BadMagic(magic)); - } - let protocol_version = reader.read_u16_le()?; - if protocol_version != PROTOCOL_VERSION { - return Err(FrameError::ProtocolVersionMismatch { - expected: PROTOCOL_VERSION, - actual: protocol_version, - }); - } - let frame_type_raw = reader.read_u16_le()?; - let frame_type = FrameType::decode(frame_type_raw).map_err(FrameError::UnknownFrameType)?; - let flags = reader.read_u32_le()?; - let header_length = reader.read_u32_le()?; - let payload_length = reader.read_u64_le()?; - let job_id: [u8; 16] = reader.read_array()?; - let frame_sequence = reader.read_u64_le()?; - - let end = reader.position(); - let consumed = end - start; - if consumed != header_length as usize { - return Err(FrameError::HeaderLengthMismatch { - declared: header_length, - actual: consumed, - }); - } - - let expected_header_checksum = reader.read_u32_le()?; - let header_bytes = - reader - .full_buffer() - .get(start..end) - .ok_or(FrameError::HeaderLengthMismatch { - declared: header_length, - actual: consumed, - })?; - let computed_header_checksum = checksum32(header_bytes); - if expected_header_checksum != computed_header_checksum { - return Err(FrameError::HeaderChecksumMismatch { - expected: expected_header_checksum, - computed: computed_header_checksum, - }); - } - - let expected_payload_checksum = reader.read_u32_le()?; - - if payload_length > max_payload_bytes { - return Err(FrameError::PayloadTooLarge { - declared: payload_length, - max: max_payload_bytes, - }); - } - let payload_len_usize = usize::try_from(payload_length).unwrap_or(usize::MAX); - let payload = reader.read_bytes_exact(payload_len_usize)?; - - let computed_payload_checksum = checksum32(&payload); - if expected_payload_checksum != computed_payload_checksum { - return Err(FrameError::PayloadChecksumMismatch { - expected: expected_payload_checksum, - computed: computed_payload_checksum, - }); - } - - Ok(( - Self { - protocol_version, - frame_type, - flags, - job_id, - frame_sequence, - }, - payload, - )) - } -} - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ shared small enums โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// `ordering` (design-doc ยง12.3): fixed at `NONE` for v2, modeled as an -/// enum so a future version can add variants without breaking the field -/// shape. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum FrameOrdering { - /// No cross-file ordering contract (design-doc ยง2.4) โ€” the only - /// value in v2. - None = 0, -} - -/// `content_semantics` (design-doc ยง12.3): fixed at -/// `UNNAMED_LOGICAL_STREAM` for v2. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum ContentSemantics { - /// Logical bytes of the unnamed/default data stream (design-doc ยง6.1) - /// โ€” the only value in v2. - UnnamedLogicalStream = 0, -} - -/// `digest_algorithm` (design-doc ยง12.3, ยง15.1): fixed at `BLAKE3` for v2. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum DigestAlgorithm { - /// Full-length, plain unkeyed BLAKE3-256 โ€” see - /// [`crate::codec::digest`]'s consumer-contract note. The only value - /// in v2. - Blake3 = 0, -} - -/// `read_mode` (design-doc ยง6, ยง20.1; addendum ยง3.6 planner naming). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum ReadMode { - /// Read from the MFT-resident attribute value. - Resident = 0, - /// Logical open + read against the VSS snapshot namespace (the - /// default nonresident path per addendum ยง3). - LogicalSnapshot = 1, - /// Benchmark-gated raw runlist/extent reconstruction against the - /// snapshot (addendum ยง3.6) โ€” disabled until UFI.3 approves it. - RawSnapshotAccelerator = 2, - /// The candidate matched the job's query but its content body was - /// intentionally not read or streamed, because its logical size - /// exceeds the job's separate content-delivery ceiling - /// (`JobBegin::max_content_delivery_bytes`). The candidate is still - /// present in the manifest and this `FILE_END` still reports - /// `Succeeded` โ€” nothing failed. See [`FileEnd::content_digest`]. - /// - /// This is a deliberate two-tier design, not a producer-invented - /// policy: query filters (ext/date/size-min/etc., matching existing - /// UFFS CLI filters) determine which files become candidates at all - /// โ€” including huge files a consumer only wants recorded as - /// metadata โ€” while the content-delivery ceiling is a second, - /// independent knob controlling which already-matched candidates - /// actually get bodies streamed. A consumer that wants metadata for - /// every file under a root, regardless of size, but content only for - /// small ones expresses that as one job: broad query + a tight - /// delivery ceiling, not as two jobs or a query that silently drops - /// large files from the candidate set (which would break reap/ - /// tombstone completeness โ€” see design-doc ยง2.3). - MetadataOnly = 3, -} - -/// `failure_stage` (design-doc ยง8.3). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum FailureStage { - /// VSS snapshot creation. - SnapshotCreate = 0, - /// VSS snapshot device open. - SnapshotOpen = 1, - /// Candidate enumeration against the snapshot. - Enumeration = 2, - /// Candidate/file identity validation. - Identity = 3, - /// Unnamed-stream resolution. - StreamResolution = 4, - /// Nonresident runlist validation. - RunlistValidation = 5, - /// The physical/logical read itself. - Read = 6, - /// Logical-byte reconstruction (VDL/EOF/sparse rules). - Reconstruction = 7, - /// Incremental digest computation. - Hash = 8, - /// Frame transport to the consumer. - Transport = 9, - /// Waiting on consumer acknowledgement. - ConsumerAck = 10, - /// An internal producer error not attributable to another stage. - Internal = 11, -} - -/// `retry_class` (design-doc ยง8.4). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum RetryClass { - /// Retry within the same job/snapshot. - RetrySameJob = 0, - /// Retry only under a new snapshot. - RetryNewSnapshot = 1, - /// Retry after an external resource condition changes. - RetryAfterResourceChange = 2, - /// Retry only via a manual/special-case handler. - RetryWithManualHandler = 3, - /// Retry only with different credentials/keys (e.g. EFS). - RetryWithCredentialOrKey = 4, - /// Not retryable. - DoNotRetry = 5, -} - -/// `consumer_status` (design-doc ยง12.9). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum ConsumerAckStatus { - /// Consumer validated byte count and digest successfully. - Accepted = 0, - /// Consumer rejected the file (e.g. digest mismatch on its side). - Rejected = 1, -} - -/// Terminal job status (design-doc ยง12.10 `job_status`). Reuses -/// [`crate::state::JobState`] rather than duplicating a second status -/// enum โ€” `JOB_END.job_status` is always one of that machine's terminal -/// states. -pub use crate::state::JobState as JobStatus; - -impl FrameOrdering { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::None), - other => Err(other), - } - } -} - -impl ContentSemantics { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::UnnamedLogicalStream), - other => Err(other), - } - } -} - -impl DigestAlgorithm { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::Blake3), - other => Err(other), - } - } -} - -impl ReadMode { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::Resident), - 1 => Ok(Self::LogicalSnapshot), - 2 => Ok(Self::RawSnapshotAccelerator), - 3 => Ok(Self::MetadataOnly), - other => Err(other), - } - } -} - -impl FailureStage { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::SnapshotCreate), - 1 => Ok(Self::SnapshotOpen), - 2 => Ok(Self::Enumeration), - 3 => Ok(Self::Identity), - 4 => Ok(Self::StreamResolution), - 5 => Ok(Self::RunlistValidation), - 6 => Ok(Self::Read), - 7 => Ok(Self::Reconstruction), - 8 => Ok(Self::Hash), - 9 => Ok(Self::Transport), - 10 => Ok(Self::ConsumerAck), - 11 => Ok(Self::Internal), - other => Err(other), - } - } -} - -impl RetryClass { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::RetrySameJob), - 1 => Ok(Self::RetryNewSnapshot), - 2 => Ok(Self::RetryAfterResourceChange), - 3 => Ok(Self::RetryWithManualHandler), - 4 => Ok(Self::RetryWithCredentialOrKey), - 5 => Ok(Self::DoNotRetry), - other => Err(other), - } - } -} - -impl ConsumerAckStatus { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::Accepted), - 1 => Ok(Self::Rejected), - other => Err(other), - } - } -} - -// โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ string/option helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - -/// Maximum byte length for a free-text `message` field. -const MAX_MESSAGE_BYTES: u16 = 4096; - -/// Append a `u16`-length-prefixed UTF-8 `message` field. -fn write_message(out: &mut Vec, message: &str) { - write_bytes_u16_prefixed(out, message.as_bytes()); -} - -/// Read and UTF-8-validate a `u16`-length-prefixed `message` field. -fn read_message(reader: &mut Reader<'_>) -> Result { - let bytes = reader.read_bytes_u16_prefixed("message", MAX_MESSAGE_BYTES)?; - String::from_utf8(bytes).map_err(|_err| FrameError::InvalidUtf8("message")) -} - -/// Append an `Option` as a presence byte followed by the value if present. -fn write_optional_i64(out: &mut Vec, value: Option) { - match value { - Some(present_value) => { - out.push(1); - write_i64_le(out, present_value); - } - None => out.push(0), - } -} - -/// Read an `Option` encoded by [`write_optional_i64`]. -fn read_optional_i64(reader: &mut Reader<'_>) -> Result, FrameError> { - let present = reader.read_u8()?; - match present { - 0 => Ok(None), - _ => Ok(Some(reader.read_i64_le()?)), - } -} - -/// Append an `Option` as a presence byte followed by the value if present. -fn write_optional_u64(out: &mut Vec, value: Option) { - match value { - Some(present_value) => { - out.push(1); - write_u64_le(out, present_value); - } - None => out.push(0), - } -} - -/// Read an `Option` encoded by [`write_optional_u64`]. -fn read_optional_u64(reader: &mut Reader<'_>) -> Result, FrameError> { - let present = reader.read_u8()?; - match present { - 0 => Ok(None), - _ => Ok(Some(reader.read_u64_le()?)), - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/uffs-content-protocol/src/frame/stream_reader.rs b/crates/uffs-content-protocol/src/frame/stream_reader.rs deleted file mode 100644 index 87219de72..000000000 --- a/crates/uffs-content-protocol/src/frame/stream_reader.rs +++ /dev/null @@ -1,227 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Incremental, transport-agnostic frame assembly. -//! -//! [`FrameEnvelope::decode`] needs a whole frame's bytes (header, both -//! checksums, and the full payload) already in one contiguous buffer โ€” -//! exactly what a consumer reading off a named pipe or socket does *not* -//! have up front: reads return whatever bytes happen to be available, -//! which may split a frame across two reads or bundle several frames -//! into one. [`FrameStreamReader`] closes that gap: feed it bytes as -//! they arrive, in whatever chunks your I/O layer produces, and pull out -//! complete decoded frames as they become available. -//! -//! This crate does no I/O itself (see this crate's `Cargo.toml` header -//! comment) โ€” [`FrameStreamReader`] doesn't change that. The caller -//! still owns the actual `read()` calls (blocking, async, anything); -//! this only assembles the bytes those reads produce into frames. - -use super::{ENVELOPE_HEADER_LEN, FrameEnvelope, FrameError}; -use crate::codec::Reader; - -/// Bytes needed before `payload_length` can even be read: `magic`(4) + -/// `protocol_version`(2) + `frame_type`(2) + `flags`(4) + -/// `header_length`(4) + `payload_length`(8) = 24 โ€” the same leading -/// field order [`FrameEnvelope::decode`] itself reads. -const PAYLOAD_LENGTH_PREFIX_LEN: usize = 24; - -/// Bytes needed before a full frame can be decoded: the fixed header -/// ([`ENVELOPE_HEADER_LEN`]) plus `header_checksum`(4) plus -/// `payload_checksum`(4) โ€” everything preceding the payload itself. -const FIXED_PREFIX_LEN: usize = ENVELOPE_HEADER_LEN + 8; - -/// Incrementally assembles [`FrameEnvelope`]s from bytes fed in as they -/// arrive off a stream. -/// -/// # Example -/// -/// ``` -/// use uffs_content_protocol::frame::FrameStreamReader; -/// -/// let mut assembler = FrameStreamReader::new(1_000_000); -/// // however your I/O layer hands you bytes: -/// // assembler.feed(&bytes_just_read); -/// while let Some((_envelope, _payload)) = assembler.try_next().unwrap() { -/// // handle one fully-decoded frame -/// } -/// // `Ok(None)` means: not enough bytes yet, read more and feed again. -/// ``` -#[derive(Debug)] -pub struct FrameStreamReader { - /// Bytes fed so far that have not yet formed a complete frame. - buffer: Vec, - /// Forwarded to [`FrameEnvelope::decode`] for every frame, and - /// checked against a peeked `payload_length` before buffering that - /// many bytes โ€” so a corrupt or hostile length claim is rejected - /// immediately rather than after accumulating unbounded payload - /// bytes waiting for the rest of a frame that will never decode. - max_payload_bytes: u64, -} - -impl FrameStreamReader { - /// Creates an empty assembler. `max_payload_bytes` bounds any single - /// frame's payload โ€” see [`FrameEnvelope::decode`]'s own parameter of - /// the same name. - #[must_use] - pub const fn new(max_payload_bytes: u64) -> Self { - Self { - buffer: Vec::new(), - max_payload_bytes, - } - } - - /// Appends newly-received bytes (e.g. the result of one `read()` - /// call) to the internal buffer. - pub fn feed(&mut self, bytes: &[u8]) { - self.buffer.extend_from_slice(bytes); - } - - /// Attempts to decode and consume the next complete frame from the - /// buffered bytes. - /// - /// Returns `Ok(None)` when there aren't enough buffered bytes yet - /// for a complete frame โ€” call [`Self::feed`] with more bytes and - /// try again. Returns `Ok(Some(..))` once a full frame decoded - /// successfully; its bytes are removed from the internal buffer, so - /// calling this again immediately may return a second already-fully- - /// buffered frame without an intervening `feed`. - /// - /// # Errors - /// Returns [`FrameError`] if the buffered bytes form a malformed - /// frame (bad magic, a checksum mismatch, an unknown discriminant, - /// ...) or a `payload_length` exceeding `max_payload_bytes`. Either - /// way the underlying stream is desynchronized โ€” there is no - /// well-defined next frame boundary to resume from, so treat this as - /// fatal for the connection (matching `uffs-content`'s own - /// command-pipe dispatcher: log and close, don't retry `try_next`). - pub fn try_next(&mut self) -> Result)>, FrameError> { - let Some(payload_length) = peek_payload_length(&self.buffer) else { - return Ok(None); - }; - if payload_length > self.max_payload_bytes { - return Err(FrameError::PayloadTooLarge { - declared: payload_length, - max: self.max_payload_bytes, - }); - } - let payload_len_usize = usize::try_from(payload_length).unwrap_or(usize::MAX); - let Some(total_len) = FIXED_PREFIX_LEN.checked_add(payload_len_usize) else { - return Err(FrameError::PayloadTooLarge { - declared: payload_length, - max: self.max_payload_bytes, - }); - }; - if self.buffer.len() < total_len { - return Ok(None); - } - - let frame_bytes: Vec = self.buffer.drain(0..total_len).collect(); - let mut reader = Reader::new(&frame_bytes); - let (envelope, payload) = FrameEnvelope::decode(&mut reader, self.max_payload_bytes)?; - Ok(Some((envelope, payload))) - } -} - -/// Peeks `payload_length` out of `buffer` without consuming anything, -/// reading the exact same leading field sequence -/// [`FrameEnvelope::decode`] does. Returns `None` if `buffer` doesn't -/// yet hold [`PAYLOAD_LENGTH_PREFIX_LEN`] bytes. -fn peek_payload_length(buffer: &[u8]) -> Option { - if buffer.len() < PAYLOAD_LENGTH_PREFIX_LEN { - return None; - } - let mut reader = Reader::new(buffer); - let _magic: [u8; 4] = reader.read_array().ok()?; - let _protocol_version = reader.read_u16_le().ok()?; - let _frame_type_raw = reader.read_u16_le().ok()?; - let _flags = reader.read_u32_le().ok()?; - let _header_length = reader.read_u32_le().ok()?; - reader.read_u64_le().ok() -} - -#[cfg(test)] -mod tests { - use super::FrameStreamReader; - use crate::frame::{FrameEnvelope, FrameType, PROTOCOL_VERSION}; - - fn sample_frame_bytes(frame_sequence: u64, payload: &[u8]) -> Vec { - FrameEnvelope { - protocol_version: PROTOCOL_VERSION, - frame_type: FrameType::Heartbeat, - flags: 0, - job_id: [9_u8; 16], - frame_sequence, - } - .encode(payload) - } - - #[test] - fn returns_none_until_enough_bytes_are_fed() { - let full = sample_frame_bytes(1, b"hello"); - let mut assembler = FrameStreamReader::new(1_000_000); - - // Feed one byte at a time; only the very last byte should - // complete the frame. - for (index, byte) in full.iter().enumerate() { - assembler.feed(core::slice::from_ref(byte)); - let result = assembler.try_next().expect("no decode error expected"); - if index + 1 < full.len() { - assert!(result.is_none(), "must not decode before all bytes arrive"); - } else { - let (envelope, payload) = result.expect("frame must be ready on the last byte"); - assert_eq!(envelope.frame_sequence, 1); - assert_eq!(payload, b"hello"); - } - } - } - - #[test] - fn assembles_two_frames_delivered_in_one_chunk() { - let mut all_bytes = sample_frame_bytes(1, b"first"); - all_bytes.extend(sample_frame_bytes(2, b"second")); - - let mut assembler = FrameStreamReader::new(1_000_000); - assembler.feed(&all_bytes); - - let (first_envelope, first_payload) = assembler - .try_next() - .expect("no decode error expected") - .expect("first frame must be ready"); - assert_eq!(first_envelope.frame_sequence, 1); - assert_eq!(first_payload, b"first"); - - let (second_envelope, second_payload) = assembler - .try_next() - .expect("no decode error expected") - .expect("second frame must be ready without an extra feed"); - assert_eq!(second_envelope.frame_sequence, 2); - assert_eq!(second_payload, b"second"); - - assert!( - assembler - .try_next() - .expect("no decode error expected") - .is_none(), - "buffer must be empty after both frames are consumed" - ); - } - - #[test] - fn rejects_a_payload_length_over_the_ceiling_without_buffering_it() { - let big_frame = sample_frame_bytes(1, &[0_u8; 64]); - // A ceiling smaller than the real payload โ€” the assembler must - // reject this from the length prefix alone, without needing the - // full (oversized) payload to ever be fed. - let mut assembler = FrameStreamReader::new(10); - assembler.feed(&big_frame); - - let err = assembler - .try_next() - .expect_err("payload_length exceeds max_payload_bytes"); - assert!(matches!(err, crate::frame::FrameError::PayloadTooLarge { - declared: 64, - max: 10 - })); - } -} diff --git a/crates/uffs-content-protocol/src/frame/tests.rs b/crates/uffs-content-protocol/src/frame/tests.rs deleted file mode 100644 index d5ce0b2e5..000000000 --- a/crates/uffs-content-protocol/src/frame/tests.rs +++ /dev/null @@ -1,589 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Unit tests for [`super`] (`frame`) and all twelve payload submodules. - -use super::{ - ConsumerAckStatus, ContentChunk, ContentSemantics, DigestAlgorithm, FailedOutcome, - FailureStage, FileAck, FileBegin, FileDeferred, FileEnd, FileFailed, FrameEnvelope, FrameError, - FrameOrdering, FrameType, Heartbeat, JobBegin, JobCancel, JobEnd, JobResume, JobStatus, - JobSubmit, PROTOCOL_VERSION, Progress, ReadMode, RetryClass, WindowUpdate, -}; -use crate::codec::Reader; -use crate::error::ErrorCode; -use crate::manifest::AuthorizationMode; -use crate::path_encoding::WindowsPath; - -fn sample_envelope(frame_type: FrameType, frame_sequence: u64) -> FrameEnvelope { - FrameEnvelope { - protocol_version: PROTOCOL_VERSION, - frame_type, - flags: 0, - job_id: [3_u8; 16], - frame_sequence, - } -} - -#[test] -fn envelope_round_trips_with_payload() { - let envelope = sample_envelope(FrameType::Heartbeat, 7); - let payload = b"hello frame payload"; - let bytes = envelope.encode(payload); - let mut reader = Reader::new(&bytes); - let (decoded_envelope, decoded_payload) = - FrameEnvelope::decode(&mut reader, 1_000_000).unwrap(); - assert_eq!(decoded_envelope, envelope); - assert_eq!(decoded_payload, payload); - assert_eq!(reader.remaining(), 0); -} - -#[test] -fn envelope_round_trips_with_empty_payload() { - let envelope = sample_envelope(FrameType::JobCancel, 1); - let bytes = envelope.encode(&[]); - let mut reader = Reader::new(&bytes); - let (decoded_envelope, decoded_payload) = - FrameEnvelope::decode(&mut reader, 1_000_000).unwrap(); - assert_eq!(decoded_envelope, envelope); - assert!(decoded_payload.is_empty()); -} - -#[test] -#[expect( - clippy::indexing_slicing, - reason = "test mutation of a known, already-validated buffer index; \ - clippy::get_unwrap is also denied, so a scoped exception on \ - direct indexing is the established pattern for this \ - conflict (see crates/uffs-daemon/tests/ipc_integration.rs)" -)] -fn envelope_rejects_bad_magic() { - let envelope = sample_envelope(FrameType::Heartbeat, 1); - let mut bytes = envelope.encode(&[]); - bytes[0] = b'Z'; - let mut reader = Reader::new(&bytes); - let err = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap_err(); - assert!(matches!(err, FrameError::BadMagic(_))); -} - -#[test] -#[expect( - clippy::indexing_slicing, - reason = "test mutation of a known, already-validated buffer index; \ - clippy::get_unwrap is also denied, so a scoped exception on \ - direct indexing is the established pattern for this \ - conflict (see crates/uffs-daemon/tests/ipc_integration.rs)" -)] -fn envelope_rejects_flipped_header_byte_via_checksum() { - let envelope = sample_envelope(FrameType::Heartbeat, 1); - let mut bytes = envelope.encode(&[]); - // job_id lives inside the checksummed header region. - bytes[20] ^= 0xFF; - let mut reader = Reader::new(&bytes); - let err = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap_err(); - assert!(matches!(err, FrameError::HeaderChecksumMismatch { .. })); -} - -#[test] -#[expect( - clippy::indexing_slicing, - reason = "test mutation of a known, already-validated buffer index; \ - clippy::get_unwrap is also denied, so a scoped exception on \ - direct indexing is the established pattern for this \ - conflict (see crates/uffs-daemon/tests/ipc_integration.rs)" -)] -fn envelope_rejects_flipped_payload_byte_via_checksum() { - let envelope = sample_envelope(FrameType::Heartbeat, 1); - let mut bytes = envelope.encode(b"payload bytes here"); - let last = bytes.len() - 1; - bytes[last] ^= 0xFF; - let mut reader = Reader::new(&bytes); - let err = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap_err(); - assert!(matches!(err, FrameError::PayloadChecksumMismatch { .. })); -} - -#[test] -fn envelope_rejects_payload_exceeding_max_before_allocation() { - let envelope = sample_envelope(FrameType::ContentChunk, 1); - let bytes = envelope.encode(&[1, 2, 3, 4, 5]); - let mut reader = Reader::new(&bytes); - let err = FrameEnvelope::decode(&mut reader, 2).unwrap_err(); - assert!(matches!(err, FrameError::PayloadTooLarge { - declared: 5, - max: 2 - })); -} - -#[test] -#[expect( - clippy::indexing_slicing, - reason = "test mutation of a known, already-validated buffer range; \ - clippy::get_unwrap is also denied, so a scoped exception on \ - direct indexing is the established pattern for this \ - conflict (see crates/uffs-daemon/tests/ipc_integration.rs)" -)] -fn envelope_rejects_unknown_frame_type() { - // Hand-craft an envelope with frame_type = 999 by encoding a - // valid one then patching the frame_type field bytes directly - // (offset 6-7: magic(4) + protocol_version(2)). - let envelope = sample_envelope(FrameType::Heartbeat, 1); - let mut bytes = envelope.encode(&[]); - bytes[6..8].copy_from_slice(&999_u16.to_le_bytes()); - let mut reader = Reader::new(&bytes); - let err = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap_err(); - assert!(matches!(err, FrameError::UnknownFrameType(999))); -} - -#[test] -#[expect( - clippy::indexing_slicing, - reason = "test mutation of a known, already-validated buffer range; \ - clippy::get_unwrap is also denied, so a scoped exception on \ - direct indexing is the established pattern for this \ - conflict (see crates/uffs-daemon/tests/ipc_integration.rs)" -)] -fn envelope_rejects_mismatched_protocol_version() { - // Patch the protocol_version field bytes directly (offset 4-5: - // magic(4), before frame_type at offset 6-7) rather than constructing - // an envelope with the "wrong" version, since `FrameEnvelope` only - // has one field for it and this crate defines what "right" means. - let envelope = sample_envelope(FrameType::Heartbeat, 1); - let mut bytes = envelope.encode(&[]); - bytes[4..6].copy_from_slice(&99_u16.to_le_bytes()); - let mut reader = Reader::new(&bytes); - let err = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap_err(); - assert!(matches!(err, FrameError::ProtocolVersionMismatch { - expected: PROTOCOL_VERSION, - actual: 99 - })); -} - -#[test] -fn frame_type_round_trips_all_variants() { - for value in 1_u16..=14 { - let frame_type = FrameType::decode(value).unwrap(); - assert_eq!(frame_type.encode(), value); - } - assert_eq!(FrameType::decode(0), Err(0)); - assert_eq!(FrameType::decode(15), Err(15)); -} - -fn sample_job_begin() -> JobBegin { - JobBegin { - job_id: [1_u8; 16], - source_id: [2_u8; 16], - snapshot_id: b"snap-1".to_vec(), - snapshot_created_at: 1_752_000_000_000, - manifest_digest: [4_u8; 32], - candidate_count: 10, - authorization_mode: AuthorizationMode::AdminExport, - ordering: FrameOrdering::None, - content_semantics: ContentSemantics::UnnamedLogicalStream, - digest_algorithm: DigestAlgorithm::Blake3, - max_chunk_bytes: 65536, - max_content_delivery_bytes: Some(64 * 1024 * 1024), - } -} - -#[test] -fn job_begin_round_trips() { - let payload = sample_job_begin(); - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = JobBegin::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); - assert_eq!(reader.remaining(), 0); -} - -#[test] -fn job_begin_round_trips_with_no_delivery_ceiling() { - let mut payload = sample_job_begin(); - payload.max_content_delivery_bytes = None; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = JobBegin::decode(&mut reader).unwrap(); - assert_eq!(decoded.max_content_delivery_bytes, None); -} - -fn sample_file_begin() -> FileBegin { - FileBegin { - candidate_id: 42, - file_reference: 0xABCD_EF01, - path: WindowsPath::from_str_lossless(r"C:\data\report.txt"), - logical_size: 2048, - mtime: 1_752_000_000_000, - read_mode: ReadMode::LogicalSnapshot, - attempt_number: 1, - content_object_id: None, - } -} - -#[test] -fn file_begin_round_trips() { - let payload = sample_file_begin(); - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = FileBegin::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); - assert_eq!(reader.remaining(), 0); -} - -#[test] -fn file_begin_round_trips_with_content_object_id() { - let mut payload = sample_file_begin(); - payload.content_object_id = Some(999); - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = FileBegin::decode(&mut reader).unwrap(); - assert_eq!(decoded.content_object_id, Some(999)); -} - -#[test] -fn content_chunk_round_trips() { - let payload = ContentChunk { - candidate_id: 1, - chunk_sequence: 0, - logical_offset: 0, - logical_length: 4, - payload: vec![1, 2, 3, 4], - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = ContentChunk::decode(&mut reader, 1024).unwrap(); - assert_eq!(decoded, payload); -} - -#[test] -fn content_chunk_rejects_payload_over_max_before_allocation() { - let payload = ContentChunk { - candidate_id: 1, - chunk_sequence: 0, - logical_offset: 0, - logical_length: 4, - payload: vec![1, 2, 3, 4], - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let err = ContentChunk::decode(&mut reader, 2).unwrap_err(); - assert!(matches!(err, FrameError::Decode(_))); -} - -#[test] -fn file_end_round_trips_with_delivered_content() { - let payload = FileEnd { - candidate_id: 1, - total_logical_bytes: 4096, - content_digest: Some([5_u8; 32]), - read_mode: ReadMode::LogicalSnapshot, - chunk_count: 1, - elapsed_ms: 12, - warning_flags: 0, - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = FileEnd::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); - assert_eq!(reader.remaining(), 0); -} - -#[test] -fn file_end_round_trips_metadata_only_with_no_digest() { - // The content-delivery-ceiling case (design-doc addendum - // discussion): candidate matched and validated, but its body - // exceeded the job's delivery ceiling, so nothing was read. - let payload = FileEnd { - candidate_id: 2, - total_logical_bytes: 0, - content_digest: None, - read_mode: ReadMode::MetadataOnly, - chunk_count: 0, - elapsed_ms: 1, - warning_flags: 0, - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = FileEnd::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); - assert_eq!(decoded.content_digest, None); - assert_eq!(decoded.read_mode, ReadMode::MetadataOnly); -} - -#[test] -fn read_mode_round_trips_all_variants_including_metadata_only() { - for value in 0_u8..=3 { - let mode = ReadMode::decode(value).unwrap(); - assert_eq!(mode.encode(), value); - } - assert_eq!(ReadMode::decode(4), Err(4)); -} - -#[test] -fn file_failed_round_trips() { - let payload = FileFailed { - candidate_id: 3, - outcome: FailedOutcome::Retryable, - failure_stage: FailureStage::Read, - error_code: ErrorCode::ReadIoTransient, - os_error_code: Some(-5), - retry_class: RetryClass::RetrySameJob, - bytes_emitted_before_failure: 100, - message: "transient read error".to_owned(), - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = FileFailed::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); - assert_eq!(reader.remaining(), 0); -} - -#[test] -fn file_failed_round_trips_without_os_error_code() { - let payload = FileFailed { - candidate_id: 4, - outcome: FailedOutcome::Terminal, - failure_stage: FailureStage::Identity, - error_code: ErrorCode::IdentityMismatch, - os_error_code: None, - retry_class: RetryClass::DoNotRetry, - bytes_emitted_before_failure: 0, - message: String::new(), - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = FileFailed::decode(&mut reader).unwrap(); - assert_eq!(decoded.os_error_code, None); - assert_eq!(decoded.message, ""); -} - -#[test] -fn file_deferred_round_trips_with_hint() { - let payload = FileDeferred { - candidate_id: 5, - reason_code: ErrorCode::CompressedManual, - manual_handler_hint: Some("ntfs-compressed-handler".to_owned()), - message: "NTFS compression not yet supported".to_owned(), - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = FileDeferred::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); -} - -#[test] -fn file_deferred_round_trips_without_hint() { - let payload = FileDeferred { - candidate_id: 6, - reason_code: ErrorCode::SpecialSemanticsManual, - manual_handler_hint: None, - message: String::new(), - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = FileDeferred::decode(&mut reader).unwrap(); - assert_eq!(decoded.manual_handler_hint, None); -} - -#[test] -fn file_ack_round_trips_accepted() { - let payload = FileAck { - candidate_id: 7, - content_digest: [6_u8; 32], - consumer_status: ConsumerAckStatus::Accepted, - consumer_error_code: None, - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = FileAck::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); -} - -#[test] -fn file_ack_round_trips_rejected_with_error_code() { - let payload = FileAck { - candidate_id: 8, - content_digest: [7_u8; 32], - consumer_status: ConsumerAckStatus::Rejected, - consumer_error_code: Some("DIGEST_MISMATCH_LOCAL".to_owned()), - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = FileAck::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); -} - -#[test] -fn job_end_round_trips_for_every_terminal_status() { - for job_status in [ - JobStatus::Completed, - JobStatus::CompletedWithFailures, - JobStatus::Cancelled, - JobStatus::Aborted, - ] { - let payload = JobEnd { - candidate_count: 10, - succeeded_count: 8, - failed_retryable_count: 1, - failed_terminal_count: 1, - deferred_manual_count: 0, - acknowledged_success_count: 8, - logical_bytes_succeeded: 4096, - failure_bucket_id: b"bucket-1".to_vec(), - manifest_digest: [8_u8; 32], - outcome_ledger_digest: [9_u8; 32], - job_status, - }; - let bytes = payload.encode().unwrap(); - let mut reader = Reader::new(&bytes); - let decoded = JobEnd::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload, "round-trip failed for {job_status:?}"); - } -} - -#[test] -fn job_end_rejects_non_terminal_job_status_at_encode_time() { - let payload = JobEnd { - candidate_count: 1, - succeeded_count: 0, - failed_retryable_count: 0, - failed_terminal_count: 0, - deferred_manual_count: 0, - acknowledged_success_count: 0, - logical_bytes_succeeded: 0, - failure_bucket_id: vec![], - manifest_digest: [0_u8; 32], - outcome_ledger_digest: [0_u8; 32], - job_status: JobStatus::Streaming, // not a legal JOB_END status - }; - let err = payload.encode().unwrap_err(); - assert!(matches!(err, FrameError::UnknownDiscriminant { - field: "job_status", - .. - })); -} - -#[test] -fn job_end_completeness_invariant_matches_sample_data() { - // Anchors design-doc ยง2.2/ยง21.7: candidate_count must equal the - // sum of the four outcome buckets. This test doesn't enforce the - // invariant in the wire format itself (that's the Coordinator's - // job) โ€” it documents the expectation against a concrete example. - let succeeded = 8_u64; - let failed_retryable = 1_u64; - let failed_terminal = 1_u64; - let deferred_manual = 0_u64; - let candidate_count = 10_u64; - assert_eq!( - candidate_count, - succeeded + failed_retryable + failed_terminal + deferred_manual - ); -} - -#[test] -fn progress_round_trips() { - let payload = Progress { - candidates_discovered: 100, - candidates_completed: 42, - logical_bytes_emitted: 1_000_000, - error_count: 2, - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = Progress::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); -} - -#[test] -fn heartbeat_round_trips_the_progress_marker() { - let payload = Heartbeat { - last_completed_candidate_id: 42, - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = Heartbeat::decode(&mut reader); - assert_eq!(decoded, payload); -} - -#[test] -fn heartbeat_with_no_progress_yet_uses_the_zero_sentinel() { - let payload = Heartbeat { - last_completed_candidate_id: 0, - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = Heartbeat::decode(&mut reader); - assert_eq!(decoded, payload); -} - -#[test] -fn heartbeat_decodes_an_old_peers_empty_payload_as_the_zero_sentinel() { - let decoded = Heartbeat::decode(&mut Reader::new(&[])); - assert_eq!(decoded, Heartbeat { - last_completed_candidate_id: 0 - }); -} - -#[test] -fn job_resume_encodes_to_empty_bytes() { - let payload = JobResume; - assert!(payload.encode().is_empty()); - let decoded = JobResume::decode(); - assert_eq!(decoded, JobResume); -} - -#[test] -fn job_submit_round_trips_arbitrary_json_bytes() { - let payload = JobSubmit { - job_spec_json: br#"{"source_id":"s","root":"C:\\","query":"*.txt"}"#.to_vec(), - }; - let bytes = payload.encode(); - let decoded = JobSubmit::decode(&bytes); - assert_eq!(decoded, payload); -} - -#[test] -fn job_submit_decodes_empty_payload_as_empty_json_bytes() { - let decoded = JobSubmit::decode(&[]); - assert_eq!(decoded, JobSubmit { - job_spec_json: Vec::new() - }); -} - -#[test] -fn job_cancel_round_trips() { - let payload = JobCancel { - reason: "user requested cancellation".to_owned(), - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = JobCancel::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); -} - -#[test] -fn window_update_round_trips() { - let payload = WindowUpdate { - additional_window_bytes: 1_048_576, - }; - let bytes = payload.encode(); - let mut reader = Reader::new(&bytes); - let decoded = WindowUpdate::decode(&mut reader).unwrap(); - assert_eq!(decoded, payload); -} - -#[test] -fn full_frame_round_trip_job_begin_inside_envelope() { - // End-to-end: a real JobBegin payload framed by a real envelope, - // exactly the shape that crosses the wire to Docenta. - let job_begin = sample_job_begin(); - let payload_bytes = job_begin.encode(); - let envelope = sample_envelope(FrameType::JobBegin, 0); - let frame_bytes = envelope.encode(&payload_bytes); - - let mut reader = Reader::new(&frame_bytes); - let (decoded_envelope, decoded_payload) = - FrameEnvelope::decode(&mut reader, 1_000_000).unwrap(); - assert_eq!(decoded_envelope.frame_type, FrameType::JobBegin); - - let mut payload_reader = Reader::new(&decoded_payload); - let decoded_job_begin = JobBegin::decode(&mut payload_reader).unwrap(); - assert_eq!(decoded_job_begin, job_begin); -} diff --git a/crates/uffs-content-protocol/src/lib.rs b/crates/uffs-content-protocol/src/lib.rs deleted file mode 100644 index 091ffa04d..000000000 --- a/crates/uffs-content-protocol/src/lib.rs +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Wire protocol between the UFFS Content Service (producer) and a -//! downstream content consumer such as Docenta. -//! -//! This is a dedicated cross-platform Layer-0 library โ€” pure enum/struct -//! definitions and (eventually) byte-shuffling, no I/O, no Windows FFI, no -//! VSS/MFT access. Both sides of the wire (the `uffs-content` coordinator -//! process and any unprivileged consumer) import the types defined here so -//! the wire format has a single source of truth, matching the pattern -//! `uffs-broker-protocol` already established for the Access Broker. -//! -//! # Design references -//! -//! (all under `docs/dev/architecture/` โ€” local-only, not tracked in git) -//! -//! - `content-stream-tool-design.md` โ€” the original, VSS-less design sketch. -//! - `uffs-content-stream-enterprise-design-review.md` โ€” the replacement-design -//! review superseding that sketch: content-delivery protocol independent of -//! read mode, logical file-ID reads as the default, raw/snapshot extent reads -//! demoted to an optional internal acceleration behind a narrow privileged -//! helper (never the public coordinator). -//! - Docenta's `uffs-ingest-protocol-v2-vss.md` โ€” the settled v2 contract this -//! crate's types are scaffolded from: one VSS snapshot per job, an immutable -//! candidate manifest, a framed chunked content stream, and a durable failure -//! bucket. -//! -//! # Status -//! -//! Under active implementation per -//! `docs/dev/architecture/uffs-ingest-implementation-plan.md` (local-only, -//! UFI.0). [`codec`] (bounds-checked LE primitives + checksums) and -//! [`state`] are implemented; the manifest header/record/trailer layout -//! (design-doc ยง11) and the frame envelope + frame types (ยง12) are next. - -pub mod codec; -pub mod error; -pub mod frame; -pub mod manifest; -pub mod path_encoding; -pub mod state; - -/// Named pipe the Content Coordinator's **data** channel listens on. -/// -/// `JOB_BEGIN`/`FILE_BEGIN`/`CONTENT_CHUNK`/`FILE_END`/`FILE_FAILED`/ -/// `FILE_DEFERRED`/`JOB_END` โ€” the content stream itself, producer to -/// consumer. Kept on a separate pipe from [`COMMAND_PIPE_NAME`] so a -/// large in-flight `CONTENT_CHUNK` write can never head-of-line-block a -/// `WINDOW_UPDATE`/`FILE_ACK`/`JOB_CANCEL` the consumer needs to send -/// promptly (named pipes have no per-message-type multiplexing the way -/// HTTP/2 streams do, so that separation has to be a second pipe). -pub const DATA_PIPE_NAME: &str = r"\\.\pipe\uffs-content-data"; - -/// Named pipe the Content Coordinator's **command** channel listens on. -/// -/// Job submission/[`frame::JobResume`] (consumer to producer), -/// [`frame::WindowUpdate`]/[`frame::FileAck`]/[`frame::JobCancel`] -/// (consumer to producer), and [`frame::Progress`]/[`frame::Heartbeat`] -/// (producer to consumer). Always low-volume regardless of job size, so -/// it stays responsive even while [`DATA_PIPE_NAME`] is saturated with a -/// huge file's content. -pub const COMMAND_PIPE_NAME: &str = r"\\.\pipe\uffs-content-command"; diff --git a/crates/uffs-content-protocol/src/manifest.rs b/crates/uffs-content-protocol/src/manifest.rs deleted file mode 100644 index 02e35af5a..000000000 --- a/crates/uffs-content-protocol/src/manifest.rs +++ /dev/null @@ -1,764 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Candidate manifest: header, per-candidate record, and trailer. -//! -//! Design-doc ยง11. Every length-prefixed field's declared length is -//! bounds-checked before allocation (see [`crate::codec::Reader`]); every -//! checksum is verified before the decoded value is trusted. - -use crate::codec::{ - Digest, Reader, checksum32, digest, write_bytes_u16_prefixed, write_i64_le, write_u16_le, - write_u32_le, write_u64_le, -}; -use crate::path_encoding::{MAX_PATH_CODE_UNITS, PathDecodeError, WindowsPath}; - -/// Manifest header magic (design-doc ยง11.1). -pub const MANIFEST_MAGIC: [u8; 4] = *b"UFM2"; -/// Manifest trailer end-magic (design-doc ยง11.3). -pub const MANIFEST_END_MAGIC: [u8; 4] = *b"UFE2"; - -/// Wire-safety bound on `volume_guid`/`snapshot_id` byte length. Both are -/// small opaque identifiers in practice (a GUID string is ~36 bytes); this -/// is generous headroom, not an observed real-world size. -pub const MAX_IDENTIFIER_BYTES: u16 = 512; - -/// Wire-safety bound on an encoded path's byte length: two bytes per -/// UTF-16 code unit, [`MAX_PATH_CODE_UNITS`] code units. -pub const MAX_PATH_BYTES: u32 = (MAX_PATH_CODE_UNITS as u32) * 2; - -/// `authorization_mode` (design-doc ยง2.7/ยง17). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum AuthorizationMode { - /// Version 1: administrator-authorized export. No per-file ACL - /// equivalence โ€” see design-doc ยง2.7 and the addendum's ยง7 scope - /// restriction to a single-user, local-admin deployment. - AdminExport = 0, - /// Future: the producer applies the authenticated caller's effective - /// Windows access token to candidate visibility and content - /// authorization (design-doc ยง17.2, addendum ยง7.3/ยง7.4). - CallerToken = 1, -} - -impl AuthorizationMode { - /// Serialize to the single-byte wire representation. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the single-byte wire representation. - /// - /// # Errors - /// - /// Returns the offending byte if it does not match a known variant. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::AdminExport), - 1 => Ok(Self::CallerToken), - other => Err(other), - } - } -} - -bitflags::bitflags! { - /// Candidate flags (design-doc ยง11.4): "facts or planning hints, not - /// guaranteed processing success." - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] - pub struct CandidateFlags: u32 { - /// Unnamed data is MFT-resident. - const RESIDENT = 1 << 0; - /// Unnamed data is nonresident (has a runlist). - const NONRESIDENT = 1 << 1; - /// Stream has a sparse layout. - const SPARSE = 1 << 2; - /// Stream is NTFS-compressed. - const COMPRESSED = 1 << 3; - /// Stream is EFS-encrypted. - const ENCRYPTED = 1 << 4; - /// Object is reparse-point-backed. - const REPARSE = 1 << 5; - /// Object is Data-Dedup-optimized or otherwise provider-backed. - const DEDUP_OR_PROVIDER = 1 << 6; - /// Logical size exceeds the producer's "large file" threshold. - const LARGE_FILE = 1 << 7; - /// Heuristically likely to require a manual handler even if not - /// yet classified as such. - const MANUAL_HANDLER_LIKELY = 1 << 8; - } -} - -/// Errors decoding a manifest header, candidate record, or trailer. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[non_exhaustive] -pub enum ManifestError { - /// Underlying bounds/length-prefix decode failure. - #[error(transparent)] - Decode(#[from] crate::codec::DecodeError), - /// Path field failed to decode. - #[error(transparent)] - Path(#[from] PathDecodeError), - /// Header or trailer magic did not match the expected constant. - #[error("bad magic: expected {expected:?}, got {actual:?}")] - BadMagic { - /// Expected magic bytes. - expected: [u8; 4], - /// Magic bytes actually present. - actual: [u8; 4], - }, - /// The declared `header_length` did not match the number of bytes - /// actually consumed while decoding the header. - #[error("header_length mismatch: declared {declared}, actual {actual}")] - HeaderLengthMismatch { - /// Declared length from the wire. - declared: u16, - /// Bytes actually consumed decoding the header. - actual: usize, - }, - /// The declared `record_length` did not match the number of bytes - /// actually consumed while decoding the candidate record. - #[error("record_length mismatch: declared {declared}, actual {actual}")] - RecordLengthMismatch { - /// Declared length from the wire. - declared: u32, - /// Bytes actually consumed decoding the record. - actual: usize, - }, - /// A header/record checksum did not match the bytes it covers. - #[error("checksum mismatch: expected 0x{expected:08x}, computed 0x{computed:08x}")] - ChecksumMismatch { - /// Checksum read from the wire. - expected: u32, - /// Checksum recomputed locally. - computed: u32, - }, - /// `authorization_mode` byte did not match a known variant. - #[error("unknown authorization_mode byte: {0}")] - UnknownAuthorizationMode(u8), - /// The trailer's `candidate_count_repeat` did not match the header's - /// `candidate_count`. - #[error("candidate_count mismatch: header {header}, trailer {trailer}")] - CandidateCountMismatch { - /// Value from the header. - header: u64, - /// Value from the trailer. - trailer: u64, - }, -} - -/// Manifest header (design-doc ยง11.1). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ManifestHeader { - /// Wire format version. - pub format_version: u16, - /// Job identifier (UUID bytes). - pub job_id: [u8; 16], - /// Source identifier (UUID bytes). - pub source_id: [u8; 16], - /// NTFS volume serial number. - pub volume_serial: u64, - /// Opaque volume GUID bytes. - pub volume_guid: Vec, - /// Opaque VSS snapshot identifier bytes. - pub snapshot_id: Vec, - /// Snapshot creation time, Unix milliseconds. - pub snapshot_created_unix_ms: i64, - /// Digest of the UFFS query that produced this candidate set. - pub query_digest: Digest, - /// Authorization model this job was authorized under. - pub authorization_mode: AuthorizationMode, - /// Total number of candidate records in the manifest. - pub candidate_count: u64, - /// Total byte length of the record section following this header. - pub record_section_length: u64, -} - -/// Bytes preceding the checksummed/length-counted header content: -/// 4 (magic) + 2 (`format_version`) + 2 (`header_length`). -const HEADER_PREFIX_LEN: usize = 8; - -impl ManifestHeader { - /// Encode this header, computing `header_length` and - /// `header_checksum` automatically. - /// - /// # Errors - /// - /// Returns `Err` if the encoded header would exceed `u16::MAX` bytes - /// (only possible with an implausibly large `volume_guid`/`snapshot_id`). - pub fn encode(&self) -> Result, ManifestError> { - let mut content = Vec::new(); - content.extend_from_slice(&self.job_id); - content.extend_from_slice(&self.source_id); - write_u64_le(&mut content, self.volume_serial); - write_bytes_u16_prefixed(&mut content, &self.volume_guid); - write_bytes_u16_prefixed(&mut content, &self.snapshot_id); - write_i64_le(&mut content, self.snapshot_created_unix_ms); - content.extend_from_slice(&self.query_digest); - content.push(self.authorization_mode.encode()); - write_u64_le(&mut content, self.candidate_count); - write_u64_le(&mut content, self.record_section_length); - - // `header_length` covers exactly the bytes `decode` measures as - // `consumed` (magic..record_section_length) โ€” it must NOT include - // the trailing `header_checksum`, which `decode` reads and - // verifies separately, after computing `consumed`. - let header_length_value = HEADER_PREFIX_LEN.checked_add(content.len()).ok_or( - ManifestError::HeaderLengthMismatch { - declared: 0, - actual: usize::MAX, - }, - )?; - let header_length = u16::try_from(header_length_value).map_err(|_err| { - ManifestError::HeaderLengthMismatch { - declared: u16::MAX, - actual: header_length_value, - } - })?; - - let mut checked = Vec::with_capacity(header_length_value + 4); - checked.extend_from_slice(&MANIFEST_MAGIC); - write_u16_le(&mut checked, self.format_version); - write_u16_le(&mut checked, header_length); - checked.extend_from_slice(&content); - let checksum = checksum32(&checked); - - let mut out = checked; - write_u32_le(&mut out, checksum); - Ok(out) - } - - /// Decode a manifest header from `reader`. - /// - /// # Errors - /// - /// See [`ManifestError`] variants: bad magic, length/checksum - /// mismatch, unknown authorization mode, or an underlying bounds - /// failure. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let start = reader.position(); - - let magic: [u8; 4] = reader.read_array()?; - if magic != MANIFEST_MAGIC { - return Err(ManifestError::BadMagic { - expected: MANIFEST_MAGIC, - actual: magic, - }); - } - let format_version = reader.read_u16_le()?; - let header_length = reader.read_u16_le()?; - - let job_id: [u8; 16] = reader.read_array()?; - let source_id: [u8; 16] = reader.read_array()?; - let volume_serial = reader.read_u64_le()?; - let volume_guid = reader.read_bytes_u16_prefixed("volume_guid", MAX_IDENTIFIER_BYTES)?; - let snapshot_id = reader.read_bytes_u16_prefixed("snapshot_id", MAX_IDENTIFIER_BYTES)?; - let snapshot_created_unix_ms = reader.read_i64_le()?; - let query_digest: Digest = reader.read_array()?; - let authorization_byte = reader.read_u8()?; - let authorization_mode = AuthorizationMode::decode(authorization_byte) - .map_err(ManifestError::UnknownAuthorizationMode)?; - let candidate_count = reader.read_u64_le()?; - let record_section_length = reader.read_u64_le()?; - - let end = reader.position(); - let consumed = end - start; - if consumed != header_length as usize { - return Err(ManifestError::HeaderLengthMismatch { - declared: header_length, - actual: consumed, - }); - } - - let expected_checksum = reader.read_u32_le()?; - let header_bytes = - reader - .full_buffer() - .get(start..end) - .ok_or(ManifestError::HeaderLengthMismatch { - declared: header_length, - actual: consumed, - })?; - let computed_checksum = checksum32(header_bytes); - if expected_checksum != computed_checksum { - return Err(ManifestError::ChecksumMismatch { - expected: expected_checksum, - computed: computed_checksum, - }); - } - - Ok(Self { - format_version, - job_id, - source_id, - volume_serial, - volume_guid, - snapshot_id, - snapshot_created_unix_ms, - query_digest, - authorization_mode, - candidate_count, - record_section_length, - }) - } -} - -/// One candidate manifest record (design-doc ยง11.2/ยง5.2/ยง5.5). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CandidateRecord { - /// Unique identifier for this candidate within this job. - pub candidate_id: u64, - /// Full NTFS file reference (MFT index + sequence number packed per - /// the platform's native 64-bit file-ID layout) โ€” never a bare MFT - /// record index (design-doc ยง5.2, enterprise-review Finding C4). - pub file_reference: u64, - /// Logical size at snapshot time. - pub logical_size: u64, - /// Valid Data Length at snapshot time. - pub valid_data_length: u64, - /// Modification time, Unix milliseconds. - pub mtime_unix_ms: i64, - /// Planning-hint flags (design-doc ยง11.4). - pub candidate_flags: CandidateFlags, - /// Lossless Windows path. - pub path: WindowsPath, -} - -/// Bytes contributed by the fixed-width fields preceding the -/// variable-length path: `candidate_id`(8) + `file_reference`(8) + -/// `logical_size`(8) + `valid_data_length`(8) + `mtime_unix_ms`(8) + -/// `candidate_flags`(4) = 44. Does not include `record_length` (4) or -/// `record_checksum` (4), which are added separately below. -const RECORD_FIXED_CONTENT_LEN: usize = 44; - -impl CandidateRecord { - /// Encode this record, computing `record_length` and - /// `record_checksum` automatically. - /// - /// # Errors - /// - /// Returns `Err` if the encoded record would exceed `u32::MAX` bytes. - pub fn encode(&self) -> Result, ManifestError> { - let mut content = Vec::with_capacity(RECORD_FIXED_CONTENT_LEN); - write_u64_le(&mut content, self.candidate_id); - write_u64_le(&mut content, self.file_reference); - write_u64_le(&mut content, self.logical_size); - write_u64_le(&mut content, self.valid_data_length); - write_i64_le(&mut content, self.mtime_unix_ms); - write_u32_le(&mut content, self.candidate_flags.bits()); - self.path.encode(&mut content); - - // `record_length` covers exactly the bytes `decode` measures as - // `consumed` (record_length field itself..end of path) โ€” it must - // NOT include the trailing `record_checksum`, which `decode` - // reads and verifies separately, after computing `consumed`. - let record_length_value = content - .len() - .checked_add(4) // + record_length field itself - .ok_or(ManifestError::RecordLengthMismatch { - declared: 0, - actual: usize::MAX, - })?; - let record_length = u32::try_from(record_length_value).map_err(|_err| { - ManifestError::RecordLengthMismatch { - declared: u32::MAX, - actual: record_length_value, - } - })?; - - let mut checked = Vec::with_capacity(record_length_value + 4); - write_u32_le(&mut checked, record_length); - checked.extend_from_slice(&content); - let checksum = checksum32(&checked); - - let mut out = checked; - write_u32_le(&mut out, checksum); - Ok(out) - } - - /// Decode a candidate record from `reader`. - /// - /// # Errors - /// - /// See [`ManifestError`] variants: length/checksum mismatch, a - /// malformed path, or an underlying bounds failure. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let start = reader.position(); - let record_length = reader.read_u32_le()?; - - let candidate_id = reader.read_u64_le()?; - let file_reference = reader.read_u64_le()?; - let logical_size = reader.read_u64_le()?; - let valid_data_length = reader.read_u64_le()?; - let mtime_unix_ms = reader.read_i64_le()?; - let flags_bits = reader.read_u32_le()?; - let candidate_flags = CandidateFlags::from_bits_truncate(flags_bits); - let path = WindowsPath::decode(reader, MAX_PATH_BYTES)?; - - let end = reader.position(); - let consumed = end - start; - if consumed != record_length as usize { - return Err(ManifestError::RecordLengthMismatch { - declared: record_length, - actual: consumed, - }); - } - - let expected_checksum = reader.read_u32_le()?; - let record_bytes = - reader - .full_buffer() - .get(start..end) - .ok_or(ManifestError::RecordLengthMismatch { - declared: record_length, - actual: consumed, - })?; - let computed_checksum = checksum32(record_bytes); - if expected_checksum != computed_checksum { - return Err(ManifestError::ChecksumMismatch { - expected: expected_checksum, - computed: computed_checksum, - }); - } - - Ok(Self { - candidate_id, - file_reference, - logical_size, - valid_data_length, - mtime_unix_ms, - candidate_flags, - path, - }) - } -} - -/// Manifest trailer (design-doc ยง11.3). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ManifestTrailer { - /// Repeats the header's `candidate_count`, so a streaming consumer - /// can validate completeness without holding the header in memory. - pub candidate_count_repeat: u64, - /// BLAKE3 digest of the entire manifest (header + record section) - /// preceding this trailer. - pub manifest_digest: Digest, -} - -impl ManifestTrailer { - /// Encode this trailer. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - write_u64_le(&mut out, self.candidate_count_repeat); - out.extend_from_slice(&self.manifest_digest); - out.extend_from_slice(&MANIFEST_END_MAGIC); - out - } - - /// Decode a trailer from `reader`. - /// - /// # Errors - /// - /// [`ManifestError::BadMagic`] if `end_magic` does not match, or an - /// underlying bounds failure. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let candidate_count_repeat = reader.read_u64_le()?; - let manifest_digest: Digest = reader.read_array()?; - let end_magic: [u8; 4] = reader.read_array()?; - if end_magic != MANIFEST_END_MAGIC { - return Err(ManifestError::BadMagic { - expected: MANIFEST_END_MAGIC, - actual: end_magic, - }); - } - Ok(Self { - candidate_count_repeat, - manifest_digest, - }) - } - - /// Compute the trailer's `manifest_digest` over `manifest_bytes` - /// (header + record section, exactly as they appear on the wire - /// preceding the trailer). - #[must_use] - pub fn compute_digest(manifest_bytes: &[u8]) -> Digest { - digest(manifest_bytes) - } -} - -#[cfg(test)] -mod tests { - use proptest::prelude::*; - - use super::{ - AuthorizationMode, CandidateFlags, CandidateRecord, ManifestError, ManifestHeader, - ManifestTrailer, - }; - use crate::codec::Reader; - use crate::path_encoding::WindowsPath; - - fn sample_header() -> ManifestHeader { - ManifestHeader { - format_version: 2, - job_id: [1_u8; 16], - source_id: [2_u8; 16], - volume_serial: 0x1234_5678_9ABC_DEF0, - volume_guid: b"{11111111-2222-3333-4444-555555555555}".to_vec(), - snapshot_id: b"snap-0001".to_vec(), - snapshot_created_unix_ms: 1_752_000_000_000, - query_digest: [7_u8; 32], - authorization_mode: AuthorizationMode::AdminExport, - candidate_count: 3, - record_section_length: 999, - } - } - - fn sample_record(candidate_id: u64) -> CandidateRecord { - CandidateRecord { - candidate_id, - file_reference: 0xABCD_EF01_2345_6789, - logical_size: 4096, - valid_data_length: 4096, - mtime_unix_ms: 1_752_000_000_000, - candidate_flags: CandidateFlags::NONRESIDENT | CandidateFlags::LARGE_FILE, - path: WindowsPath::from_str_lossless(r"C:\Users\robert\data\file.bin"), - } - } - - #[test] - fn header_round_trips() { - let header = sample_header(); - let bytes = header.encode().unwrap(); - let mut reader = Reader::new(&bytes); - let decoded = ManifestHeader::decode(&mut reader).unwrap(); - assert_eq!(decoded, header); - assert_eq!( - reader.remaining(), - 0, - "decode must consume the whole header" - ); - } - - #[test] - #[expect( - clippy::indexing_slicing, - reason = "test mutation of a known, already-validated buffer index; \ - clippy::get_unwrap is also denied, so a scoped exception on \ - direct indexing is the established pattern for this \ - conflict (see crates/uffs-daemon/tests/ipc_integration.rs)" - )] - fn header_rejects_bad_magic() { - let header = sample_header(); - let mut bytes = header.encode().unwrap(); - bytes[0] = b'X'; - let mut reader = Reader::new(&bytes); - let err = ManifestHeader::decode(&mut reader).unwrap_err(); - assert!(matches!(err, ManifestError::BadMagic { .. })); - } - - #[test] - #[expect( - clippy::indexing_slicing, - reason = "test mutation of a known, already-validated buffer index; \ - clippy::get_unwrap is also denied, so a scoped exception on \ - direct indexing is the established pattern for this \ - conflict (see crates/uffs-daemon/tests/ipc_integration.rs)" - )] - fn header_rejects_flipped_byte_via_checksum() { - let header = sample_header(); - let mut bytes = header.encode().unwrap(); - // Flip a byte inside the checksummed region (well past the magic - // and length fields, inside `job_id`). - let flip_index = 10; - bytes[flip_index] ^= 0xFF; - let mut reader = Reader::new(&bytes); - let err = ManifestHeader::decode(&mut reader).unwrap_err(); - assert!(matches!(err, ManifestError::ChecksumMismatch { .. })); - } - - #[test] - fn header_rejects_unknown_authorization_mode() { - let header = sample_header(); - let bytes = header.encode().unwrap(); - // Re-encode is awkward to mutate mid-structure directly since the - // checksum covers it; instead, hand-verify the decode path - // rejects an out-of-range byte by constructing a header with a - // manually patched authorization byte and a recomputed checksum - // would require re-implementing encode. Simplest robust check: - // AuthorizationMode::decode itself rejects out-of-range bytes, - // exercised directly. - assert_eq!(AuthorizationMode::decode(2), Err(2)); - // Sanity: the real header still round-trips (guards against a - // future refactor accidentally breaking the happy path while - // "testing" the unhappy one above). - let mut reader = Reader::new(&bytes); - ManifestHeader::decode(&mut reader).unwrap(); - } - - #[test] - fn record_round_trips() { - let record = sample_record(42); - let bytes = record.encode().unwrap(); - let mut reader = Reader::new(&bytes); - let decoded = CandidateRecord::decode(&mut reader).unwrap(); - assert_eq!(decoded, record); - assert_eq!( - reader.remaining(), - 0, - "decode must consume the whole record" - ); - } - - #[test] - #[expect( - clippy::indexing_slicing, - reason = "test mutation of a known, already-validated buffer index; \ - clippy::get_unwrap is also denied, so a scoped exception on \ - direct indexing is the established pattern for this \ - conflict (see crates/uffs-daemon/tests/ipc_integration.rs)" - )] - fn record_rejects_flipped_byte_via_checksum() { - let record = sample_record(1); - let mut bytes = record.encode().unwrap(); - let last = bytes.len() - 5; // inside the record body, before the trailing checksum - bytes[last] ^= 0xFF; - let mut reader = Reader::new(&bytes); - let err = CandidateRecord::decode(&mut reader).unwrap_err(); - assert!(matches!(err, ManifestError::ChecksumMismatch { .. })); - } - - #[test] - fn record_flags_round_trip_bit_pattern() { - let mut record = sample_record(7); - record.candidate_flags = CandidateFlags::RESIDENT - | CandidateFlags::SPARSE - | CandidateFlags::MANUAL_HANDLER_LIKELY; - let bytes = record.encode().unwrap(); - let mut reader = Reader::new(&bytes); - let decoded = CandidateRecord::decode(&mut reader).unwrap(); - assert_eq!(decoded.candidate_flags, record.candidate_flags); - } - - #[test] - fn trailer_round_trips() { - let trailer = ManifestTrailer { - candidate_count_repeat: 3, - manifest_digest: [9_u8; 32], - }; - let bytes = trailer.encode(); - let mut reader = Reader::new(&bytes); - let decoded = ManifestTrailer::decode(&mut reader).unwrap(); - assert_eq!(decoded, trailer); - } - - #[test] - #[expect( - clippy::indexing_slicing, - reason = "test mutation of a known, already-validated buffer index; \ - clippy::get_unwrap is also denied, so a scoped exception on \ - direct indexing is the established pattern for this \ - conflict (see crates/uffs-daemon/tests/ipc_integration.rs)" - )] - fn trailer_rejects_bad_end_magic() { - let trailer = ManifestTrailer { - candidate_count_repeat: 1, - manifest_digest: [0_u8; 32], - }; - let mut bytes = trailer.encode(); - let last_index = bytes.len() - 1; - bytes[last_index] = b'?'; - let mut reader = Reader::new(&bytes); - let err = ManifestTrailer::decode(&mut reader).unwrap_err(); - assert!(matches!(err, ManifestError::BadMagic { .. })); - } - - #[test] - fn full_manifest_round_trip_with_multiple_candidates() { - // End-to-end: header + N records + trailer, exactly the shape a - // real manifest file has on disk, decoded back sequentially from - // one contiguous buffer. - let candidates: Vec = (0..5).map(sample_record).collect(); - - let mut record_bytes = Vec::new(); - for candidate in &candidates { - record_bytes.extend_from_slice(&candidate.encode().unwrap()); - } - - let mut header = sample_header(); - header.candidate_count = candidates.len() as u64; - header.record_section_length = record_bytes.len() as u64; - let mut manifest_bytes = header.encode().unwrap(); - manifest_bytes.extend_from_slice(&record_bytes); - - let trailer = ManifestTrailer { - candidate_count_repeat: header.candidate_count, - manifest_digest: ManifestTrailer::compute_digest(&manifest_bytes), - }; - manifest_bytes.extend_from_slice(&trailer.encode()); - - // Now decode the whole thing back. - let mut reader = Reader::new(&manifest_bytes); - let decoded_header = ManifestHeader::decode(&mut reader).unwrap(); - assert_eq!(decoded_header, header); - - let mut decoded_candidates = Vec::new(); - for _ in 0..decoded_header.candidate_count { - decoded_candidates.push(CandidateRecord::decode(&mut reader).unwrap()); - } - assert_eq!(decoded_candidates, candidates); - - let decoded_trailer = ManifestTrailer::decode(&mut reader).unwrap(); - assert_eq!( - decoded_trailer.candidate_count_repeat, - header.candidate_count - ); - assert_eq!( - reader.remaining(), - 0, - "trailer must be the last thing in the manifest" - ); - - // Completeness invariant sanity: every decoded candidate_id is - // unique and matches what was encoded (design-doc ยง21.7). - let mut ids: Vec = decoded_candidates - .iter() - .map(|candidate| candidate.candidate_id) - .collect(); - ids.sort_unstable(); - ids.dedup(); - assert_eq!( - ids.len(), - decoded_candidates.len(), - "candidate_id must be unique" - ); - } - - proptest! { - #![proptest_config(ProptestConfig::with_cases(200))] - - #[test] - fn candidate_record_round_trips_for_arbitrary_fields( - candidate_id: u64, - file_reference: u64, - logical_size: u64, - valid_data_length: u64, - mtime_unix_ms: i64, - flags_bits: u32, - path_str in "[a-zA-Z0-9_/\\\\:. ]{0,200}", - ) { - let record = CandidateRecord { - candidate_id, - file_reference, - logical_size, - valid_data_length, - mtime_unix_ms, - candidate_flags: CandidateFlags::from_bits_truncate(flags_bits), - path: WindowsPath::from_str_lossless(&path_str), - }; - let bytes = record.encode().unwrap(); - let mut reader = Reader::new(&bytes); - let decoded = CandidateRecord::decode(&mut reader).unwrap(); - prop_assert_eq!(decoded, record); - } - } -} diff --git a/crates/uffs-content-protocol/src/path_encoding.rs b/crates/uffs-content-protocol/src/path_encoding.rs deleted file mode 100644 index d00c57ee5..000000000 --- a/crates/uffs-content-protocol/src/path_encoding.rs +++ /dev/null @@ -1,291 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Lossless Windows path representation (design-doc ยง5.4). -//! -//! "The authoritative Windows path representation MUST be lossless. The -//! manifest stores either UTF-16LE path code units; or another explicitly -//! lossless Windows path encoding such as WTF-8. A lossy UTF-8 display -//! path MAY also be included for logs and UI, but it MUST NOT be the sole -//! identity or unique key." -//! -//! This module models the authoritative form as raw UTF-16LE code units -//! (`Vec`), because that is exactly what `GetFileInformationByHandleEx` -//! / NTFS directory entries hand back โ€” no intermediate lossy conversion -//! ever has to happen on the producer side. A Windows path can contain -//! unpaired surrogate code units (rare, but real โ€” some tools and legacy -//! software create them); those are the values a `String`-based -//! representation cannot hold at all, which is exactly why the wire -//! format uses raw code units instead of `String`/`OsString`. - -use crate::codec::Reader; - -/// The path-encoding discriminant carried on the wire (design-doc ยง11.2 -/// `path_encoding` field). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum PathEncoding { - /// Raw UTF-16LE code units, exactly as returned by the Windows API. - /// May contain unpaired surrogates. - Utf16Le = 0, -} - -impl PathEncoding { - /// Serialize to the single-byte wire representation. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the single-byte wire representation. - /// - /// # Errors - /// - /// Returns `Err` with the offending byte if it does not match a known - /// encoding. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::Utf16Le), - other => Err(other), - } - } -} - -/// A lossless Windows path: raw UTF-16LE code units plus a cached lossy -/// UTF-8 display form. -/// -/// The lossy `display` string exists only for logs/UI (design-doc ยง5.4) -/// and MUST NOT be used as an identity key โ€” every comparison and every -/// wire round-trip in this crate operates on `code_units`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct WindowsPath { - /// Authoritative, lossless UTF-16LE code units. - code_units: Vec, -} - -/// Maximum path length in UTF-16 code units this crate will decode. -/// -/// Windows itself has historically allowed paths well beyond `MAX_PATH` -/// (260) via the `\\?\` prefix and long-path opt-in; this is a generous -/// wire-safety bound, not a Windows API limit. -pub const MAX_PATH_CODE_UNITS: u16 = 32_767; - -impl WindowsPath { - /// Build a [`WindowsPath`] from raw UTF-16 code units (e.g. as - /// returned by a Windows API call). No validation is performed here - /// beyond what the type system already guarantees โ€” unpaired - /// surrogates are accepted, matching real-world Windows path data. - #[must_use] - pub const fn from_code_units(code_units: Vec) -> Self { - Self { code_units } - } - - /// Build a [`WindowsPath`] from a Rust `&str`. Since `str` is - /// guaranteed valid UTF-8 (and therefore representable in UTF-16 - /// without unpaired surrogates), this conversion is always lossless. - #[must_use] - pub fn from_str_lossless(value: &str) -> Self { - Self { - code_units: value.encode_utf16().collect(), - } - } - - /// The raw UTF-16LE code units. - #[must_use] - pub fn code_units(&self) -> &[u16] { - &self.code_units - } - - /// A lossy UTF-8 display form, for logs/UI only. Unpaired surrogates - /// are replaced with U+FFFD, per [`String::from_utf16_lossy`]-equivalent - /// semantics (implemented via [`char::decode_utf16`] directly so the - /// replacement policy is explicit and testable here rather than - /// inherited implicitly from a std helper). - #[must_use] - pub fn display_lossy(&self) -> String { - char::decode_utf16(self.code_units.iter().copied()) - .map(|result| result.unwrap_or(char::REPLACEMENT_CHARACTER)) - .collect() - } - - /// Encode as a wire `(path_encoding: u8, path_length: u32, path: bytes)` - /// triple (design-doc ยง11.2). The length prefix counts *bytes*, not - /// code units โ€” two bytes per UTF-16 code unit. - pub fn encode(&self, out: &mut Vec) { - out.push(PathEncoding::Utf16Le.encode()); - let mut byte_buf = Vec::with_capacity(self.code_units.len() * 2); - for unit in &self.code_units { - byte_buf.extend_from_slice(&unit.to_le_bytes()); - } - // `write_bytes_u32_prefixed` is the ยง11.2 layout (u32 path_length); - // reused here via the u16-style helper's u32 sibling would be - // clearer, but ยง11.2 explicitly specifies `path_length u32` โ€” use - // that variant directly. - crate::codec::write_bytes_u32_prefixed(out, &byte_buf); - } - - /// Decode from a `(path_encoding, path_length, path)` wire triple. - /// - /// `max_bytes` bounds the length-prefixed path payload before any - /// allocation (Finding H10 discipline, same as every other - /// length-prefixed field in this crate). - /// - /// # Errors - /// - /// - a [`crate::codec::DecodeError`] if the bytes are truncated or the - /// declared length exceeds `max_bytes`; - /// - `Err` wrapping the raw byte if `path_encoding` is not a known - /// [`PathEncoding`] variant (surfaced as - /// [`crate::codec::DecodeError::UnknownDiscriminant`]); - /// - `Err` if the payload's byte length is odd (not a whole number of - /// UTF-16 code units). - pub fn decode(reader: &mut Reader<'_>, max_bytes: u32) -> Result { - let encoding_byte = reader.read_u8()?; - PathEncoding::decode(encoding_byte).map_err(PathDecodeError::UnsupportedEncoding)?; - let bytes = reader.read_bytes_u32_prefixed("path", max_bytes)?; - if bytes.len() % 2 != 0 { - return Err(PathDecodeError::OddByteLength(bytes.len())); - } - #[expect( - clippy::chunks_exact_to_as_chunks, - reason = "slice::as_chunks is nightly-unstable library API; adopting it would \ - require an unstable #![feature(...)] crate-root gate for one call site. \ - chunks_exact(2) is already panic-free here (see the try_into fallback \ - below), so there is no correctness reason to take on that commitment." - )] - let pairs = bytes.chunks_exact(2); - let code_units: Vec = pairs - .map(|pair| { - // `chunks_exact(2)` guarantees `pair.len() == 2`; `try_into` - // therefore never hits the fallback, but expressing it this - // way (instead of indexing) keeps this closure panic-free - // by construction rather than "panic-free because - // chunks_exact happens to guarantee it." - let array: [u8; 2] = pair.try_into().unwrap_or([0, 0]); - u16::from_le_bytes(array) - }) - .collect(); - Ok(Self { code_units }) - } -} - -/// Errors decoding a [`WindowsPath`] from the wire. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[non_exhaustive] -pub enum PathDecodeError { - /// Underlying bounds/length-prefix decode failure. - #[error(transparent)] - Decode(#[from] crate::codec::DecodeError), - /// The `path_encoding` byte did not match a known [`PathEncoding`] variant. - #[error("unsupported path encoding byte: {0}")] - UnsupportedEncoding(u8), - /// The path payload's byte length was odd (not a whole number of - /// UTF-16 code units). - #[error("path payload has odd byte length {0}, not a whole number of UTF-16 code units")] - OddByteLength(usize), -} - -#[cfg(test)] -mod tests { - use super::{PathDecodeError, PathEncoding, WindowsPath}; - use crate::codec::Reader; - - #[test] - fn ascii_path_round_trips() { - let path = WindowsPath::from_str_lossless(r"C:\Users\robert\file.txt"); - let mut buf = Vec::new(); - path.encode(&mut buf); - let mut reader = Reader::new(&buf); - let decoded = WindowsPath::decode(&mut reader, 10_000).unwrap(); - assert_eq!(decoded, path); - assert_eq!(decoded.display_lossy(), r"C:\Users\robert\file.txt"); - } - - #[test] - fn non_bmp_path_round_trips() { - // U+1F600 (๐Ÿ˜€) requires a UTF-16 surrogate pair โ€” this is exactly - // the class of path the enterprise-review commit history flagged - // as a past UTF-16 anti-pattern bug source. - let path = WindowsPath::from_str_lossless("C:\\notes\\๐Ÿ˜€.txt"); - let mut buf = Vec::new(); - path.encode(&mut buf); - let mut reader = Reader::new(&buf); - let decoded = WindowsPath::decode(&mut reader, 10_000).unwrap(); - assert_eq!(decoded, path); - assert_eq!(decoded.display_lossy(), "C:\\notes\\๐Ÿ˜€.txt"); - } - - #[test] - fn unpaired_surrogate_round_trips_losslessly_but_displays_as_replacement_char() { - // 0xD800 is a lone high surrogate โ€” not representable in `str` at - // all, which is exactly why the authoritative form is `Vec` - // and not `String`. It must still round-trip byte-for-byte. - let path = WindowsPath::from_code_units(vec![ - u16::from(b'C'), - u16::from(b':'), - u16::from(b'\\'), - 0xD800, - u16::from(b'x'), - ]); - let mut buf = Vec::new(); - path.encode(&mut buf); - let mut reader = Reader::new(&buf); - let decoded = WindowsPath::decode(&mut reader, 10_000).unwrap(); - assert_eq!( - decoded, path, - "lone surrogate must survive the wire round-trip exactly" - ); - assert!( - decoded.display_lossy().contains('\u{FFFD}'), - "lossy display must substitute the replacement character for the unpaired surrogate" - ); - } - - #[test] - fn empty_path_round_trips() { - let path = WindowsPath::from_code_units(vec![]); - let mut buf = Vec::new(); - path.encode(&mut buf); - let mut reader = Reader::new(&buf); - let decoded = WindowsPath::decode(&mut reader, 10_000).unwrap(); - assert_eq!(decoded.code_units(), &[] as &[u16]); - } - - #[test] - fn decode_rejects_declared_length_over_max_bytes() { - let path = WindowsPath::from_str_lossless(r"C:\a\long\enough\path.txt"); - let mut buf = Vec::new(); - path.encode(&mut buf); - let mut reader = Reader::new(&buf); - let err = WindowsPath::decode(&mut reader, 4).unwrap_err(); - assert!(matches!(err, PathDecodeError::Decode(_))); - } - - #[test] - fn decode_rejects_odd_byte_length_payload() { - // Hand-craft a wire triple with an odd-length payload: encoding - // byte, then a u32 length of 3, then 3 raw bytes. - let mut buf = Vec::new(); - buf.push(PathEncoding::Utf16Le.encode()); - crate::codec::write_bytes_u32_prefixed(&mut buf, &[1, 2, 3]); - let mut reader = Reader::new(&buf); - let err = WindowsPath::decode(&mut reader, 10_000).unwrap_err(); - assert_eq!(err, PathDecodeError::OddByteLength(3)); - } - - #[test] - fn decode_rejects_unknown_encoding_byte() { - let mut buf = Vec::new(); - buf.push(0xFF); // not a known PathEncoding discriminant - crate::codec::write_bytes_u32_prefixed(&mut buf, b""); - let mut reader = Reader::new(&buf); - let err = WindowsPath::decode(&mut reader, 10_000).unwrap_err(); - assert_eq!(err, PathDecodeError::UnsupportedEncoding(0xFF)); - } - - #[test] - fn path_encoding_round_trips() { - assert_eq!(PathEncoding::decode(0).unwrap(), PathEncoding::Utf16Le); - assert_eq!(PathEncoding::decode(1), Err(1)); - } -} diff --git a/crates/uffs-content-protocol/src/state.rs b/crates/uffs-content-protocol/src/state.rs deleted file mode 100644 index 506b12136..000000000 --- a/crates/uffs-content-protocol/src/state.rs +++ /dev/null @@ -1,212 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Job and candidate state machines (design-doc ยง9). - -/// One candidate's terminal outcome (design-doc ยง2.2, ยง9.2). -/// -/// Every candidate in a job's manifest MUST eventually reach exactly one -/// of these states, and the job is complete only once every candidate has -/// one. Only [`CandidateOutcome::Succeeded`] candidates are delivered as -/// content to the downstream consumer โ€” but every outcome, including the -/// three non-success ones, means the candidate is *present*, not deleted. -/// A consumer's reap/tombstone reconciliation MUST key off manifest -/// membership across all four outcomes, never off `Succeeded` alone: a -/// candidate that merely failed to read is not the same thing as a -/// candidate that no longer exists (design-doc ยง2.3). -/// -/// Filtering which files even become candidates (by size, extension, -/// path, etc.) is the query's job, not this enum's โ€” a consumer that -/// wants content only for files under some size threshold expresses that -/// as a filter on the UFFS query passed into the job, the same way any -/// other UFFS search filter works. This tool produces one manifest and -/// one content stream for whatever the query matched; it does not itself -/// decide per-candidate whether to deliver a body. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum CandidateOutcome { - /// Content was read, streamed, and verified successfully. - Succeeded, - /// A transient failure occurred; the candidate may be retried in a - /// later job attempt against a new snapshot. - FailedRetryable, - /// A permanent failure occurred; retrying will not help. - FailedTerminal, - /// The candidate was explicitly deferred to manual or later handling - /// (e.g. compressed/encrypted/reparse-backed files in v2). - DeferredManual, -} - -/// Job lifecycle state (design-doc ยง9.1). -/// -/// The legal transition graph is [`JobState::can_transition_to`] โ€” mirrors -/// the shape of `uffs-daemon`'s `ShardState::can_transition_to` -/// (`crates/uffs-daemon/src/cache/shard.rs`), the existing reviewed -/// pattern in this codebase for a small state machine with a proptested -/// transition graph. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum JobState { - /// Job row created; not yet authorized/validated. - Created, - /// Broker snapshot-lease creation is in flight. - SnapshotCreating, - /// Snapshot lease is ready; candidate enumeration has not started. - SnapshotReady, - /// Evaluating the UFFS query against the snapshot to build candidates. - Enumerating, - /// The candidate manifest is finalized and checksummed (design-doc - /// ยง4.1 step 8) โ€” completeness accounting starts being meaningful - /// from this state onward. - ManifestFinalized, - /// Candidates are being processed and streamed to the consumer. - Streaming, - /// All candidates have a terminal outcome; finalizing job-level - /// accounting and the `JOB_END` record. - Completing, - /// Terminal: every candidate succeeded. - Completed, - /// Terminal: every candidate reached a terminal outcome, but at least - /// one did not succeed (failed or was deferred). - CompletedWithFailures, - /// Terminal: the job was cancelled before all candidates reached a - /// terminal outcome. - Cancelled, - /// Terminal: the job failed as a whole before the manifest was - /// finalized (design-doc ยง4.2) โ€” no candidate-completeness claim is - /// made. - Aborted, -} - -impl JobState { - /// Returns true iff a transition `self -> to` is in the legal graph. - /// - /// Legal transitions: - /// * `Created` -> `SnapshotCreating`, `Aborted` - /// * `SnapshotCreating` -> `SnapshotReady`, `Aborted` - /// * `SnapshotReady` -> `Enumerating`, `Aborted` - /// * `Enumerating` -> `ManifestFinalized`, `Aborted` - /// * `ManifestFinalized` -> `Streaming`, `Cancelled` - /// * `Streaming` -> `Completing`, `Cancelled` - /// * `Completing` -> `Completed`, `CompletedWithFailures`, `Cancelled` - /// * `Completed`, `CompletedWithFailures`, `Cancelled`, `Aborted` -> (none; - /// terminal) - #[must_use] - pub const fn can_transition_to(self, to: Self) -> bool { - matches!( - (self, to), - (Self::Created, Self::SnapshotCreating | Self::Aborted) - | (Self::SnapshotCreating, Self::SnapshotReady | Self::Aborted) - | (Self::SnapshotReady, Self::Enumerating | Self::Aborted) - | (Self::Enumerating, Self::ManifestFinalized | Self::Aborted) - | (Self::ManifestFinalized, Self::Streaming | Self::Cancelled) - | (Self::Streaming, Self::Completing | Self::Cancelled) - | ( - Self::Completing, - Self::Completed | Self::CompletedWithFailures | Self::Cancelled - ) - ) - } - - /// Whether this state is terminal (no further transitions are legal). - #[must_use] - pub const fn is_terminal(self) -> bool { - matches!( - self, - Self::Completed | Self::CompletedWithFailures | Self::Cancelled | Self::Aborted - ) - } - - /// All variants, for exhaustive transition-graph testing. - #[cfg(test)] - const ALL: &'static [Self] = &[ - Self::Created, - Self::SnapshotCreating, - Self::SnapshotReady, - Self::Enumerating, - Self::ManifestFinalized, - Self::Streaming, - Self::Completing, - Self::Completed, - Self::CompletedWithFailures, - Self::Cancelled, - Self::Aborted, - ]; -} - -#[cfg(test)] -mod tests { - use super::JobState; - - #[test] - fn terminal_states_have_no_legal_outgoing_transition() { - for &from in JobState::ALL { - if from.is_terminal() { - for &to in JobState::ALL { - assert!( - !from.can_transition_to(to), - "{from:?} is terminal but claims a legal transition to {to:?}" - ); - } - } - } - } - - #[test] - fn every_non_terminal_state_has_at_least_one_legal_transition() { - for &from in JobState::ALL { - if !from.is_terminal() { - let has_any = JobState::ALL.iter().any(|&to| from.can_transition_to(to)); - assert!( - has_any, - "{from:?} is non-terminal but has no legal transition out" - ); - } - } - } - - #[test] - fn every_non_terminal_state_can_reach_aborted_or_cancelled_or_terminal() { - // Every non-terminal state must have some path to a terminal - // state directly (this test checks the *direct* edge only, which - // is true by construction here โ€” every non-terminal state's - // transition set includes at least one terminal state or a state - // one step from terminal). This guards against accidentally - // adding a state that can never resolve. - for &from in JobState::ALL { - if from.is_terminal() { - continue; - } - let reaches_terminal_or_progresses = - JobState::ALL.iter().any(|&to| from.can_transition_to(to)); - assert!( - reaches_terminal_or_progresses, - "{from:?} must be able to progress somewhere" - ); - } - } - - #[test] - fn no_state_transitions_to_itself() { - for &state in JobState::ALL { - assert!( - !state.can_transition_to(state), - "{state:?} must not self-transition" - ); - } - } - - #[test] - fn created_cannot_skip_directly_to_streaming() { - // Regression anchor: a job must pass through snapshot creation and - // enumeration before streaming โ€” skipping straight to Streaming - // would violate the "one snapshot defines the job" invariant - // (design-doc ยง2.1). - assert!(!JobState::Created.can_transition_to(JobState::Streaming)); - } - - #[test] - fn manifest_finalized_can_still_be_cancelled() { - // A job may be cancelled after the manifest is finalized but - // before/while streaming (design-doc ยง19.1). - assert!(JobState::ManifestFinalized.can_transition_to(JobState::Cancelled)); - } -} diff --git a/crates/uffs-content-protocol/tests/fixtures/candidate_record_basic.bin b/crates/uffs-content-protocol/tests/fixtures/candidate_record_basic.bin deleted file mode 100644 index fac2658d9..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/candidate_record_basic.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/fixtures/frame_corrupt_checksum.bin b/crates/uffs-content-protocol/tests/fixtures/frame_corrupt_checksum.bin deleted file mode 100644 index 5ba6d98c6..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/frame_corrupt_checksum.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/fixtures/frame_file_end_metadata_only.bin b/crates/uffs-content-protocol/tests/fixtures/frame_file_end_metadata_only.bin deleted file mode 100644 index bb7d0fcde..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/frame_file_end_metadata_only.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/fixtures/frame_file_end_success.bin b/crates/uffs-content-protocol/tests/fixtures/frame_file_end_success.bin deleted file mode 100644 index c6d881d98..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/frame_file_end_success.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/fixtures/frame_file_failed.bin b/crates/uffs-content-protocol/tests/fixtures/frame_file_failed.bin deleted file mode 100644 index 9828777aa..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/frame_file_failed.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/fixtures/frame_job_begin.bin b/crates/uffs-content-protocol/tests/fixtures/frame_job_begin.bin deleted file mode 100644 index c374687df..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/frame_job_begin.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/fixtures/frame_job_end.bin b/crates/uffs-content-protocol/tests/fixtures/frame_job_end.bin deleted file mode 100644 index 507b3a655..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/frame_job_end.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/fixtures/frame_truncated.bin b/crates/uffs-content-protocol/tests/fixtures/frame_truncated.bin deleted file mode 100644 index 2c9865613..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/frame_truncated.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/fixtures/job_stream_two_candidates.bin b/crates/uffs-content-protocol/tests/fixtures/job_stream_two_candidates.bin deleted file mode 100644 index 825951ac8..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/job_stream_two_candidates.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/fixtures/manifest_header_basic.bin b/crates/uffs-content-protocol/tests/fixtures/manifest_header_basic.bin deleted file mode 100644 index 964360254..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/manifest_header_basic.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/fixtures/manifest_trailer_basic.bin b/crates/uffs-content-protocol/tests/fixtures/manifest_trailer_basic.bin deleted file mode 100644 index 5ed4a14e7..000000000 Binary files a/crates/uffs-content-protocol/tests/fixtures/manifest_trailer_basic.bin and /dev/null differ diff --git a/crates/uffs-content-protocol/tests/golden_fixtures.rs b/crates/uffs-content-protocol/tests/golden_fixtures.rs deleted file mode 100644 index 9ee9811cf..000000000 --- a/crates/uffs-content-protocol/tests/golden_fixtures.rs +++ /dev/null @@ -1,725 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Golden fixture conformance tests (design-doc ยง21.6, addendum ยง5.5). -//! -//! Each fixture under `tests/fixtures/*.bin` is a **frozen** wire-format -//! sample, committed to the repository. These tests decode the frozen -//! bytes and assert the expected values โ€” they do not regenerate the -//! bytes from the current encoder on every run. That distinction is the -//! entire point: if a future change to `encode()`/`decode()` silently -//! drifts the wire format, a test that regenerated its own expected -//! bytes each time would never catch it, because it would just compare -//! the new (wrong) output against itself. Comparing against frozen bytes -//! is what makes this a *regression* guard rather than a tautology. -//! -//! This is also the cross-language contract surface: per addendum ยง5.5, -//! "a future implementation in another language is supported only after -//! passing the same conformance corpus" โ€” these files are that corpus's -//! first slice. -//! -//! To regenerate a fixture deliberately (a real, intentional wire-format -//! version bump โ€” not a routine change), run: -//! `UFFS_REGENERATE_FIXTURES=1 cargo test -p uffs-content-protocol --test -//! golden_fixtures -- --ignored --nocapture` then inspect the diff before -//! committing it. - -// These are `uffs-content-protocol`'s own dependencies, not this test -// binary's โ€” an integration test is a separate compilation unit, so it -// sees them as unused unless a marker import says otherwise. See -// `crates/uffs-daemon/src/lib.rs`'s `use uffs_version as _;` for the same -// pattern. -use bitflags as _; -use blake3 as _; -use proptest as _; -use thiserror as _; - -#[cfg(test)] -mod tests { - use std::fs; - use std::path::{Path, PathBuf}; - - use uffs_content_protocol::codec::Reader; - use uffs_content_protocol::error::ErrorCode; - use uffs_content_protocol::frame::{ - ContentChunk, ContentSemantics, DigestAlgorithm, FailedOutcome, FailureStage, FileBegin, - FileEnd, FileFailed, FrameEnvelope, FrameError, FrameOrdering, FrameType, JobBegin, JobEnd, - JobStatus, ReadMode, RetryClass, - }; - use uffs_content_protocol::manifest::{ - AuthorizationMode, CandidateFlags, CandidateRecord, ManifestHeader, ManifestTrailer, - }; - use uffs_content_protocol::path_encoding::WindowsPath; - - fn fixture_path(name: &str) -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests") - .join("fixtures") - .join(name) - } - - fn load_fixture(name: &str) -> Vec { - fs::read(fixture_path(name)).unwrap_or_else(|err| { - panic!( - "missing golden fixture {name}: {err}. Run with \ - UFFS_REGENERATE_FIXTURES=1 --ignored first." - ) - }) - } - - /// Writes `bytes` to the fixture path only when explicitly requested - /// via `UFFS_REGENERATE_FIXTURES=1`. Every call site that uses this - /// is `#[ignore]`d so a normal `cargo test` run never touches the - /// fixture files. - fn regenerate_fixture(name: &str, bytes: &[u8]) { - assert!( - std::env::var("UFFS_REGENERATE_FIXTURES").as_deref() == Ok("1"), - "refusing to write fixture {name}: set UFFS_REGENERATE_FIXTURES=1 to confirm this \ - is an intentional wire-format change" - ); - fs::write(fixture_path(name), bytes).unwrap_or_else(|err| panic!("writing {name}: {err}")); - } - - fn sample_manifest_header() -> ManifestHeader { - ManifestHeader { - format_version: 2, - job_id: [0x11_u8; 16], - source_id: [0x22_u8; 16], - volume_serial: 0x0102_0304_0506_0708, - volume_guid: b"{AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE}".to_vec(), - snapshot_id: b"vss-snapshot-0001".to_vec(), - snapshot_created_unix_ms: 1_752_000_000_000, - query_digest: [0x33_u8; 32], - authorization_mode: AuthorizationMode::AdminExport, - candidate_count: 3, - record_section_length: 999, - } - } - - fn sample_candidate_record() -> CandidateRecord { - CandidateRecord { - candidate_id: 12345, - file_reference: 0xABCD_EF01_2345_6789, - logical_size: 4_194_304, - valid_data_length: 4_194_304, - mtime_unix_ms: 1_752_000_000_000, - candidate_flags: CandidateFlags::NONRESIDENT | CandidateFlags::LARGE_FILE, - path: WindowsPath::from_str_lossless(r"C:\Users\robert\Documents\report-final.docx"), - } - } - - const fn sample_manifest_trailer() -> ManifestTrailer { - ManifestTrailer { - candidate_count_repeat: 3, - manifest_digest: [0x44_u8; 32], - } - } - - fn sample_job_begin_frame() -> (FrameEnvelope, JobBegin) { - let envelope = FrameEnvelope { - protocol_version: 2, - frame_type: FrameType::JobBegin, - flags: 0, - job_id: [0x11_u8; 16], - frame_sequence: 0, - }; - let payload = JobBegin { - job_id: [0x11_u8; 16], - source_id: [0x22_u8; 16], - snapshot_id: b"vss-snapshot-0001".to_vec(), - snapshot_created_at: 1_752_000_000_000, - manifest_digest: [0x55_u8; 32], - candidate_count: 3, - authorization_mode: AuthorizationMode::AdminExport, - ordering: FrameOrdering::None, - content_semantics: ContentSemantics::UnnamedLogicalStream, - digest_algorithm: DigestAlgorithm::Blake3, - max_chunk_bytes: 1_048_576, - max_content_delivery_bytes: Some(64 * 1024 * 1024), - }; - (envelope, payload) - } - - const fn sample_file_end_success() -> FileEnd { - FileEnd { - candidate_id: 12345, - total_logical_bytes: 4_194_304, - content_digest: Some([0x66_u8; 32]), - read_mode: ReadMode::LogicalSnapshot, - chunk_count: 64, - elapsed_ms: 250, - warning_flags: 0, - } - } - - const fn sample_file_end_metadata_only() -> FileEnd { - FileEnd { - candidate_id: 99999, - total_logical_bytes: 0, - content_digest: None, - read_mode: ReadMode::MetadataOnly, - chunk_count: 0, - elapsed_ms: 1, - warning_flags: 0, - } - } - - fn sample_file_failed() -> FileFailed { - FileFailed { - candidate_id: 777, - outcome: FailedOutcome::Retryable, - failure_stage: FailureStage::Read, - error_code: ErrorCode::ReadIoTransient, - os_error_code: Some(-5), - retry_class: RetryClass::RetrySameJob, - bytes_emitted_before_failure: 0, - message: "transient I/O error reading extent".to_owned(), - } - } - - fn sample_job_end() -> JobEnd { - JobEnd { - candidate_count: 10, - succeeded_count: 7, - failed_retryable_count: 1, - failed_terminal_count: 1, - deferred_manual_count: 1, - acknowledged_success_count: 7, - logical_bytes_succeeded: 100_000_000, - failure_bucket_id: b"job-0001-failures".to_vec(), - manifest_digest: [0x77_u8; 32], - outcome_ledger_digest: [0x88_u8; 32], - job_status: JobStatus::CompletedWithFailures, - } - } - - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ manifest header โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_manifest_header_fixture() { - let bytes = sample_manifest_header().encode().unwrap(); - regenerate_fixture("manifest_header_basic.bin", &bytes); - } - - #[test] - fn manifest_header_fixture_decodes_to_expected_values() { - let bytes = load_fixture("manifest_header_basic.bin"); - let mut reader = Reader::new(&bytes); - let decoded = ManifestHeader::decode(&mut reader).unwrap(); - assert_eq!(decoded, sample_manifest_header()); - // Also guards encode-side drift: re-encoding the decoded value - // must reproduce the exact frozen bytes. - assert_eq!(decoded.encode().unwrap(), bytes); - } - - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ candidate record โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_candidate_record_fixture() { - let bytes = sample_candidate_record().encode().unwrap(); - regenerate_fixture("candidate_record_basic.bin", &bytes); - } - - #[test] - fn candidate_record_fixture_decodes_to_expected_values() { - let bytes = load_fixture("candidate_record_basic.bin"); - let mut reader = Reader::new(&bytes); - let decoded = CandidateRecord::decode(&mut reader).unwrap(); - assert_eq!(decoded, sample_candidate_record()); - assert_eq!(decoded.encode().unwrap(), bytes); - } - - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ manifest trailer โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_manifest_trailer_fixture() { - let bytes = sample_manifest_trailer().encode(); - regenerate_fixture("manifest_trailer_basic.bin", &bytes); - } - - #[test] - fn manifest_trailer_fixture_decodes_to_expected_values() { - let bytes = load_fixture("manifest_trailer_basic.bin"); - let mut reader = Reader::new(&bytes); - let decoded = ManifestTrailer::decode(&mut reader).unwrap(); - assert_eq!(decoded, sample_manifest_trailer()); - assert_eq!(decoded.encode(), bytes); - } - - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ frames: JOB_BEGIN โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_frame_job_begin_fixture() { - let (envelope, payload) = sample_job_begin_frame(); - let bytes = envelope.encode(&payload.encode()); - regenerate_fixture("frame_job_begin.bin", &bytes); - } - - #[test] - fn frame_job_begin_fixture_decodes_to_expected_values() { - let bytes = load_fixture("frame_job_begin.bin"); - let mut reader = Reader::new(&bytes); - let (decoded_envelope, decoded_payload_bytes) = - FrameEnvelope::decode(&mut reader, 1_000_000).unwrap(); - let (expected_envelope, expected_payload) = sample_job_begin_frame(); - assert_eq!(decoded_envelope, expected_envelope); - let mut payload_reader = Reader::new(&decoded_payload_bytes); - let decoded_payload = JobBegin::decode(&mut payload_reader).unwrap(); - assert_eq!(decoded_payload, expected_payload); - } - - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ frames: FILE_END (success) - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_frame_file_end_success_fixture() { - let payload = sample_file_end_success(); - let envelope = FrameEnvelope { - protocol_version: 2, - frame_type: FrameType::FileEnd, - flags: 0, - job_id: [0x11_u8; 16], - frame_sequence: 10, - }; - let bytes = envelope.encode(&payload.encode()); - regenerate_fixture("frame_file_end_success.bin", &bytes); - } - - #[test] - fn frame_file_end_success_fixture_decodes_to_expected_values() { - let bytes = load_fixture("frame_file_end_success.bin"); - let mut reader = Reader::new(&bytes); - let (envelope, payload_bytes) = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap(); - assert_eq!(envelope.frame_type, FrameType::FileEnd); - let mut payload_reader = Reader::new(&payload_bytes); - let decoded = FileEnd::decode(&mut payload_reader).unwrap(); - assert_eq!(decoded, sample_file_end_success()); - assert!(decoded.content_digest.is_some()); - } - - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ frames: FILE_END (metadata-only) - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_frame_file_end_metadata_only_fixture() { - let payload = sample_file_end_metadata_only(); - let envelope = FrameEnvelope { - protocol_version: 2, - frame_type: FrameType::FileEnd, - flags: 0, - job_id: [0x11_u8; 16], - frame_sequence: 11, - }; - let bytes = envelope.encode(&payload.encode()); - regenerate_fixture("frame_file_end_metadata_only.bin", &bytes); - } - - #[test] - fn frame_file_end_metadata_only_fixture_decodes_to_expected_values() { - // This is the content-delivery-ceiling fixture: a candidate that - // matched the job's query but exceeded the delivery ceiling, so - // it has no content body โ€” see frame::ReadMode::MetadataOnly. - let bytes = load_fixture("frame_file_end_metadata_only.bin"); - let mut reader = Reader::new(&bytes); - let (_envelope, payload_bytes) = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap(); - let mut payload_reader = Reader::new(&payload_bytes); - let decoded = FileEnd::decode(&mut payload_reader).unwrap(); - assert_eq!(decoded, sample_file_end_metadata_only()); - assert_eq!(decoded.content_digest, None); - assert_eq!(decoded.read_mode, ReadMode::MetadataOnly); - assert_eq!(decoded.chunk_count, 0); - } - - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ frames: FILE_FAILED โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_frame_file_failed_fixture() { - let payload = sample_file_failed(); - let envelope = FrameEnvelope { - protocol_version: 2, - frame_type: FrameType::FileFailed, - flags: 0, - job_id: [0x11_u8; 16], - frame_sequence: 12, - }; - let bytes = envelope.encode(&payload.encode()); - regenerate_fixture("frame_file_failed.bin", &bytes); - } - - #[test] - fn frame_file_failed_fixture_decodes_to_expected_values() { - let bytes = load_fixture("frame_file_failed.bin"); - let mut reader = Reader::new(&bytes); - let (_envelope, payload_bytes) = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap(); - let mut payload_reader = Reader::new(&payload_bytes); - let decoded = FileFailed::decode(&mut payload_reader).unwrap(); - assert_eq!(decoded, sample_file_failed()); - } - - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ frames: JOB_END โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_frame_job_end_fixture() { - let payload = sample_job_end(); - let envelope = FrameEnvelope { - protocol_version: 2, - frame_type: FrameType::JobEnd, - flags: 0, - job_id: [0x11_u8; 16], - frame_sequence: 999, - }; - let bytes = envelope.encode(&payload.encode().unwrap()); - regenerate_fixture("frame_job_end.bin", &bytes); - } - - #[test] - fn frame_job_end_fixture_decodes_to_expected_values() { - let bytes = load_fixture("frame_job_end.bin"); - let mut reader = Reader::new(&bytes); - let (_envelope, payload_bytes) = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap(); - let mut payload_reader = Reader::new(&payload_bytes); - let decoded = JobEnd::decode(&mut payload_reader).unwrap(); - assert_eq!(decoded, sample_job_end()); - // Completeness invariant sanity (design-doc ยง2.2/ยง21.7). - assert_eq!( - decoded.candidate_count, - decoded.succeeded_count - + decoded.failed_retryable_count - + decoded.failed_terminal_count - + decoded.deferred_manual_count - ); - } - - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ job stream (full sequence) - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - // - // A recorded frame stream for one representative job, end to end: - // JOB_BEGIN, then FILE_BEGIN/[CONTENT_CHUNK]*/FILE_END per candidate, - // then JOB_END โ€” no framing beyond each frame's own envelope, exactly - // as a consumer sees it over the wire. This is the "replay fixture" - // a consumer without a Windows/VSS host (e.g. Docenta) can decode - // against to validate their own decoder end to end, not just against - // one frame type in isolation. Candidate 1 is an ordinary two-chunk - // success; candidate 2 exceeds `JOB_BEGIN.max_content_delivery_bytes` - // and is reported `ReadMode::MetadataOnly`, so this stream also - // covers that still-recent wire shape in full sequence context. - - const JOB_STREAM_JOB_ID: [u8; 16] = [0xAA_u8; 16]; - - const fn job_stream_content() -> &'static [u8] { - b"hello world" - } - - fn sample_job_stream_job_begin() -> JobBegin { - JobBegin { - job_id: JOB_STREAM_JOB_ID, - source_id: [0xBB_u8; 16], - snapshot_id: b"vss-snapshot-job-stream-0001".to_vec(), - snapshot_created_at: 1_752_100_000_000, - manifest_digest: [0xCC_u8; 32], - candidate_count: 2, - authorization_mode: AuthorizationMode::AdminExport, - ordering: FrameOrdering::None, - content_semantics: ContentSemantics::UnnamedLogicalStream, - digest_algorithm: DigestAlgorithm::Blake3, - max_chunk_bytes: 1_048_576, - max_content_delivery_bytes: Some(1024), - } - } - - fn sample_job_stream_file_begin_1() -> FileBegin { - FileBegin { - candidate_id: 1, - file_reference: 0x1000_0000_0000_0001, - path: WindowsPath::from_str_lossless(r"C:\Users\robert\Documents\report.docx"), - logical_size: 11, - mtime: 1_752_100_000_000, - read_mode: ReadMode::LogicalSnapshot, - attempt_number: 1, - content_object_id: None, - } - } - - fn sample_job_stream_chunk_1a() -> ContentChunk { - ContentChunk { - candidate_id: 1, - chunk_sequence: 0, - logical_offset: 0, - logical_length: 6, - payload: b"hello ".to_vec(), - } - } - - fn sample_job_stream_chunk_1b() -> ContentChunk { - ContentChunk { - candidate_id: 1, - chunk_sequence: 1, - logical_offset: 6, - logical_length: 5, - payload: b"world".to_vec(), - } - } - - fn sample_job_stream_file_end_1() -> FileEnd { - FileEnd { - candidate_id: 1, - total_logical_bytes: 11, - content_digest: Some(*blake3::hash(job_stream_content()).as_bytes()), - read_mode: ReadMode::LogicalSnapshot, - chunk_count: 2, - elapsed_ms: 3, - warning_flags: 0, - } - } - - fn sample_job_stream_file_begin_2() -> FileBegin { - FileBegin { - candidate_id: 2, - file_reference: 0x1000_0000_0000_0002, - path: WindowsPath::from_str_lossless(r"C:\Data\bigfile.bin"), - logical_size: 10_737_418_240, - mtime: 1_752_100_000_000, - read_mode: ReadMode::MetadataOnly, - attempt_number: 1, - content_object_id: None, - } - } - - const fn sample_job_stream_file_end_2() -> FileEnd { - FileEnd { - candidate_id: 2, - total_logical_bytes: 0, - content_digest: None, - read_mode: ReadMode::MetadataOnly, - chunk_count: 0, - elapsed_ms: 0, - warning_flags: 0, - } - } - - fn sample_job_stream_job_end() -> JobEnd { - JobEnd { - candidate_count: 2, - succeeded_count: 2, - failed_retryable_count: 0, - failed_terminal_count: 0, - deferred_manual_count: 0, - acknowledged_success_count: 0, - logical_bytes_succeeded: 11, - failure_bucket_id: b"job-stream-0001-failures".to_vec(), - manifest_digest: [0xCC_u8; 32], - outcome_ledger_digest: [0xDD_u8; 32], - job_status: JobStatus::Completed, - } - } - - /// Wraps `payload` in a `FrameEnvelope` for the job-stream fixture, - /// assigning `frame_sequence` in emission order. - fn job_stream_envelope(frame_type: FrameType, frame_sequence: u64, payload: &[u8]) -> Vec { - FrameEnvelope { - protocol_version: 2, - frame_type, - flags: 0, - job_id: JOB_STREAM_JOB_ID, - frame_sequence, - } - .encode(payload) - } - - /// Builds the full, ordered byte stream: `JOB_BEGIN`, candidate 1's - /// `FILE_BEGIN`/two `CONTENT_CHUNK`s/`FILE_END`, candidate 2's - /// `FILE_BEGIN`/`FILE_END` (metadata-only), then `JOB_END`. - fn build_job_stream_bytes() -> Vec { - let mut out = Vec::new(); - out.extend(job_stream_envelope( - FrameType::JobBegin, - 0, - &sample_job_stream_job_begin().encode(), - )); - out.extend(job_stream_envelope( - FrameType::FileBegin, - 1, - &sample_job_stream_file_begin_1().encode(), - )); - out.extend(job_stream_envelope( - FrameType::ContentChunk, - 2, - &sample_job_stream_chunk_1a().encode(), - )); - out.extend(job_stream_envelope( - FrameType::ContentChunk, - 3, - &sample_job_stream_chunk_1b().encode(), - )); - out.extend(job_stream_envelope( - FrameType::FileEnd, - 4, - &sample_job_stream_file_end_1().encode(), - )); - out.extend(job_stream_envelope( - FrameType::FileBegin, - 5, - &sample_job_stream_file_begin_2().encode(), - )); - out.extend(job_stream_envelope( - FrameType::FileEnd, - 6, - &sample_job_stream_file_end_2().encode(), - )); - out.extend(job_stream_envelope( - FrameType::JobEnd, - 7, - &sample_job_stream_job_end() - .encode() - .unwrap_or_else(|err| panic!("sample_job_stream_job_end must encode: {err}")), - )); - out - } - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_job_stream_fixture() { - regenerate_fixture("job_stream_two_candidates.bin", &build_job_stream_bytes()); - } - - #[test] - fn job_stream_fixture_decodes_to_expected_full_sequence() { - let bytes = load_fixture("job_stream_two_candidates.bin"); - let mut reader = Reader::new(&bytes); - - let mut decoded_types = Vec::new(); - let mut total_chunk_bytes = Vec::new(); - let mut file_ends = Vec::new(); - while reader.remaining() > 0 { - let (envelope, payload) = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap(); - assert_eq!(envelope.job_id, JOB_STREAM_JOB_ID); - decoded_types.push(envelope.frame_type); - - let mut payload_reader = Reader::new(&payload); - match envelope.frame_type { - FrameType::JobBegin => { - assert_eq!( - JobBegin::decode(&mut payload_reader).unwrap(), - sample_job_stream_job_begin() - ); - } - FrameType::ContentChunk => { - let chunk = ContentChunk::decode(&mut payload_reader, 1_000_000).unwrap(); - if chunk.candidate_id == 1 { - total_chunk_bytes.extend(chunk.payload); - } - } - FrameType::FileEnd => { - file_ends.push(FileEnd::decode(&mut payload_reader).unwrap()); - } - FrameType::JobEnd => { - assert_eq!( - JobEnd::decode(&mut payload_reader).unwrap(), - sample_job_stream_job_end() - ); - } - FrameType::FileBegin => { - // Decoded via the per-candidate assertions below. - } - other @ (FrameType::FileFailed - | FrameType::FileDeferred - | FrameType::FileAck - | FrameType::Progress - | FrameType::Heartbeat - | FrameType::JobCancel - | FrameType::WindowUpdate - | FrameType::JobResume - | FrameType::JobSubmit) => { - panic!("unexpected frame type in job stream: {other:?}") - } - } - } - - assert_eq!(decoded_types, vec![ - FrameType::JobBegin, - FrameType::FileBegin, - FrameType::ContentChunk, - FrameType::ContentChunk, - FrameType::FileEnd, - FrameType::FileBegin, - FrameType::FileEnd, - FrameType::JobEnd, - ]); - - // Candidate 1: chunks reassemble to the exact original content, - // and the digest FILE_END reports matches an independent BLAKE3 - // recomputation over those reassembled bytes. - assert_eq!(total_chunk_bytes, job_stream_content()); - assert_eq!(file_ends.len(), 2); - let candidate_1_end = file_ends - .iter() - .find(|end| end.candidate_id == 1) - .expect("candidate 1's FILE_END must be present"); - assert_eq!(candidate_1_end, &sample_job_stream_file_end_1()); - assert_eq!( - candidate_1_end.content_digest, - Some(*blake3::hash(&total_chunk_bytes).as_bytes()) - ); - - // Candidate 2: over the delivery ceiling, so metadata-only โ€” - // matches ReadMode::MetadataOnly's own doc comment (design-doc's - // two-tier delivery-ceiling model). - let candidate_2_end = file_ends - .iter() - .find(|end| end.candidate_id == 2) - .expect("candidate 2's FILE_END must be present"); - assert_eq!(candidate_2_end, &sample_job_stream_file_end_2()); - assert_eq!(candidate_2_end.content_digest, None); - assert_eq!(candidate_2_end.chunk_count, 0); - } - - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ invalid fixtures (must be rejected) - // โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_corrupt_checksum_fixture() { - let (envelope, payload) = sample_job_begin_frame(); - let mut bytes = envelope.encode(&payload.encode()); - let last = bytes.len() - 1; - if let Some(byte) = bytes.get_mut(last) { - *byte ^= 0xFF; - } - regenerate_fixture("frame_corrupt_checksum.bin", &bytes); - } - - #[test] - fn corrupt_checksum_fixture_is_rejected() { - let bytes = load_fixture("frame_corrupt_checksum.bin"); - let mut reader = Reader::new(&bytes); - let err = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap_err(); - assert!(matches!(err, FrameError::PayloadChecksumMismatch { .. })); - } - - #[test] - #[ignore = "writes a golden fixture; run with UFFS_REGENERATE_FIXTURES=1 --ignored"] - fn regenerate_truncated_frame_fixture() { - let (envelope, payload) = sample_job_begin_frame(); - let bytes = envelope.encode(&payload.encode()); - // Truncate to just past the fixed header, before any checksum or - // payload bytes are fully present. - let truncated = bytes.get(0..30).unwrap_or(&bytes).to_vec(); - regenerate_fixture("frame_truncated.bin", &truncated); - } - - #[test] - fn truncated_frame_fixture_is_rejected() { - let bytes = load_fixture("frame_truncated.bin"); - let mut reader = Reader::new(&bytes); - let err = FrameEnvelope::decode(&mut reader, 1_000_000).unwrap_err(); - assert!(matches!(err, FrameError::Decode(_))); - } -} diff --git a/crates/uffs-content-reader-protocol/Cargo.toml b/crates/uffs-content-reader-protocol/Cargo.toml deleted file mode 100644 index ebc2cd507..000000000 --- a/crates/uffs-content-reader-protocol/Cargo.toml +++ /dev/null @@ -1,53 +0,0 @@ -# ============================================================================ -# uffs-content-reader-protocol: Content Coordinator <-> Snapshot Reader -# wire-protocol types -# ============================================================================ -# Layer 0 Foundation crate. Tiny dedicated lib โ€” pure-logic byte-shuffling, -# no Windows FFI, no I/O. Mirrors `uffs-broker-protocol`'s shape and size. -# -# Private wire format between `uffs-content` (the unprivileged Content -# Coordinator) and the privileged Snapshot Reader process (addendum ยง2.1- -# ยง2.4). Deliberately a SEPARATE crate from `uffs-content-protocol` (the -# public UFFS<->Docenta protocol) even though the two share some field -# shapes โ€” addendum ยง5.2 is explicit that the public protocol crate "MUST -# NOT contain... privileged IPC implementation", and Docenta must never be -# able to depend on this crate's evolution. -# -# This crate does NOT depend on `uffs-content-protocol` even though both -# are Layer 0: per docs/architecture/crate-graph.md, "Layer-0 โ†’ Layer-0 -# deps... should stay absent โ€” each Layer-0 crate is independently -# buildable" (the same rule that keeps uffs-text and uffs-time from -# depending on each other). The small amount of duplicated bounds-checked -# LE-decode logic in `codec.rs` is a deliberate, small trade against that -# independence guarantee, not an oversight. -# ============================================================================ - -[package] -name = "uffs-content-reader-protocol" -description = "Private wire protocol between uffs-content (Coordinator) and the Snapshot Reader (cross-platform, Windows-only machinery lives in the Reader binary)" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -authors.workspace = true -readme.workspace = true -keywords.workspace = true -categories.workspace = true -# Intentionally NOT published. Private, internal-only IPC contract between -# two processes in this workspace; no meaning outside it. Reserve the name -# on crates.io to prevent squatting, but never carry content. Mirrors -# `uffs-broker-protocol`'s rationale. -publish.workspace = true - -[package.metadata.docs.rs] -all-features = true -rustdoc-args = ["--cfg", "docsrs"] - -[dependencies] -thiserror.workspace = true - -[dev-dependencies] -proptest.workspace = true - -[lints] -workspace = true diff --git a/crates/uffs-content-reader-protocol/src/codec.rs b/crates/uffs-content-reader-protocol/src/codec.rs deleted file mode 100644 index 9076f09d1..000000000 --- a/crates/uffs-content-reader-protocol/src/codec.rs +++ /dev/null @@ -1,320 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Minimal bounds-checked little-endian decode primitives. -//! -//! A deliberately small, independent duplicate of -//! `uffs-content-protocol::codec::Reader`'s shape โ€” see this crate's -//! `Cargo.toml` for why the two Layer-0 crates don't share this code via -//! an internal dependency. Only the handful of primitives -//! [`ReadRequest`](crate::ReadRequest)/[`ReadResponse`](crate::ReadResponse) -//! actually need are implemented here. - -/// Errors decoding wire bytes. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[non_exhaustive] -pub enum DecodeError { - /// Fewer bytes remained than the field being read requires. - #[error("truncated input: needed {needed} bytes, only {available} remained")] - Truncated { - /// Bytes required to satisfy the read. - needed: usize, - /// Bytes actually remaining in the input. - available: usize, - }, - /// A length-prefixed field declared more bytes than the caller's - /// configured maximum allows, checked before allocation. - #[error("field '{field}' declared length {declared} exceeds maximum {max}")] - LengthOutOfBounds { - /// Name of the offending field, for diagnostics. - field: &'static str, - /// Length the wire bytes claimed. - declared: u64, - /// Maximum length the caller configured. - max: u64, - }, - /// A discriminant byte did not match any known enum variant. - #[error("unknown discriminant for '{field}': {value}")] - UnknownDiscriminant { - /// Name of the field being decoded, for diagnostics. - field: &'static str, - /// The unrecognized value. - value: u64, - }, - /// A string field was not valid UTF-8. - #[error("field '{0}' is not valid UTF-8")] - InvalidUtf8(&'static str), -} - -/// Bounds-checked little-endian cursor over a decode input buffer. -#[derive(Debug, Clone, Copy)] -pub struct Reader<'a> { - /// Backing bytes being decoded. - buf: &'a [u8], - /// Read offset into `buf`; always `<= buf.len()`. - pos: usize, -} - -impl<'a> Reader<'a> { - /// Wrap `buf` for bounds-checked reading, starting at offset 0. - #[must_use] - pub const fn new(buf: &'a [u8]) -> Self { - Self { buf, pos: 0 } - } - - /// Bytes not yet consumed. - #[must_use] - pub const fn remaining(&self) -> usize { - self.buf.len() - self.pos - } - - /// Consume and return exactly `len` bytes, or a - /// [`DecodeError::Truncated`] if fewer remain. - fn take(&mut self, len: usize) -> Result<&'a [u8], DecodeError> { - let available = self.remaining(); - if available < len { - return Err(DecodeError::Truncated { - needed: len, - available, - }); - } - let start = self.pos; - let slice = self - .buf - .get(start..start + len) - .ok_or(DecodeError::Truncated { - needed: len, - available, - })?; - self.pos += len; - Ok(slice) - } - - /// Read a single byte. - /// - /// # Errors - /// [`DecodeError::Truncated`] if no bytes remain. - pub fn read_u8(&mut self) -> Result { - let bytes = self.take(1)?; - bytes.first().copied().ok_or(DecodeError::Truncated { - needed: 1, - available: 0, - }) - } - - /// Read exactly `N` raw bytes. - /// - /// # Errors - /// [`DecodeError::Truncated`] if fewer than `N` bytes remain. - pub fn read_array(&mut self) -> Result<[u8; N], DecodeError> { - let bytes = self.take(N)?; - let mut out = [0_u8; N]; - out.copy_from_slice(bytes); - Ok(out) - } - - /// Read a little-endian `u32`. - /// - /// # Errors - /// [`DecodeError::Truncated`] if fewer than 4 bytes remain. - pub fn read_u32_le(&mut self) -> Result { - Ok(u32::from_le_bytes(self.read_array()?)) - } - - /// Read a little-endian `u64`. - /// - /// # Errors - /// [`DecodeError::Truncated`] if fewer than 8 bytes remain. - pub fn read_u64_le(&mut self) -> Result { - Ok(u64::from_le_bytes(self.read_array()?)) - } - - /// Read a `u32`-length-prefixed byte string, rejecting (before any - /// allocation) a declared length exceeding `max_len` or the bytes - /// actually remaining. - /// - /// # Errors - /// - /// - [`DecodeError::LengthOutOfBounds`] if the declared length exceeds - /// `max_len`. - /// - [`DecodeError::Truncated`] if the declared length exceeds the bytes - /// remaining. - pub fn read_bytes_u32_prefixed( - &mut self, - field: &'static str, - max_len: u32, - ) -> Result, DecodeError> { - let len = self.read_u32_le()?; - if len > max_len { - return Err(DecodeError::LengthOutOfBounds { - field, - declared: u64::from(len), - max: u64::from(max_len), - }); - } - let bytes = self.take(len as usize)?; - Ok(bytes.to_vec()) - } - - /// Read a `u16`-length-prefixed UTF-8 string, same bounds discipline - /// as [`read_bytes_u32_prefixed`](Self::read_bytes_u32_prefixed). - /// - /// # Errors - /// - /// Same as [`read_bytes_u32_prefixed`](Self::read_bytes_u32_prefixed), - /// plus [`DecodeError::InvalidUtf8`] if the bytes are not valid UTF-8. - pub fn read_string_u16_prefixed( - &mut self, - field: &'static str, - max_len: u16, - ) -> Result { - let len = self.read_u16_le()?; - if len > max_len { - return Err(DecodeError::LengthOutOfBounds { - field, - declared: u64::from(len), - max: u64::from(max_len), - }); - } - let bytes = self.take(len as usize)?; - String::from_utf8(bytes.to_vec()).map_err(|_err| DecodeError::InvalidUtf8(field)) - } - - /// Read a little-endian `u16`. - /// - /// # Errors - /// [`DecodeError::Truncated`] if fewer than 2 bytes remain. - pub fn read_u16_le(&mut self) -> Result { - Ok(u16::from_le_bytes(self.read_array()?)) - } -} - -/// Append a little-endian `u16` to `out`. -pub fn write_u16_le(out: &mut Vec, value: u16) { - out.extend_from_slice(&value.to_le_bytes()); -} - -/// Append a little-endian `u32` to `out`. -pub fn write_u32_le(out: &mut Vec, value: u32) { - out.extend_from_slice(&value.to_le_bytes()); -} - -/// Append a little-endian `u64` to `out`. -pub fn write_u64_le(out: &mut Vec, value: u64) { - out.extend_from_slice(&value.to_le_bytes()); -} - -/// Append a `u32`-length-prefixed byte string to `out`. -pub fn write_bytes_u32_prefixed(out: &mut Vec, bytes: &[u8]) { - #[expect( - clippy::cast_possible_truncation, - reason = "encode-side only; callers are expected to keep byte strings \ - within u32::MAX. The decode side enforces the real, \ - non-panicking rejection via Reader::read_bytes_u32_prefixed." - )] - let len = bytes.len() as u32; - write_u32_le(out, len); - out.extend_from_slice(bytes); -} - -/// Append a `u16`-length-prefixed UTF-8 string to `out`. -pub fn write_string_u16_prefixed(out: &mut Vec, value: &str) { - #[expect( - clippy::cast_possible_truncation, - reason = "encode-side only; see write_bytes_u32_prefixed." - )] - let len = value.len() as u16; - write_u16_le(out, len); - out.extend_from_slice(value.as_bytes()); -} - -#[cfg(test)] -mod tests { - use super::{ - DecodeError, Reader, write_bytes_u32_prefixed, write_string_u16_prefixed, write_u16_le, - write_u32_le, write_u64_le, - }; - - #[test] - fn read_u8_truncated_on_empty() { - let mut reader = Reader::new(&[]); - assert_eq!(reader.read_u8().unwrap_err(), DecodeError::Truncated { - needed: 1, - available: 0 - }); - } - - #[test] - fn read_u32_le_round_trip() { - let mut buf = Vec::new(); - write_u32_le(&mut buf, 0x1234_5678); - let mut reader = Reader::new(&buf); - assert_eq!(reader.read_u32_le().unwrap(), 0x1234_5678); - } - - #[test] - fn read_u64_le_round_trip() { - let mut buf = Vec::new(); - write_u64_le(&mut buf, 0x0102_0304_0506_0708); - let mut reader = Reader::new(&buf); - assert_eq!(reader.read_u64_le().unwrap(), 0x0102_0304_0506_0708); - } - - #[test] - fn read_u16_le_round_trip() { - let mut buf = Vec::new(); - write_u16_le(&mut buf, 0xABCD); - let mut reader = Reader::new(&buf); - assert_eq!(reader.read_u16_le().unwrap(), 0xABCD); - } - - #[test] - fn length_prefixed_bytes_round_trip() { - let mut buf = Vec::new(); - write_bytes_u32_prefixed(&mut buf, b"hello"); - let mut reader = Reader::new(&buf); - assert_eq!(reader.read_bytes_u32_prefixed("f", 100).unwrap(), b"hello"); - } - - #[test] - fn length_prefixed_bytes_rejects_over_max_before_truncation() { - let mut buf = Vec::new(); - write_u32_le(&mut buf, 1000); - let mut reader = Reader::new(&buf); - let err = reader.read_bytes_u32_prefixed("f", 10).unwrap_err(); - assert_eq!(err, DecodeError::LengthOutOfBounds { - field: "f", - declared: 1000, - max: 10, - }); - } - - #[test] - fn string_round_trip() { - let mut buf = Vec::new(); - write_string_u16_prefixed(&mut buf, "hello world"); - let mut reader = Reader::new(&buf); - assert_eq!( - reader.read_string_u16_prefixed("f", 100).unwrap(), - "hello world" - ); - } - - #[test] - fn string_rejects_invalid_utf8() { - let mut buf = Vec::new(); - write_u16_le(&mut buf, 2); - buf.extend_from_slice(&[0xFF, 0xFE]); // invalid UTF-8 - let mut reader = Reader::new(&buf); - let err = reader.read_string_u16_prefixed("f", 100).unwrap_err(); - assert_eq!(err, DecodeError::InvalidUtf8("f")); - } - - #[test] - fn array_read_exact_width() { - let mut reader = Reader::new(&[1, 2, 3, 4, 5]); - let arr: [u8; 3] = reader.read_array().unwrap(); - assert_eq!(arr, [1, 2, 3]); - assert_eq!(reader.remaining(), 2); - } -} diff --git a/crates/uffs-content-reader-protocol/src/lib.rs b/crates/uffs-content-reader-protocol/src/lib.rs deleted file mode 100644 index e17446b05..000000000 --- a/crates/uffs-content-reader-protocol/src/lib.rs +++ /dev/null @@ -1,637 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Private wire protocol between `uffs-content` (the unprivileged Content -//! Coordinator) and the privileged Snapshot Reader process. -//! -//! Addendum ยง2.1-ยง2.4: the Coordinator never receives the snapshot -//! handle, LCNs, extents, or raw MFT records. It sends a typed -//! [`ReadRequest`] naming a candidate and a logical byte range; the -//! Reader resolves the stream itself, validates the range against the -//! snapshot's own EOF/VDL, and returns bounded logical bytes or a typed -//! error โ€” never accepting an arbitrary `(physical_offset, length)`. -//! -//! # Status -//! -//! Scaffold: types and wire codec only. No transport (named pipe) -//! implementation yet โ€” that lives in `uffs-content` (client side) and -//! the Snapshot Reader binary (server side) once they exist. - -pub mod codec; - -use codec::{ - DecodeError, Reader, write_bytes_u32_prefixed, write_string_u16_prefixed, write_u32_le, - write_u64_le, -}; - -/// Named-pipe path the Snapshot Reader listens on. -/// -/// Deliberately separate from `uffs-broker-protocol::PIPE_NAME` (daemon -/// <-> Broker) and from the Broker's Snapshot Manager endpoint -/// (`uffs-broker-protocol::SNAPSHOT_PIPE_NAME`, once that lands) โ€” -/// Coordinator<->Reader is a distinct channel with a distinct peer and -/// distinct trust check, per -/// `uffs-ingest-implementation-plan.md` ยง3.3. -pub const READER_PIPE_NAME: &str = r"\\.\pipe\uffs-content-reader"; - -/// Maximum byte length for `snapshot_device_identity`-style opaque -/// identifier fields in this protocol. -pub const MAX_IDENTIFIER_BYTES: u32 = 512; - -/// Maximum byte length for a free-text diagnostic message. -const MAX_MESSAGE_BYTES: u16 = 4096; - -/// Append an `Option` as a presence byte followed by the value if -/// present โ€” mirrors `uffs-content-protocol::frame::write_optional_u64` -/// (this crate deliberately duplicates rather than depends on that -/// Layer-0 crate; see [`codec`]'s own module doc). -fn write_optional_u64(out: &mut Vec, value: Option) { - match value { - Some(present_value) => { - out.push(1); - write_u64_le(out, present_value); - } - None => out.push(0), - } -} - -/// Read an `Option` encoded by [`write_optional_u64`]. -fn read_optional_u64(reader: &mut Reader<'_>) -> Result, DecodeError> { - let present = reader.read_u8()?; - match present { - 0 => Ok(None), - _ => Ok(Some(reader.read_u64_le()?)), - } -} - -/// A volume's identity, as carried in a [`ReadRequest`] (addendum ยง2.3). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct VolumeIdentity { - /// NTFS volume serial number. - pub volume_serial: u64, - /// Opaque volume GUID bytes. - pub volume_guid: Vec, -} - -impl VolumeIdentity { - /// Append this identity's wire encoding to `out`. - fn encode(&self, out: &mut Vec) { - write_u64_le(out, self.volume_serial); - write_bytes_u32_prefixed(out, &self.volume_guid); - } - - /// Decode an identity from `reader`. - fn decode(reader: &mut Reader<'_>) -> Result { - let volume_serial = reader.read_u64_le()?; - let volume_guid = reader.read_bytes_u32_prefixed("volume_guid", MAX_IDENTIFIER_BYTES)?; - Ok(Self { - volume_serial, - volume_guid, - }) - } -} - -/// Which stream a [`ReadRequest`] targets. -/// -/// Only [`StreamKind::UnnamedData`] exists in v2 (design-doc ยง2.6: no ADS -/// in this version) โ€” modeled as an enum so a future version can add -/// variants without breaking the field shape. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum StreamKind { - /// The unnamed/default `$DATA` stream. - UnnamedData = 0, -} - -impl StreamKind { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::UnnamedData), - other => Err(other), - } - } -} - -/// The read mode a [`ReadRequest`] asks the Reader to use. -/// -/// Addendum ยง2.3/ยง3.6 planner. Distinct from -/// `uffs_content_protocol::frame::ReadMode` (which reports what mode was -/// *actually* used, after the fact, and includes `Resident`/`MetadataOnly` -/// โ€” concepts that don't apply to an outgoing request). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum RequestedReadMode { - /// Let the Reader choose the best available mode (including resident - /// inline reads where applicable). - Auto = 0, - /// Require a logical snapshot-namespace read. - Logical = 1, - /// Require the benchmark-gated raw accelerator (only valid once - /// UFI.3 approves it โ€” see addendum ยง3). - RawAccelerator = 2, -} - -impl RequestedReadMode { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::Auto), - 1 => Ok(Self::Logical), - 2 => Ok(Self::RawAccelerator), - other => Err(other), - } - } -} - -/// The read mode a [`ReadResponse::Bytes`] reports as actually used. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -pub enum ActualReadMode { - /// Read from the MFT-resident attribute value. - Resident = 0, - /// Logical open + read against the snapshot namespace. - Logical = 1, - /// Raw runlist/extent reconstruction. - RawAccelerator = 2, -} - -impl ActualReadMode { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::Resident), - 1 => Ok(Self::Logical), - 2 => Ok(Self::RawAccelerator), - other => Err(other), - } - } -} - -/// Stable error codes for a [`ReadResponse::Error`]. -/// -/// A narrow subset of `uffs_content_protocol::error::ErrorCode` relevant -/// specifically to a single read โ€” duplicated rather than shared, per -/// this crate's Layer-0-independence rationale (see `Cargo.toml`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u8)] -#[non_exhaustive] -pub enum ReaderErrorCode { - /// The requested snapshot lease is not known to the Reader. - LeaseInvalid = 0, - /// The requested snapshot lease has expired. - LeaseExpired = 1, - /// The candidate is not part of the finalized manifest the Reader - /// was given for this job. - CandidateNotInManifest = 2, - /// The opened object's identity does not match the manifest - /// (design-doc ยง5.2/ยง16 `IDENTITY_MISMATCH`). - IdentityMismatch = 3, - /// The requested stream was not found. - StreamNotFound = 4, - /// The requested logical range violates the EOF/VDL relationship. - VdlEofInvalid = 5, - /// The requested range falls outside validated bounds. - ExtentOutOfBounds = 6, - /// A transient I/O error occurred. - ReadIoTransient = 7, - /// A permanent I/O error occurred. - ReadIoPermanent = 8, - /// An internal Reader error not covered by another code. - InternalError = 9, -} - -impl ReaderErrorCode { - /// Serialize to the wire byte. - #[must_use] - pub const fn encode(self) -> u8 { - self as u8 - } - - /// Parse the wire byte. - /// - /// # Errors - /// Returns the offending byte if unrecognized. - pub const fn decode(byte: u8) -> Result { - match byte { - 0 => Ok(Self::LeaseInvalid), - 1 => Ok(Self::LeaseExpired), - 2 => Ok(Self::CandidateNotInManifest), - 3 => Ok(Self::IdentityMismatch), - 4 => Ok(Self::StreamNotFound), - 5 => Ok(Self::VdlEofInvalid), - 6 => Ok(Self::ExtentOutOfBounds), - 7 => Ok(Self::ReadIoTransient), - 8 => Ok(Self::ReadIoPermanent), - 9 => Ok(Self::InternalError), - other => Err(other), - } - } -} - -/// A read request from the Coordinator to the Reader (addendum ยง2.3). -/// -/// The Reader MUST treat every field here as untrusted input to be -/// revalidated against the snapshot itself โ€” `logical_offset` + -/// `maximum_logical_length` is never trusted as an in-bounds range purely -/// because the Coordinator sent it (design-doc ยง15.4, this crate's -/// `README`-level contract in the implementation plan ยง3.2). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ReadRequest { - /// Job this read belongs to. - pub job_id: [u8; 16], - /// Snapshot lease this read is scoped to. - pub snapshot_lease_id: u64, - /// Candidate being read. - pub candidate_id: u64, - /// Volume the candidate lives on. - pub volume_identity: VolumeIdentity, - /// Full NTFS file reference (never a bare MFT record index). - pub full_file_reference: u64, - /// The candidate's logical size, if the Coordinator already knows it - /// from the manifest that named this candidate. `Some` lets the - /// Reader skip its own `GetFileSizeEx` re-resolution and trust this - /// value directly โ€” a real (if rare) trust tradeoff, so this is - /// opt-in per request, not a blanket assumption: only the real VSS - /// Coordinator (`uffs-content::job::content_source::VssContentSource`) - /// populates it today, since its manifest size was itself read from - /// the same frozen snapshot this request targets. Any other/future - /// caller that leaves this `None` gets the Reader's original - /// always-re-verify behavior with no code changes required on its - /// part โ€” see `uffs-content-reader::reader::logical`'s module doc. - pub known_logical_size: Option, - /// Which stream to read. - pub stream_kind: StreamKind, - /// Logical byte offset to start reading at. - pub logical_offset: u64, - /// Maximum bytes to return for this request. - pub maximum_logical_length: u32, - /// Which read mode to use. - pub requested_mode: RequestedReadMode, - /// Caller-chosen nonce, echoed back for request/response correlation. - pub request_nonce: u64, -} - -impl ReadRequest { - /// Encode this request. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - out.extend_from_slice(&self.job_id); - write_u64_le(&mut out, self.snapshot_lease_id); - write_u64_le(&mut out, self.candidate_id); - self.volume_identity.encode(&mut out); - write_u64_le(&mut out, self.full_file_reference); - write_optional_u64(&mut out, self.known_logical_size); - out.push(self.stream_kind.encode()); - write_u64_le(&mut out, self.logical_offset); - write_u32_le(&mut out, self.maximum_logical_length); - out.push(self.requested_mode.encode()); - write_u64_le(&mut out, self.request_nonce); - out - } - - /// Decode a request from `reader`. - /// - /// # Errors - /// See [`DecodeError`]. - pub fn decode(reader: &mut Reader<'_>) -> Result { - let job_id: [u8; 16] = reader.read_array()?; - let snapshot_lease_id = reader.read_u64_le()?; - let candidate_id = reader.read_u64_le()?; - let volume_identity = VolumeIdentity::decode(reader)?; - let full_file_reference = reader.read_u64_le()?; - let known_logical_size = read_optional_u64(reader)?; - let stream_kind_byte = reader.read_u8()?; - let stream_kind = StreamKind::decode(stream_kind_byte).map_err(|byte| { - DecodeError::UnknownDiscriminant { - field: "stream_kind", - value: u64::from(byte), - } - })?; - let logical_offset = reader.read_u64_le()?; - let maximum_logical_length = reader.read_u32_le()?; - let requested_mode_byte = reader.read_u8()?; - let requested_mode = RequestedReadMode::decode(requested_mode_byte).map_err(|byte| { - DecodeError::UnknownDiscriminant { - field: "requested_mode", - value: u64::from(byte), - } - })?; - let request_nonce = reader.read_u64_le()?; - Ok(Self { - job_id, - snapshot_lease_id, - candidate_id, - volume_identity, - full_file_reference, - known_logical_size, - stream_kind, - logical_offset, - maximum_logical_length, - requested_mode, - request_nonce, - }) - } -} - -/// A read response from the Reader to the Coordinator (addendum ยง3.2). -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ReadResponse { - /// The request succeeded; `payload` is bounded by the request's - /// `maximum_logical_length`. - Bytes { - /// Logical offset these bytes start at (echoes the request). - logical_offset: u64, - /// Which mode the Reader actually used. - actual_mode: ActualReadMode, - /// The logical bytes read. - payload: Vec, - }, - /// The request failed. - Error { - /// Stable error code. - code: ReaderErrorCode, - /// Human-readable diagnostic message. - message: String, - }, -} - -/// Wire discriminant for [`ReadResponse`]'s two variants. -const RESPONSE_TAG_BYTES: u8 = 0; -/// Wire discriminant for [`ReadResponse`]'s two variants. -const RESPONSE_TAG_ERROR: u8 = 1; - -/// Bound on a single [`ReadResponse::Bytes`] payload. -/// -/// A conservative chunk ceiling; the Coordinator's own `max_chunk_bytes` -/// (see `uffs_content_protocol::frame::JobBegin`) governs the real -/// negotiated value. -pub const MAX_RESPONSE_PAYLOAD_BYTES: u32 = 64 * 1024 * 1024; - -impl ReadResponse { - /// Encode this response. - #[must_use] - pub fn encode(&self) -> Vec { - let mut out = Vec::new(); - match self { - Self::Bytes { - logical_offset, - actual_mode, - payload, - } => { - out.push(RESPONSE_TAG_BYTES); - write_u64_le(&mut out, *logical_offset); - out.push(actual_mode.encode()); - write_bytes_u32_prefixed(&mut out, payload); - } - Self::Error { code, message } => { - out.push(RESPONSE_TAG_ERROR); - out.push(code.encode()); - write_string_u16_prefixed(&mut out, message); - } - } - out - } - - /// Decode a response from `reader`. - /// - /// `max_payload_bytes` bounds the `Bytes` payload before allocation. - /// - /// # Errors - /// See [`DecodeError`]. - pub fn decode(reader: &mut Reader<'_>, max_payload_bytes: u32) -> Result { - let tag = reader.read_u8()?; - match tag { - RESPONSE_TAG_BYTES => { - let logical_offset = reader.read_u64_le()?; - let mode_byte = reader.read_u8()?; - let actual_mode = ActualReadMode::decode(mode_byte).map_err(|byte| { - DecodeError::UnknownDiscriminant { - field: "actual_mode", - value: u64::from(byte), - } - })?; - let payload = reader.read_bytes_u32_prefixed("payload", max_payload_bytes)?; - Ok(Self::Bytes { - logical_offset, - actual_mode, - payload, - }) - } - RESPONSE_TAG_ERROR => { - let code_byte = reader.read_u8()?; - let code = ReaderErrorCode::decode(code_byte).map_err(|byte| { - DecodeError::UnknownDiscriminant { - field: "code", - value: u64::from(byte), - } - })?; - let message = reader.read_string_u16_prefixed("message", MAX_MESSAGE_BYTES)?; - Ok(Self::Error { code, message }) - } - other => Err(DecodeError::UnknownDiscriminant { - field: "response_tag", - value: u64::from(other), - }), - } - } -} - -#[cfg(test)] -mod tests { - use proptest::prelude::*; - - use super::{ - ActualReadMode, DecodeError, ReadRequest, ReadResponse, Reader, ReaderErrorCode, - RequestedReadMode, StreamKind, VolumeIdentity, - }; - - fn sample_request() -> ReadRequest { - ReadRequest { - job_id: [1_u8; 16], - snapshot_lease_id: 42, - candidate_id: 12345, - volume_identity: VolumeIdentity { - volume_serial: 0x0102_0304_0506_0708, - volume_guid: b"{AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE}".to_vec(), - }, - full_file_reference: 0xABCD_EF01_2345_6789, - known_logical_size: Some(65536), - stream_kind: StreamKind::UnnamedData, - logical_offset: 4096, - maximum_logical_length: 65536, - requested_mode: RequestedReadMode::Auto, - request_nonce: 999, - } - } - - #[test] - fn read_request_round_trips() { - let request = sample_request(); - let bytes = request.encode(); - let mut reader = Reader::new(&bytes); - let decoded = ReadRequest::decode(&mut reader).unwrap(); - assert_eq!(decoded, request); - assert_eq!(reader.remaining(), 0); - } - - #[test] - fn read_request_with_no_known_logical_size_round_trips() { - let request = ReadRequest { - known_logical_size: None, - ..sample_request() - }; - let bytes = request.encode(); - let mut reader = Reader::new(&bytes); - let decoded = ReadRequest::decode(&mut reader).unwrap(); - assert_eq!(decoded, request); - assert_eq!(reader.remaining(), 0); - } - - #[test] - fn read_response_bytes_round_trips() { - let response = ReadResponse::Bytes { - logical_offset: 4096, - actual_mode: ActualReadMode::Logical, - payload: vec![1, 2, 3, 4, 5], - }; - let bytes = response.encode(); - let mut reader = Reader::new(&bytes); - let decoded = ReadResponse::decode(&mut reader, 1024).unwrap(); - assert_eq!(decoded, response); - } - - #[test] - fn read_response_error_round_trips() { - let response = ReadResponse::Error { - code: ReaderErrorCode::VdlEofInvalid, - message: "requested range past EOF".to_owned(), - }; - let bytes = response.encode(); - let mut reader = Reader::new(&bytes); - let decoded = ReadResponse::decode(&mut reader, 1024).unwrap(); - assert_eq!(decoded, response); - } - - #[test] - fn read_response_bytes_rejects_payload_over_max_before_allocation() { - let response = ReadResponse::Bytes { - logical_offset: 0, - actual_mode: ActualReadMode::Resident, - payload: vec![0_u8; 100], - }; - let bytes = response.encode(); - let mut reader = Reader::new(&bytes); - let err = ReadResponse::decode(&mut reader, 10).unwrap_err(); - assert!(matches!(err, DecodeError::LengthOutOfBounds { .. })); - } - - #[test] - fn read_response_rejects_unknown_tag() { - let bytes = vec![0xFF]; - let mut reader = Reader::new(&bytes); - let err = ReadResponse::decode(&mut reader, 1024).unwrap_err(); - assert!(matches!(err, DecodeError::UnknownDiscriminant { - field: "response_tag", - .. - })); - } - - #[test] - fn requested_read_mode_round_trips_all_variants() { - for value in 0_u8..=2 { - let mode = RequestedReadMode::decode(value).unwrap(); - assert_eq!(mode.encode(), value); - } - assert_eq!(RequestedReadMode::decode(3), Err(3)); - } - - #[test] - fn actual_read_mode_round_trips_all_variants() { - for value in 0_u8..=2 { - let mode = ActualReadMode::decode(value).unwrap(); - assert_eq!(mode.encode(), value); - } - assert_eq!(ActualReadMode::decode(3), Err(3)); - } - - #[test] - fn reader_error_code_round_trips_all_variants() { - for value in 0_u8..=9 { - let code = ReaderErrorCode::decode(value).unwrap(); - assert_eq!(code.encode(), value); - } - assert_eq!(ReaderErrorCode::decode(10), Err(10)); - } - - #[test] - fn stream_kind_round_trips() { - assert_eq!(StreamKind::decode(0).unwrap(), StreamKind::UnnamedData); - assert_eq!(StreamKind::decode(1), Err(1)); - } - - proptest! { - #![proptest_config(ProptestConfig::with_cases(200))] - - #[test] - fn read_request_round_trips_for_arbitrary_fields( - snapshot_lease_id: u64, - candidate_id: u64, - volume_serial: u64, - full_file_reference: u64, - known_logical_size: Option, - logical_offset: u64, - maximum_logical_length: u32, - request_nonce: u64, - ) { - let request = ReadRequest { - job_id: [7_u8; 16], - snapshot_lease_id, - candidate_id, - volume_identity: VolumeIdentity { - volume_serial, - volume_guid: b"{guid}".to_vec(), - }, - full_file_reference, - known_logical_size, - stream_kind: StreamKind::UnnamedData, - logical_offset, - maximum_logical_length, - requested_mode: RequestedReadMode::Logical, - request_nonce, - }; - let bytes = request.encode(); - let mut reader = Reader::new(&bytes); - let decoded = ReadRequest::decode(&mut reader).unwrap(); - prop_assert_eq!(decoded, request); - } - } -} diff --git a/crates/uffs-content-reader/Cargo.toml b/crates/uffs-content-reader/Cargo.toml deleted file mode 100644 index ed0acd218..000000000 --- a/crates/uffs-content-reader/Cargo.toml +++ /dev/null @@ -1,82 +0,0 @@ -# ============================================================================ -# uffs-content-reader: Privileged narrow Snapshot Reader -# ============================================================================ -# Layer 1 bin-only crate. The privileged half of the Coordinator/Reader -# split (uffs-ingest-implementation-plan.md ยง5): opens files directly -# against a leased VSS snapshot device by NTFS file reference -# (`OpenFileById`), re-resolves EOF from the opened handle, and returns -# bounded logical bytes to `uffs-content` (the unprivileged Coordinator) -# over a private named-pipe protocol (`uffs-content-reader-protocol`). -# -# Depends only on `uffs-content-reader-protocol` for its wire types โ€” NOT -# `uffs-core` (no query-engine surface belongs in a privileged reader) and -# NOT `uffs-mft` for v1 (no MFT parsing needed yet: `OpenFileById` locates -# the file directly by its 64-bit file reference, no path resolution or -# identity revalidation against the snapshot's own MFT required โ€” that's -# a deferred hardening step, see `src/reader/logical.rs`). -# ============================================================================ - -[package] -name = "uffs-content-reader" -description = "Privileged narrow Snapshot Reader for uffs-content โ€” opens VSS-snapshot files by NTFS file reference (Windows-only; other platforms build a stub)" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -authors.workspace = true -readme.workspace = true -keywords.workspace = true -categories.workspace = true -# Intentionally NOT published. Privileged, job-scoped companion process -# spawned only by uffs-content โ€” no standalone-useful library API outside -# the UFFS architecture. Reserve the name on crates.io to prevent -# squatting, but never carry content. Mirrors `uffs-broker`'s rationale. -publish.workspace = true - -[package.metadata.docs.rs] -all-features = true -rustdoc-args = ["--cfg", "docsrs"] -targets = ["x86_64-pc-windows-msvc"] -default-target = "x86_64-pc-windows-msvc" - -[[bin]] -name = "uffs-content-reader" -path = "src/main.rs" - -# Cross-platform: `main.rs` calls `uffs_version::handle_version!` on every -# platform, and the pure `read_plan` VDL/EOF function (the plan's own -# "highest-value unit-test target") is deliberately platform-independent -# so it runs on every CI lane, not just Windows. -[dependencies] -uffs-version.workspace = true -uffs-content-reader-protocol.workspace = true -# `ReadPlanError` (`src/reader/read_plan.rs`) โ€” the pure VDL/EOF function -# runs on every platform, so its error type's dependency does too. -thiserror.workspace = true - -# Windows-only deps: the actual privileged logic (`reader.rs` and -# `reader/`) is fully `#[cfg(windows)]` โ€” scoped so the non-Windows stub -# build (a tiny `eprintln!`-only `main.rs` path) doesn't pull these in -# unused. Mirrors `uffs-broker`'s own split (F5 / issue #205). -[target.'cfg(windows)'.dependencies] -anyhow.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true -windows.workspace = true -# Owner-only pipe DACL + `PipeName` validation โ€” the same primitives -# `uffs-daemon`'s named-pipe server uses. -uffs-security.workspace = true -# Safe named-pipe server wrapper (`tokio::net::windows::named_pipe`) โ€” -# avoids hand-rolling unsafe `CreateNamedPipeW` FFI for the pipe-server -# half; only a "current_thread" runtime is used, no multi-threaded pool. -tokio = { workspace = true, features = ["rt", "net", "io-util"] } - -[build-dependencies] -uffs-version = { workspace = true, features = ["build"] } -winresource.workspace = true - -[dev-dependencies] -proptest.workspace = true - -[lints] -workspace = true diff --git a/crates/uffs-content-reader/build.rs b/crates/uffs-content-reader/build.rs deleted file mode 100644 index f9efe4622..000000000 --- a/crates/uffs-content-reader/build.rs +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -// Build scripts run on the build host, not the shipping binary's target, so the -// workspace `deny(expect_used)` / `deny(unwrap_used)` runtime lints do not -// apply here; panicking on a build-host failure (missing icon / no resource -// compiler) is the idiomatic shape for a build script. -#![allow( - clippy::expect_used, - reason = "build scripts may panic on build-host failure; workspace deny-expect targets runtime code" -)] - -//! Build script for `uffs-content-reader`. -//! -//! Embeds Windows PE resources โ€” the UFFS icon, version info (company, product, -//! description), and the shared `app.manifest` โ€” into `uffs-content-reader.exe` -//! via [`winresource`](https://crates.io/crates/winresource), so the shipped -//! binary carries proper metadata instead of shipping bare. A bare binary is -//! both unbranded and a mild antivirus false-positive signal. MSVC-Windows -//! only; a no-op on every other build target. Mirrors `uffs-broker`'s build -//! script. - -fn main() { - uffs_version::emit_build_env(); - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-changed=../../assets/brand/icons/uffs.ico"); - println!("cargo:rerun-if-changed=../../assets/brand/app.manifest"); - - let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); - let target_env = std::env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); - if target_os != "windows" || target_env != "msvc" { - return; - } - - let mut res = winresource::WindowsResource::new(); - res.set_icon("../../assets/brand/icons/uffs.ico") - .set("ProductName", "UltraFastFileSearch") - .set( - "FileDescription", - "UFFS Content Reader (privileged VSS snapshot content reader)", - ) - .set("CompanyName", "SKY, LLC.") - .set("LegalCopyright", "(c) 2025-2026 SKY, LLC. MPL-2.0.") - .set("OriginalFilename", "uffs-content-reader.exe") - .set_manifest_file("../../assets/brand/app.manifest"); - res.compile() - .expect("winresource: failed to embed uffs-content-reader resources"); -} diff --git a/crates/uffs-content-reader/src/main.rs b/crates/uffs-content-reader/src/main.rs deleted file mode 100644 index 16a9baf49..000000000 --- a/crates/uffs-content-reader/src/main.rs +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! UFFS Content Reader โ€” privileged narrow Snapshot Reader. -//! -//! Spawned once per job by `uffs-content` (the Coordinator), while the -//! Coordinator is itself elevated. Opens files directly against one or -//! more leased VSS snapshot devices by NTFS file reference -//! (`OpenFileById`) and streams bounded logical byte ranges back over a -//! private named pipe (`uffs-content-reader-protocol::READER_PIPE_NAME`). -//! Not meant to be run manually outside of debugging. -//! -//! # Usage -//! -//! ```bash -//! uffs-content-reader --version -//! uffs-content-reader --device = [--device ...] -//! ``` - -// `reader::pipe_server` needs `alloc::sync::Arc`, so bring the crate -// into scope (Windows-only, matching `uffs-broker`'s own convention). -#[cfg(windows)] -extern crate alloc; - -// `reader::read_plan` is cross-platform (pure logic, no I/O โ€” see its -// own doc comment); the rest of `reader` (`logical`, `pipe_server`, -// dispatch) is `#[cfg(windows)]`-gated within the module itself, so -// this declaration stays unconditional. -mod reader; - -// Used only by the Windows-only pieces of the `reader` module tree -// (`reader.rs`'s dispatch, `reader/pipe_server.rs`'s framing) โ€” on -// non-Windows platforms this bin is a tiny `eprintln!`-only stub for -// everything except `reader::read_plan`, so this would otherwise be -// visible-but-unused. Mirrors `uffs-broker`'s / `uffs-vss-requestor`'s -// own convention (F5 / issue #205). -#[cfg(not(windows))] -use uffs_content_reader_protocol as _; - -/// Parse repeatable `--device DEVICE_PATH=LEASE_ID` arguments into a -/// `snapshot_lease_id -> device_path` lookup table. -/// -/// # Errors -/// Returns an error on an unknown argument, a malformed `--device` -/// value, or zero `--device` arguments (at least one is required). -#[cfg(windows)] -fn parse_device_args() -> anyhow::Result> { - let mut devices = std::collections::HashMap::new(); - let mut args = std::env::args().skip(1); - while let Some(arg) = args.next() { - if arg == "--device" { - let spec = args - .next() - .ok_or_else(|| anyhow::anyhow!("--device requires a DEVICE_PATH=LEASE_ID value"))?; - let (path, lease_id_str) = spec - .rsplit_once('=') - .ok_or_else(|| anyhow::anyhow!("--device value '{spec}' is missing '='"))?; - anyhow::ensure!( - !path.is_empty(), - "--device value '{spec}' has an empty path" - ); - let lease_id: u64 = lease_id_str.parse().map_err(|err| { - anyhow::anyhow!("--device value '{spec}' has an invalid lease id: {err}") - })?; - devices.insert(lease_id, path.to_owned()); - } else { - anyhow::bail!("unknown argument '{arg}'"); - } - } - anyhow::ensure!( - !devices.is_empty(), - "at least one --device = is required" - ); - Ok(devices) -} - -#[expect( - clippy::print_stderr, - reason = "no tracing subscriber exists yet at this point in startup; stderr is the \ - only diagnostic channel and is captured by the spawning Coordinator" -)] -fn main() { - uffs_version::handle_version!("uffs-content-reader"); - - #[cfg(windows)] - { - // `.with_max_level(DEBUG)`: without an explicit level, this - // subscriber's own default caps below `read_logical`'s per-phase - // timing instrumentation (`tracing::debug!` in - // `reader/logical.rs`) -- real-hardware runs confirmed zero of - // those lines ever reached the log file despite candidates - // actually being read, even though the binary had the - // instrumentation built in. This is diagnostic-only: every event - // still lands in this job's own per-run temp log file (see - // `uffs-content::job::reader_client`), not anywhere persistent. - let _guard = tracing_subscriber::fmt() - .with_max_level(tracing::Level::DEBUG) - .with_writer(std::io::stderr) - .try_init(); - let result = parse_device_args().and_then(|devices| reader::run(&devices)); - if let Err(err) = result { - eprintln!("uffs-content-reader: {err:#}"); - std::process::exit(1); - } - } - - #[cfg(not(windows))] - { - eprintln!( - "uffs-content-reader reads VSS snapshot content and requires Windows (elevated)." - ); - std::process::exit(1); - } -} diff --git a/crates/uffs-content-reader/src/reader.rs b/crates/uffs-content-reader/src/reader.rs deleted file mode 100644 index be75823f3..000000000 --- a/crates/uffs-content-reader/src/reader.rs +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Reader logic: a cross-platform pure core ([`read_plan`]) plus -//! Windows-only I/O (`logical`, `pipe_server`). -//! -//! [`read_plan`] is deliberately NOT gated behind `#[cfg(windows)]` โ€” -//! see its own module doc for why it runs (and is exhaustively tested) -//! on every platform, unlike the rest of this module. -//! -//! # Trust model (v1, Windows-only pieces below) -//! -//! This process is spawned directly by `uffs-content` (the Coordinator) -//! while it is itself elevated โ€” there is no Broker-mediated handle -//! duplication or Authenticode identity check on the connecting client -//! for this pipe (unlike the Broker's Snapshot Manager pipe). The named -//! pipe's owner-only DACL (`uffs_security::pipe::OwnerOnlySd`) is the -//! security boundary: only the current elevated user's linked/primary -//! token can open it at all. `FIRST_PIPE_INSTANCE` protects against -//! another process squatting the well-known name first. -//! -//! # Resolving a lease id to a snapshot device path -//! -//! `ReadRequest` carries `snapshot_lease_id` + `volume_identity`, but -//! (by design โ€” addendum ยง2.1-ยง2.4: the Coordinator's requests never -//! carry the snapshot handle or a raw device path over this wire -//! protocol) no device path field. This process instead receives every -//! `(device_path, snapshot_lease_id)` pair it will ever need as -//! `--device PATH=LEASE_ID` startup arguments โ€” mirroring `uffsd`'s own -//! `--device PATH=LETTER` flag for the same reason (multi-drive jobs -//! lease more than one snapshot) โ€” and looks up `snapshot_lease_id` in -//! that table per request. - -pub(crate) mod read_plan; - -#[cfg(windows)] -mod logical; -#[cfg(windows)] -pub(crate) mod pipe_server; - -#[cfg(windows)] -use std::collections::HashMap; - -#[cfg(windows)] -use uffs_content_reader_protocol::{ReadRequest, ReadResponse, ReaderErrorCode}; - -/// Dispatch one decoded `ReadRequest` into a `ReadResponse`, threading -/// this connection's [`logical::ReadHandleCache`] through the call โ€” -/// see that type's own doc comment for why. Always returns a fresh -/// cache: `Some` (the possibly-reused-or-reopened handle) on success, -/// `ReadHandleCache::empty()` on any failure, since a failed read leaves -/// the cached handle's state unknown. -/// -/// Every failure mode (unknown lease, open failure, read failure, -/// invalid VDL/EOF metadata) is caught here and turned into a typed -/// `ReadResponse::Error` โ€” this function never panics or propagates an -/// `Err` up to its caller, since a malformed *single* request must not -/// tear down the whole connection. -#[cfg(windows)] -fn dispatch_request( - request: &ReadRequest, - devices: &HashMap, - cache: logical::ReadHandleCache, -) -> (ReadResponse, logical::ReadHandleCache) { - let Some(device_path) = devices.get(&request.snapshot_lease_id) else { - return ( - ReadResponse::Error { - code: ReaderErrorCode::LeaseInvalid, - message: format!( - "snapshot_lease_id {} is not one of this process's --device leases", - request.snapshot_lease_id - ), - }, - logical::ReadHandleCache::empty(), - ); - }; - - match logical::read_logical( - device_path, - request.full_file_reference, - request.known_logical_size, - request.logical_offset, - request.maximum_logical_length, - cache, - ) { - Ok((payload, actual_mode, updated_cache)) => ( - ReadResponse::Bytes { - logical_offset: request.logical_offset, - actual_mode, - payload, - }, - updated_cache, - ), - Err(err) => ( - ReadResponse::Error { - code: ReaderErrorCode::ReadIoTransient, - message: format!("{err:#}"), - }, - logical::ReadHandleCache::empty(), - ), - } -} - -/// Run the Reader for the process's whole lifetime, resolving each -/// request's `snapshot_lease_id` against `devices`. -/// -/// # Errors -/// Returns an error only if the pipe itself cannot be created at all; -/// per-request failures are turned into `ReadResponse::Error` and never -/// propagate here. -#[cfg(windows)] -pub(crate) fn run(devices: &HashMap) -> anyhow::Result<()> { - pipe_server::run(devices) -} diff --git a/crates/uffs-content-reader/src/reader/logical.rs b/crates/uffs-content-reader/src/reader/logical.rs deleted file mode 100644 index 2f9e4eba7..000000000 --- a/crates/uffs-content-reader/src/reader/logical.rs +++ /dev/null @@ -1,444 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Stage A: the logical reader (`uffs-ingest-implementation-plan.md` -//! ยง5.1). -//! -//! Opens the target file directly against the snapshot device by its -//! NTFS file reference (`OpenFileById`) โ€” no path resolution needed, no -//! `uffs-mft` dependency. Re-resolves `EOF` from the freshly opened -//! handle, applies the VDL/EOF zero-synthesis rule -//! ([`super::read_plan::read_plan`]), and reads real bytes via -//! `ReadFile` at the requested offset. -//! -//! # Handle caching across a connection's consecutive requests -//! -//! `open_file_by_id` is not cheap: `OpenFileById` has to resolve/validate -//! the target FRS against the MFT, and against a VSS snapshot's -//! copy-on-write device that cost varies a lot request to request. -//! Real-hardware benchmarking found this dominating read time on large -//! files far more than actual disk throughput did โ€” a 2.76 GB file at -//! the old 64 KiB chunk size needed roughly 42,000 full open/close -//! cycles, one per chunk. [`ReadHandleCache`] lets [`read_logical`] reuse -//! the same open file handle across consecutive requests for the same -//! `full_file_reference`, opening fresh only when the target file -//! actually changes. The caller (`pipe_server`) owns one cache per -//! connection and threads it through every request on that connection โ€” -//! this only pays off because the Coordinator now pins one connection -//! per candidate's whole sequential read (see -//! `uffs-content::job::reader_client`'s module doc) rather than -//! round-robining a chunk at a time across the pool. -//! -//! [`ReadHandleCache`] also caches [`open_volume_hint`]'s handle, -//! independent of which file is currently being read: real-hardware -//! benchmarking against small-file-heavy drives (thousands of tiny -//! `.txt`/driver-readme files, each its own candidate) found this handle -//! was being opened and closed **fresh for every single candidate**, -//! even though it only identifies the *volume* โ€” never the file โ€” and -//! every candidate on one connection is read from the same volume (one -//! connection is only ever drawn from one lease's pool, so `device_path` -//! never actually changes mid-connection). Caching it removes one whole -//! `CreateFileW`+`CloseHandle` cycle per candidate at no correctness -//! cost, on top of the file-handle caching above. -//! -//! # Trusting a caller-supplied size (opt-in per request) -//! -//! [`read_logical`]'s `known_logical_size` parameter, when `Some`, skips -//! the `GetFileSizeEx` re-resolution on a cache miss entirely and uses -//! the caller's value as `EOF` directly โ€” real-hardware benchmarking -//! against small-file-heavy drives found `GetFileSizeEx` a real fraction -//! of per-candidate time even after the caching above. This is a genuine -//! (if rare) trust tradeoff: the value could theoretically be stale, so -//! it is opt-in per request via `ReadRequest::known_logical_size`, never -//! a blanket change to this function's own default behavior. Only the -//! real Coordinator populates it (`VssCandidateSource`'s manifest size -//! comes from parsing the exact same frozen snapshot this read targets, -//! so the two should always agree); any request that leaves it `None` -//! gets the original always-re-verify behavior with no code changes -//! required on the caller's part. -//! -//! # v1 simplifications (documented, not silent) -//! -//! - **VDL is treated as equal to EOF.** Getting the true NTFS valid data -//! length requires undocumented/internal APIs; treating it as EOF is correct -//! for the overwhelming majority of files โ€” only a sparse-tail file extended -//! via `SetFileValidData`/`SetEndOfFile` without writing has `vdl < eof`. -//! Revisit if that edge case matters in practice. -//! - **No identity revalidation against the snapshot's own MFT.** -//! `OpenFileById` inherently defends against the classic "FRS reused by a -//! different file" attack โ€” the encoded sequence number (high 16 bits of -//! `full_file_reference`) must match the live file's, or the open fails -//! outright. The deeper `identity.rs` piece from the design doc (re-parsing -//! the MFT record to cross-check size/ attributes after open) is deferred. -//! - **No Broker-mediated lease/volume cross-validation.** The Reader trusts -//! the Coordinator's `snapshot_device_identity` string as-is; only the -//! Coordinator's own successful VSS lease (validated by the Broker) gated -//! whether that string was ever handed out at all. - -use core::mem::size_of; -use core::time::Duration; -use std::os::windows::ffi::OsStrExt as _; -use std::time::Instant; - -use uffs_content_reader_protocol::ActualReadMode; -use windows::Win32::Foundation::{CloseHandle, HANDLE}; -use windows::Win32::Storage::FileSystem::{ - CreateFileW, FILE_BEGIN, FILE_FLAG_BACKUP_SEMANTICS, FILE_GENERIC_READ, FILE_ID_DESCRIPTOR, - FILE_ID_DESCRIPTOR_0, FILE_ID_TYPE, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, - GetFileSizeEx, OPEN_EXISTING, OpenFileById, ReadFile, SetFilePointerEx, -}; -use windows::core::PCWSTR; - -use super::read_plan::read_plan; - -/// `FileIdType` (the classic 64-bit NTFS file ID variant) โ€” matches -/// `uffs-core::compact::CompactRecord::file_ref`'s -/// `(sequence_number << 48) | frs` encoding, which is exactly the shape -/// `OpenFileById` expects for this discriminant. -const FILE_ID_TYPE_CLASSIC: FILE_ID_TYPE = FILE_ID_TYPE(0); - -/// RAII wrapper closing a raw `HANDLE` on drop. -struct OwnedHandle(HANDLE); - -impl Drop for OwnedHandle { - fn drop(&mut self) { - #[expect( - unsafe_code, - reason = "CloseHandle is an FFI call; `self.0` is always a valid, \ - owned handle this module opened" - )] - // SAFETY: `self.0` was returned by a successful `CreateFileW` or - // `OpenFileById` call in this module and is closed at most once - // (ownership is exclusive to this struct). - unsafe { - drop(CloseHandle(self.0)); - } - } -} - -#[expect( - unsafe_code, - reason = "windows file handles are thread-safe kernel objects, not thread-affine" -)] -// SAFETY: `OwnedHandle` owns a Windows `HANDLE` to a kernel-managed file -// object with no thread affinity and no unsynchronized interior -// mutability of its own โ€” moving ownership to another thread (as -// `ReadHandleCache` does across a `spawn_blocking` boundary in -// `pipe_server`) does not invalidate any aliasing assumptions. Handle -// cleanup remains centralized in `Drop`, above. -unsafe impl Send for OwnedHandle {} - -/// A previously-opened file handle, cached across a connection's -/// consecutive requests โ€” see this module's doc comment for why. -/// `pub(crate)` (rather than living entirely inside this module) because -/// `pipe_server` owns one instance per connection and threads it through -/// every request on that connection, across a `spawn_blocking` boundary. -/// -/// Also caches the [`open_volume_hint`] handle used to resolve -/// `OpenFileById` calls, independent of which file is being read โ€” -/// real-hardware benchmarking against small-file-heavy drives (drives -/// dominated by tiny `.txt`/driver-readme files) found this handle was -/// being opened and closed **fresh for every single candidate**, even -/// though it identifies only the *volume*, not the file, and every -/// candidate on one connection is read from the same volume. Caching it -/// removes one whole `CreateFileW`+`CloseHandle` cycle per candidate -/// with no correctness cost. -#[derive(Default)] -pub(crate) struct ReadHandleCache { - /// The cached open file handle, if any โ€” see [`CachedFileHandle`]. - file: Option, - /// The cached volume-hint handle, if any โ€” see [`CachedVolumeHint`]. - volume_hint: Option, -} - -impl ReadHandleCache { - /// A fresh cache holding nothing โ€” one per new connection. - pub(crate) const fn empty() -> Self { - Self { - file: None, - volume_hint: None, - } - } -} - -/// One cached open file handle plus the file reference and `EOF` it was -/// opened/resolved for โ€” [`read_logical`] reuses it only when a new -/// request's `full_file_reference` matches exactly. -struct CachedFileHandle { - /// The file reference this handle was opened against. - full_file_reference: u64, - /// The open handle itself. - handle: OwnedHandle, - /// `EOF` as resolved when this handle was opened. Deliberately not - /// re-queried on every cached-hit read: the file is being read - /// against a frozen VSS snapshot, not the live volume, so its size - /// cannot legitimately change for the life of that snapshot โ€” a - /// changed `EOF` on a subsequent query would indicate something has - /// gone wrong (a corrupted snapshot, a bug), not a real update to - /// react to. - eof: u64, -} - -/// One cached [`open_volume_hint`] handle plus the `device_path` it was -/// opened against โ€” [`read_logical`] reuses it for every candidate on -/// this connection, only reopening if `device_path` ever actually -/// changes (never happens in practice, since one physical connection is -/// only ever drawn from one lease's pool โ€” see -/// `uffs-content::job::reader_client`'s module doc โ€” but checking rather -/// than assuming keeps this correct even if that ever changed). -struct CachedVolumeHint { - /// The device path this handle was opened against. - device_path: String, - /// The open handle itself. - handle: OwnedHandle, -} - -/// Encode `text` as a NUL-terminated UTF-16 buffer for `PCWSTR` FFI calls. -fn to_wide_null(text: &str) -> Vec { - std::ffi::OsStr::new(text) - .encode_wide() - .chain(core::iter::once(0)) - .collect() -} - -/// Open a handle to the snapshot device's volume root โ€” used only as -/// `OpenFileById`'s volume hint. The file handle `OpenFileById` returns -/// is entirely independent of this one once opened, so the caller is -/// free to keep this handle around and reuse it across many -/// `OpenFileById` calls rather than reopening it per file โ€” see -/// [`CachedVolumeHint`]. -fn open_volume_hint(device_path: &str) -> anyhow::Result { - let wide = to_wide_null(device_path); - #[expect( - unsafe_code, - reason = "CreateFileW is an FFI call opening the snapshot volume root" - )] - // SAFETY: `wide` is a NUL-terminated UTF-16 buffer kept alive for the - // duration of the call; no other pointer arguments are passed. - let result = unsafe { - CreateFileW( - PCWSTR::from_raw(wide.as_ptr()), - FILE_GENERIC_READ.0, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - None, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, - None, - ) - }; - let handle = result.map_err(|err| { - anyhow::anyhow!("failed to open snapshot volume root {device_path}: {err}") - })?; - Ok(OwnedHandle(handle)) -} - -/// Open the target file directly by its 64-bit NTFS file reference, -/// using `volume_hint` to identify which volume it lives on. -fn open_file_by_id( - volume_hint: &OwnedHandle, - full_file_reference: u64, -) -> anyhow::Result { - let descriptor = FILE_ID_DESCRIPTOR { - dwSize: u32::try_from(size_of::()).unwrap_or(u32::MAX), - Type: FILE_ID_TYPE_CLASSIC, - Anonymous: FILE_ID_DESCRIPTOR_0 { - // NTFS classic file IDs are a bit-pattern reinterpretation, - // not a numeric value โ€” `cast_signed` preserves every bit. - FileId: full_file_reference.cast_signed(), - }, - }; - #[expect( - unsafe_code, - reason = "OpenFileById is an FFI call; `descriptor` is a valid, \ - fully-initialized FILE_ID_DESCRIPTOR for its lifetime" - )] - // SAFETY: `volume_hint.0` is a valid open handle on the target - // volume; `descriptor` is `#[repr(C)]`, matches the classic 64-bit - // `FileId` union arm the `Type` field selects, and lives until the - // call returns. - let result = unsafe { - OpenFileById( - volume_hint.0, - &raw const descriptor, - FILE_GENERIC_READ.0, - FILE_SHARE_READ, - None, - FILE_FLAG_BACKUP_SEMANTICS, - ) - }; - let handle = result.map_err(|err| { - anyhow::anyhow!("failed to open file by id {full_file_reference:#018x}: {err}") - })?; - Ok(OwnedHandle(handle)) -} - -/// Query a handle's current size (this call's own re-resolution of -/// `EOF` โ€” never trusted from manifest metadata, per ยง5.1 point 2). -fn file_size(handle: &OwnedHandle) -> anyhow::Result { - let mut size: i64 = 0; - #[expect(unsafe_code, reason = "GetFileSizeEx is an FFI call")] - // SAFETY: `handle.0` is a valid open file handle; `size` is a valid - // `&mut i64` for the duration of the call. - let result = unsafe { GetFileSizeEx(handle.0, &raw mut size) }; - result.map_err(|err| anyhow::anyhow!("GetFileSizeEx failed: {err}"))?; - Ok(size.cast_unsigned()) -} - -/// Move the handle's file pointer to `offset` from the start of the file. -fn seek(handle: &OwnedHandle, offset: u64) -> anyhow::Result<()> { - #[expect(unsafe_code, reason = "SetFilePointerEx is an FFI call")] - // SAFETY: `handle.0` is a valid open file handle; no output pointer - // is requested. - let result = unsafe { SetFilePointerEx(handle.0, offset.cast_signed(), None, FILE_BEGIN) }; - result.map_err(|err| anyhow::anyhow!("SetFilePointerEx failed: {err}")) -} - -/// Read exactly `buf.len()` bytes from the handle's current position. -fn read_exact(handle: &OwnedHandle, buf: &mut [u8]) -> anyhow::Result<()> { - let mut total_read: u32 = 0; - while (total_read as usize) < buf.len() { - let mut bytes_read: u32 = 0; - let dest = buf - .get_mut(total_read as usize..) - .ok_or_else(|| anyhow::anyhow!("read_exact: buffer index out of bounds"))?; - #[expect(unsafe_code, reason = "ReadFile is an FFI call")] - // SAFETY: `handle.0` is a valid open file handle; `dest` is a - // valid, exclusively-borrowed byte slice for the call's - // duration; no overlapped I/O is requested. - let result = unsafe { ReadFile(handle.0, Some(dest), Some(&raw mut bytes_read), None) }; - result.map_err(|err| anyhow::anyhow!("ReadFile failed: {err}"))?; - if bytes_read == 0 { - anyhow::bail!("ReadFile returned 0 bytes before the requested range was satisfied"); - } - total_read += bytes_read; - } - Ok(()) -} - -/// Perform one logical read: reuse `cache`'s open handle if it's already -/// open against `full_file_reference` (see this module's doc comment), -/// else open fresh by file reference and resolve `EOF` โ€” trusting -/// `known_logical_size` directly if `Some`, else re-querying via -/// `GetFileSizeEx` (see the "Trusting a caller-supplied size" doc -/// section for the tradeoff); apply the VDL/EOF rule, and read the -/// resulting real-byte range (zero-extending per the plan). -/// -/// Returns the (possibly newly-opened) handle back as an updated -/// [`ReadHandleCache`] for the caller to reuse on its next call โ€” on -/// success this always holds `Some`, even when nothing changed, so the -/// caller never has to special-case "cache unchanged" against "cache -/// now empty". -/// -/// # Errors -/// Returns an error if the device/file can't be opened, the read fails, -/// or the resolved metadata is invalid (`vdl > eof` โ€” never possible -/// here since VDL is derived from EOF, but `read_plan`'s contract keeps -/// that check in one place regardless of caller). The returned error -/// carries no cache โ€” the caller must treat the connection's cache as -/// empty afterward, matching how a round-trip failure already discards -/// the connection itself one level up. -pub(crate) fn read_logical( - device_path: &str, - full_file_reference: u64, - known_logical_size: Option, - logical_offset: u64, - maximum_logical_length: u32, - cache: ReadHandleCache, -) -> anyhow::Result<(Vec, ActualReadMode, ReadHandleCache)> { - let call_started_at = Instant::now(); - let cache_hit = matches!( - &cache.file, - Some(cached) if cached.full_file_reference == full_file_reference - ); - - let mut open_volume_hint_time = Duration::ZERO; - let volume_hint = match cache.volume_hint { - Some(cached) if cached.device_path == device_path => cached, - _ => { - let volume_hint_started_at = Instant::now(); - let handle = open_volume_hint(device_path)?; - open_volume_hint_time = volume_hint_started_at.elapsed(); - CachedVolumeHint { - device_path: device_path.to_owned(), - handle, - } - } - }; - - let mut open_file_by_id_time = Duration::ZERO; - let mut file_size_time = Duration::ZERO; - let mut trusted_known_size = false; - let (file_handle, eof) = match cache.file { - Some(cached) if cached.full_file_reference == full_file_reference => { - (cached.handle, cached.eof) - } - _ => { - let open_by_id_started_at = Instant::now(); - let file_handle = open_file_by_id(&volume_hint.handle, full_file_reference)?; - open_file_by_id_time = open_by_id_started_at.elapsed(); - - let eof = if let Some(size) = known_logical_size { - trusted_known_size = true; - size - } else { - let file_size_started_at = Instant::now(); - let eof = file_size(&file_handle)?; - file_size_time = file_size_started_at.elapsed(); - eof - }; - - (file_handle, eof) - } - }; - let vdl = eof; // v1 simplification โ€” see module doc. - - let plan = read_plan(vdl, eof, logical_offset, maximum_logical_length) - .map_err(|err| anyhow::anyhow!("{err}"))?; - - let mut payload = Vec::with_capacity(plan.total_len() as usize); - let read_started_at = Instant::now(); - if plan.real_bytes > 0 { - seek(&file_handle, logical_offset)?; - let mut real_buf = vec![0_u8; plan.real_bytes as usize]; - read_exact(&file_handle, &mut real_buf)?; - payload.extend_from_slice(&real_buf); - } - let read_time = read_started_at.elapsed(); - payload.resize(payload.len() + plan.zero_bytes as usize, 0); - - // PROFILING (temporary โ€” see the "confirm the per-file open/close - // overhead theory" investigation): one structured line per call, - // broken down by phase, so a real run's log can be aggregated - // (`grep 'read_logical: per-phase timing'` + average each field) to - // see whether open/close overhead or actual disk I/O dominates total - // read time for a given drive's workload. - tracing::debug!( - cache_hit, - trusted_known_size, - real_bytes = plan.real_bytes, - open_volume_hint_us = duration_micros(open_volume_hint_time), - open_file_by_id_us = duration_micros(open_file_by_id_time), - file_size_us = duration_micros(file_size_time), - read_us = duration_micros(read_time), - total_us = duration_micros(call_started_at.elapsed()), - "read_logical: per-phase timing" - ); - - let updated_cache = ReadHandleCache { - file: Some(CachedFileHandle { - full_file_reference, - handle: file_handle, - eof, - }), - volume_hint: Some(volume_hint), - }; - - Ok((payload, ActualReadMode::Logical, updated_cache)) -} - -/// Converts `duration` to whole microseconds, saturating instead of -/// panicking โ€” only used for diagnostic log fields, where a saturated -/// value is still obviously "very large" rather than a silent wrap. -fn duration_micros(duration: Duration) -> u64 { - u64::try_from(duration.as_micros()).unwrap_or(u64::MAX) -} diff --git a/crates/uffs-content-reader/src/reader/pipe_server.rs b/crates/uffs-content-reader/src/reader/pipe_server.rs deleted file mode 100644 index 79759ac63..000000000 --- a/crates/uffs-content-reader/src/reader/pipe_server.rs +++ /dev/null @@ -1,234 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Named-pipe server for [`READER_PIPE_NAME`]. -//! -//! Accepts every connection the Coordinator opens โ€” one per leased -//! drive, for read parallelism (see the local-only content-engine -//! architecture doc's Reader-parallelism section) โ€” and serves framed -//! `ReadRequest`/`ReadResponse` messages on each independently, until -//! that connection's own peer disconnects. Each request's disk I/O runs -//! via [`tokio::task::spawn_blocking`], so concurrent connections' -//! reads actually execute in parallel instead of serializing behind one -//! current-thread runtime. -//! -//! This process has no "last connection closed" lifecycle logic of its -//! own to worry about: the Coordinator kills it directly -//! (`ContentReader::shutdown`) once the job is done, mirroring -//! `uffs-ingest-implementation-plan.md`'s one-Reader-per-job lifecycle โ€” -//! the accept loop below just runs for as long as the process does, -//! same shape as `uffs-content::serve`'s own command-pipe accept loop. - -use alloc::sync::Arc; -use std::collections::HashMap; - -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; -use tokio::net::windows::named_pipe::{NamedPipeServer, PipeMode, ServerOptions}; -use uffs_content_reader_protocol::{READER_PIPE_NAME, ReadRequest, ReadResponse, ReaderErrorCode}; - -/// Matches the Coordinator-side `MAX_REQUEST_BYTES`-style bound used -/// for the Broker's Snapshot Manager pipe โ€” a generous ceiling for this -/// small, narrow API. -const MAX_REQUEST_BYTES: u32 = 64 * 1024; - -/// How long to back off before retrying pipe-instance creation after a -/// transient failure. -const PIPE_RETRY_BACKOFF: core::time::Duration = core::time::Duration::from_millis(100); - -/// Run the Reader's pipe server for the process's whole lifetime. -/// -/// # Errors -/// Returns an error only if the pipe itself cannot be created at all. -pub(crate) fn run(devices: &HashMap) -> anyhow::Result<()> { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - rt.block_on(serve(devices)) -} - -/// Accept every connection the Coordinator opens, spawning one task per -/// connection so multiple drives' reads run concurrently โ€” the async -/// body of [`run`]. -#[expect( - clippy::infinite_loop, - reason = "process-lifetime accept loop: this process is killed externally by the \ - Coordinator once the job is done, matching uffs-content::serve's own \ - command-pipe accept loop" -)] -async fn serve(devices: &HashMap) -> anyhow::Result<()> { - let pipe_name = uffs_security::pipe::PipeName::parse(READER_PIPE_NAME) - .map_err(|err| anyhow::anyhow!("invalid READER_PIPE_NAME: {err}"))?; - let sd = uffs_security::pipe::OwnerOnlySd::for_current_user() - .map_err(|err| anyhow::anyhow!("owner-only DACL build failed: {err}"))?; - let shared_devices = Arc::new(devices.clone()); - - let mut first_instance = true; - loop { - let mut server = match create_server(&pipe_name, &sd, first_instance) { - Ok(server) => server, - Err(err) => { - tracing::warn!(error = %err, "pipe instance unavailable; retrying shortly"); - tokio::time::sleep(PIPE_RETRY_BACKOFF).await; - continue; - } - }; - first_instance = false; - if server.connect().await.is_err() { - continue; - } - tracing::info!("Coordinator connected"); - - let devices_for_connection = Arc::clone(&shared_devices); - tokio::spawn(async move { - serve_requests(&mut server, &devices_for_connection).await; - }); - } -} - -/// Drain requests off `server` until the Coordinator disconnects that -/// connection (or sends a malformed request, which also ends it โ€” see -/// [`read_one_request`]). Every request's blocking disk I/O runs via -/// `spawn_blocking`, so a slow read on one connection never blocks any -/// other connection. -/// -/// Owns this connection's [`super::logical::ReadHandleCache`] for the -/// connection's whole lifetime, starting empty and threading the -/// updated cache back out of every `dispatch_request_blocking` call โ€” -/// see that type's own doc comment for why a per-connection cache is -/// safe and effective here (the Coordinator pins one connection per -/// candidate's whole sequential read). -async fn serve_requests(server: &mut NamedPipeServer, devices: &Arc>) { - let mut cache = super::logical::ReadHandleCache::empty(); - loop { - match read_one_request(server).await { - Ok(Some(request)) => { - match respond_to_one_request(server, request, devices, cache).await { - Some(updated_cache) => cache = updated_cache, - None => return, - } - } - Ok(None) => { - tracing::info!("Coordinator disconnected this connection"); - return; - } - Err(err) => { - tracing::warn!(error = %err, "malformed request; closing connection"); - return; - } - } - } -} - -/// Dispatch one request and write its response. Returns the updated -/// [`super::logical::ReadHandleCache`] for the caller to keep using on -/// this connection's next request, or `None` if the connection should -/// close (a write failure โ€” the read side already handles its own -/// EOF/malformed-request cases in [`serve_requests`]). -async fn respond_to_one_request( - server: &mut NamedPipeServer, - request: ReadRequest, - devices: &Arc>, - cache: super::logical::ReadHandleCache, -) -> Option { - let (response, updated_cache) = - dispatch_request_blocking(request, Arc::clone(devices), cache).await; - if let Err(err) = write_one_response(server, &response).await { - tracing::warn!(error = %err, "failed to write response; closing connection"); - return None; - } - Some(updated_cache) -} - -/// Run [`super::dispatch_request`]'s blocking disk I/O on tokio's -/// blocking thread pool, turning a panic there into a -/// [`ReaderErrorCode::InternalError`] response (with an emptied cache) -/// rather than propagating it (a single request's panic must not tear -/// down the whole connection, matching `dispatch_request`'s own -/// never-panics contract). -async fn dispatch_request_blocking( - request: ReadRequest, - devices: Arc>, - cache: super::logical::ReadHandleCache, -) -> (ReadResponse, super::logical::ReadHandleCache) { - tokio::task::spawn_blocking(move || super::dispatch_request(&request, &devices, cache)) - .await - .unwrap_or_else(|join_err| { - ( - ReadResponse::Error { - code: ReaderErrorCode::InternalError, - message: format!("read task panicked: {join_err}"), - }, - super::logical::ReadHandleCache::empty(), - ) - }) -} - -/// Read one `[u32 LE length][payload]`-framed [`ReadRequest`], or `Ok(None)` -/// on a clean EOF (the Coordinator disconnected between requests). -async fn read_one_request(server: &mut NamedPipeServer) -> anyhow::Result> { - let mut length_bytes = [0_u8; 4]; - match server.read_exact(&mut length_bytes).await { - Ok(_) => {} - Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), - Err(err) => return Err(err.into()), - } - let length = u32::from_le_bytes(length_bytes); - anyhow::ensure!( - length <= MAX_REQUEST_BYTES, - "request length {length} exceeds maximum {MAX_REQUEST_BYTES}" - ); - let mut payload = vec![0_u8; length as usize]; - server.read_exact(&mut payload).await?; - let mut reader = uffs_content_reader_protocol::codec::Reader::new(&payload); - let request = ReadRequest::decode(&mut reader)?; - Ok(Some(request)) -} - -/// Write one `[u32 LE length][payload]`-framed [`ReadResponse`]. -async fn write_one_response( - server: &mut NamedPipeServer, - response: &ReadResponse, -) -> anyhow::Result<()> { - let payload = response.encode(); - let length = u32::try_from(payload.len()) - .map_err(|err| anyhow::anyhow!("response payload too large to frame: {err}"))?; - server.write_all(&length.to_le_bytes()).await?; - server.write_all(&payload).await?; - server.flush().await?; - Ok(()) -} - -/// Build a single named-pipe server instance bound to `pipe_name` with -/// the owner-only `sd`. Set `first = true` ONLY for the initial -/// instance (enables `FIRST_PIPE_INSTANCE` squat protection) โ€” mirrors -/// `uffs-daemon`'s own `create_pipe_server` exactly. -fn create_server( - pipe_name: &uffs_security::pipe::PipeName, - sd: &uffs_security::pipe::OwnerOnlySd, - first: bool, -) -> anyhow::Result { - let mut sa = sd.as_security_attributes(); - - let mut opts = ServerOptions::new(); - opts.access_inbound(true) - .access_outbound(true) - .pipe_mode(PipeMode::Byte) - .in_buffer_size(65_536) - .out_buffer_size(65_536) - .reject_remote_clients(true); - if first { - opts.first_pipe_instance(true); - } - - #[expect(unsafe_code, reason = "Win32 FFI โ€” create named-pipe server")] - // SAFETY: `sa` is a valid `SECURITY_ATTRIBUTES` borrowing a - // `SECURITY_DESCRIPTOR` owned by `sd`, which outlives this call. - let server = unsafe { - opts.create_with_security_attributes_raw( - pipe_name.as_str(), - core::ptr::from_mut(&mut sa).cast(), - ) - }?; - - Ok(server) -} diff --git a/crates/uffs-content-reader/src/reader/read_plan.rs b/crates/uffs-content-reader/src/reader/read_plan.rs deleted file mode 100644 index 2c460cbee..000000000 --- a/crates/uffs-content-reader/src/reader/read_plan.rs +++ /dev/null @@ -1,314 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! The VDL/EOF zero-synthesis rule as a pure, allocation-free function. -//! -//! Per `uffs-ingest-implementation-plan.md` ยง5.1 point 4 (design-doc -//! ยง6.2): -//! -//! ```text -//! 0 <= offset < min(VDL, EOF) -> real bytes -//! VDL <= offset < EOF -> zeros -//! offset >= EOF -> nothing -//! ``` -//! -//! Deliberately platform-independent โ€” no file handle, no I/O โ€” so it -//! runs on every CI lane and is exhaustively unit-testable. This is -//! "the single highest-value unit-test target in the whole Reader" -//! per the implementation plan. -//! -//! Its only real (non-test) caller, `super::logical`, is -//! `#[cfg(windows)]` โ€” so on every other platform this module is -//! genuinely unused outside its own tests, permanently (not "deferred -//! until wired up" the way other dead-code states in this workspace -//! are). The `expect(dead_code)` attributes below reflect that on -//! purpose rather than silently suppressing an oversight. - -/// A validated request range rejected before any read is attempted. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[cfg_attr( - all(not(windows), not(test)), - expect( - dead_code, - reason = "only real (non-test) caller is #[cfg(windows)]; see module doc" - ) -)] -pub(crate) enum ReadPlanError { - /// `vdl > eof` is never valid NTFS metadata โ€” valid data length can - /// never exceed the file's own end-of-file. Reject rather than - /// silently clamp: this signals a Reader-side bug (or a stale/ - /// mismatched handle) worth surfacing, not papering over. - #[error("invalid metadata: valid data length {vdl} exceeds end of file {eof}")] - InvalidMetadata { - /// The (invalid) valid data length. - vdl: u64, - /// The (invalid, smaller-than-`vdl`) end of file. - eof: u64, - }, -} - -/// How to satisfy one `(vdl, eof, offset, requested_len)` read request: -/// read `real_bytes` from the file at `offset`, then synthesize -/// `zero_bytes` immediately after. Both may be zero. `real_bytes + -/// zero_bytes <= requested_len` always holds (see -/// [`read_plan`]'s invariant tests). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[cfg_attr( - all(not(windows), not(test)), - expect( - dead_code, - reason = "only real (non-test) caller is #[cfg(windows)]; see module doc" - ) -)] -pub(crate) struct ReadPlan { - /// Bytes to read from the real file, starting at `offset`. - pub real_bytes: u32, - /// Zero bytes to synthesize immediately after `real_bytes`. - pub zero_bytes: u32, -} - -impl ReadPlan { - /// Total bytes this plan produces (`real_bytes + zero_bytes`). - #[must_use] - #[cfg_attr( - all(not(windows), not(test)), - expect( - dead_code, - reason = "only real (non-test) caller is #[cfg(windows)]; see module doc" - ) - )] - pub(crate) const fn total_len(self) -> u32 { - self.real_bytes + self.zero_bytes - } -} - -/// Resolve a logical read request against a file's `vdl` (valid data -/// length) and `eof` (end of file) into a [`ReadPlan`]. -/// -/// # Errors -/// Returns [`ReadPlanError::InvalidMetadata`] if `vdl > eof` โ€” this -/// combination can never legitimately occur and must never be trusted -/// enough to compute a plan from. -#[cfg_attr( - all(not(windows), not(test)), - expect( - dead_code, - reason = "only real (non-test) caller is #[cfg(windows)]; see module doc" - ) -)] -pub(crate) const fn read_plan( - vdl: u64, - eof: u64, - offset: u64, - requested_len: u32, -) -> Result { - if vdl > eof { - return Err(ReadPlanError::InvalidMetadata { vdl, eof }); - } - - if offset >= eof || requested_len == 0 { - return Ok(ReadPlan { - real_bytes: 0, - zero_bytes: 0, - }); - } - - // `offset < eof` here, so this subtraction never underflows. - let available_to_eof = eof - offset; - let requested_len_u64 = requested_len as u64; - let capped_len = if available_to_eof < requested_len_u64 { - available_to_eof - } else { - requested_len_u64 - }; - // `capped_len <= requested_len_u64 <= u32::MAX`, so this cast is - // always exact โ€” no `as` truncation risk despite the lack of a - // `try_into` (this function is `const fn`, which `TryFrom` doesn't - // support as of this edition). - #[expect( - clippy::cast_possible_truncation, - reason = "capped_len <= requested_len (u32) by construction above" - )] - let capped_len_u32 = capped_len as u32; - - if offset >= vdl { - // Entirely within the zero region `[vdl, eof)`. - return Ok(ReadPlan { - real_bytes: 0, - zero_bytes: capped_len_u32, - }); - } - - // `offset < vdl` here, so this subtraction never underflows. - let real_available = vdl - offset; - let real_len = if real_available < capped_len { - real_available - } else { - capped_len - }; - #[expect( - clippy::cast_possible_truncation, - reason = "real_len <= capped_len_u32 by construction above" - )] - let real_len_u32 = real_len as u32; - - Ok(ReadPlan { - real_bytes: real_len_u32, - zero_bytes: capped_len_u32 - real_len_u32, - }) -} - -#[cfg(test)] -mod tests { - use proptest::prelude::*; - - use super::{ReadPlan, ReadPlanError, read_plan}; - - #[test] - fn offset_zero_within_vdl_is_all_real_bytes() { - let plan = read_plan(100, 100, 0, 50).expect("valid metadata"); - assert_eq!(plan, ReadPlan { - real_bytes: 50, - zero_bytes: 0 - }); - } - - #[test] - fn offset_at_eof_is_empty() { - let plan = read_plan(100, 100, 100, 50).expect("valid metadata"); - assert_eq!(plan, ReadPlan { - real_bytes: 0, - zero_bytes: 0 - }); - } - - #[test] - fn offset_past_eof_is_empty() { - let plan = read_plan(100, 100, 500, 50).expect("valid metadata"); - assert_eq!(plan, ReadPlan { - real_bytes: 0, - zero_bytes: 0 - }); - } - - #[test] - fn offset_exactly_at_vdl_is_all_zeros() { - // vdl=60, eof=100: offset==vdl starts the zero region exactly. - let plan = read_plan(60, 100, 60, 20).expect("valid metadata"); - assert_eq!(plan, ReadPlan { - real_bytes: 0, - zero_bytes: 20 - }); - } - - #[test] - fn offset_in_zero_region_is_all_zeros_capped_at_eof() { - // vdl=60, eof=100, offset=80, requested 50 -> only 20 bytes - // remain before eof, all zeros. - let plan = read_plan(60, 100, 80, 50).expect("valid metadata"); - assert_eq!(plan, ReadPlan { - real_bytes: 0, - zero_bytes: 20 - }); - } - - #[test] - fn range_spanning_vdl_eof_boundary_splits_real_then_zero() { - // vdl=60, eof=100, offset=50, requested 40 -> 10 real bytes - // (50..60) then 30 zero bytes (60..90). - let plan = read_plan(60, 100, 50, 40).expect("valid metadata"); - assert_eq!(plan, ReadPlan { - real_bytes: 10, - zero_bytes: 30 - }); - } - - #[test] - fn range_spanning_vdl_and_eof_is_capped_at_eof() { - // vdl=60, eof=100, offset=50, requested 1000 -> capped at eof: - // 10 real (50..60) + 40 zero (60..100). - let plan = read_plan(60, 100, 50, 1000).expect("valid metadata"); - assert_eq!(plan, ReadPlan { - real_bytes: 10, - zero_bytes: 40 - }); - } - - #[test] - fn zero_length_file_is_always_empty() { - let plan = read_plan(0, 0, 0, 100).expect("valid metadata"); - assert_eq!(plan, ReadPlan { - real_bytes: 0, - zero_bytes: 0 - }); - } - - #[test] - fn zero_requested_len_is_always_empty() { - let plan = read_plan(100, 100, 0, 0).expect("valid metadata"); - assert_eq!(plan, ReadPlan { - real_bytes: 0, - zero_bytes: 0 - }); - } - - #[test] - fn vdl_equal_eof_never_produces_zero_bytes() { - // No sparse tail at all: every in-bounds read is pure real bytes. - let plan = read_plan(100, 100, 10, 50).expect("valid metadata"); - assert_eq!(plan, ReadPlan { - real_bytes: 50, - zero_bytes: 0 - }); - } - - #[test] - fn vdl_greater_than_eof_is_rejected() { - let err = read_plan(100, 50, 0, 10).expect_err("vdl > eof must be rejected"); - assert_eq!(err, ReadPlanError::InvalidMetadata { vdl: 100, eof: 50 }); - } - - #[test] - fn total_len_never_exceeds_requested_len() { - for (vdl, eof, offset, requested_len) in [ - (0_u64, 0_u64, 0_u64, 10_u32), - (0, 100, 0, 10), - (50, 100, 0, 200), - (50, 100, 49, 5), - (50, 100, 50, 5), - (50, 100, 99, 5), - (50, 100, 100, 5), - (50, 100, 1000, 5), - ] { - let plan = read_plan(vdl, eof, offset, requested_len).expect("valid metadata"); - assert!( - plan.total_len() <= requested_len, - "plan {plan:?} exceeds requested_len {requested_len} for \ - (vdl={vdl}, eof={eof}, offset={offset})" - ); - } - } - - proptest! { - #![proptest_config(ProptestConfig::with_cases(1000))] - - /// Core invariant, fuzzed: whenever metadata is valid (`vdl <= - /// eof`), the plan's total length never exceeds either the - /// caller's requested length or the bytes actually remaining - /// before `eof`. - #[test] - fn total_len_is_bounded_for_arbitrary_valid_inputs( - vdl in 0_u64..1_000_000, - extra_to_eof in 0_u64..1_000_000, - offset in 0_u64..2_000_000, - requested_len in 0_u32..1_000_000, - ) { - let eof = vdl + extra_to_eof; - let plan = read_plan(vdl, eof, offset, requested_len) - .expect("vdl <= eof by construction"); - let remaining_to_eof = eof.saturating_sub(offset); - prop_assert!(u64::from(plan.total_len()) <= remaining_to_eof); - prop_assert!(plan.total_len() <= requested_len); - } - } -} diff --git a/crates/uffs-content/Cargo.toml b/crates/uffs-content/Cargo.toml deleted file mode 100644 index 0d2051bf6..000000000 --- a/crates/uffs-content/Cargo.toml +++ /dev/null @@ -1,155 +0,0 @@ -# ============================================================================ -# uffs-content: UFFS Content Service -# ============================================================================ -# Layer 4 App. Privileged, job-oriented process: for one volume/root, it -# creates a VSS snapshot, evaluates a UFFS candidate query against it, -# writes an immutable candidate manifest, and streams the successful -# candidates' logical file content to a downstream consumer (e.g. Docenta) -# over the framed protocol defined by `uffs-content-protocol`. -# -# Invocation model: like `uffsd`, driven by a structured JSON job spec (the -# query/filters/root/quotas), not interactive flags. `uffs-content` is the -# unprivileged coordinator: read-mode planning, manifest handling, protocol -# framing, and output live here; any privileged VSS-snapshot/raw-extent -# capability stays a narrow internal helper (extends `uffs-broker`'s -# pattern), never owned by this binary directly. Design references -# (docs/dev/architecture/, local-only, not tracked): the original -# `content-stream-tool-design.md` sketch, its -# `uffs-content-stream-enterprise-design-review.md` replacement-design -# review, and Docenta's `uffs-ingest-protocol-v2-vss.md`. Scaffold only; -# no VSS/MFT/query logic is wired up yet. -# ============================================================================ - -[package] -name = "uffs-content" -description = "UFFS Content Service (uffs-content) โ€” VSS-snapshot-scoped file content export for downstream consumers" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -authors.workspace = true -readme.workspace = true -keywords.workspace = true -categories.workspace = true -# Intentionally NOT published (yet). `uffs-content` is a privileged, -# Windows-only content-export producer with no standalone-useful library -# API outside the UFFS architecture. Reserve the name on crates.io to -# prevent squatting, but never carry content. Mirrors `uffs-broker`'s -# rationale in `crates/uffs-broker/Cargo.toml`. -publish.workspace = true - -[package.metadata.docs.rs] -all-features = true -rustdoc-args = ["--cfg", "docsrs"] -targets = ["x86_64-pc-windows-msvc"] -default-target = "x86_64-pc-windows-msvc" - -[[bin]] -name = "uffs-content" -path = "src/main.rs" - -[lib] -name = "uffs_content" -path = "src/lib.rs" - -[dependencies] -uffs-version.workspace = true -# Shared wire-protocol types (manifest, frames, job/candidate states) โ€” see -# `crates/uffs-content-protocol/`. -uffs-content-protocol.workspace = true -# Failure-log records + the finalized run summary โ€” see `src/run/`. Run -# state is intentionally ephemeral (no transactional per-candidate job -# database โ€” see `src/run/mod.rs` for the full rationale), so this is -# the only serialization this crate needs. -serde = { workspace = true, features = ["derive"] } -serde_json.workspace = true -# Job/run identifiers (`ManifestHeader::job_id`, etc.) โ€” see `src/job/`. -uuid.workspace = true -# Structured logging: `workflow::run_job`'s per-candidate content-read -# failure log (`src/job/workflow.rs`) runs in both the cross-platform -# fake-reader pipeline and the Windows-only real VSS pipeline, so this -# stays unconditional even though most other job-pipeline logging below -# is Windows-only. The macros themselves have no OS-specific behavior; -# only installing a subscriber to consume them (`tracing-subscriber`, -# Windows-only below) is Windows-specific. -tracing.workspace = true -# `workflow::read_lease_run_pipelined`'s bounded sliding-window content -# reader: a per-lease-run pool of worker threads pulling candidate -# indices off a bounded input channel and returning results through a -# bounded output channel, so it stays unconditional the same way -# `workflow.rs` itself does (exercised by both the cross-platform fake -# harness and the real Windows VSS pipeline). Already a workspace -# dependency (`uffs-mft`/this crate's own `reader_client.rs` use it the -# same way). -crossbeam-channel.workspace = true - -# Windows-only deps: the real Snapshot Manager pipe client -# (`src/job/snapshot_client.rs`) and the real VSS+MFT-query -# `CandidateSource` it backs are meaningful only on Windows (no VSS, no -# live MFT snapshot device, no Broker to talk to elsewhere) โ€” the fake -# `DirWalkCandidateSource`/`FsContentSource` backends remain -# cross-platform. Scoped so a non-Windows build doesn't pull in -# `uffs-broker-protocol`/`uffs-client`, matching `uffs-broker`'s own -# rationale for this split (F5 / issue #205). -# -# Deliberately NOT depending on `uffs-core` here, and not on `uffs-mft` -# for MFT reading/parsing/querying: this crate never reads or queries an -# MFT itself. It leases a VSS snapshot from the Broker, spawns an -# ephemeral `uffsd --device =` instance to do the actual -# MFT read + query evaluation (the daemon already owns all -# `uffs-mft`/`uffs-core` usage for that), and talks to it over the same -# RPC protocol `uffs-client` already implements for every other UFFS -# client. The one `uffs-mft` dependency below is narrowly scoped to its -# `detect_ntfs_drives` platform helper (a `GetLogicalDrives`/ -# `GetVolumeInformationW` check, no MFT involved) for the "no roots -# given, default to every local NTFS drive" case โ€” the same helper -# `uffs-daemon` itself uses for its own no-`--drive`-flag default. -[target.'cfg(windows)'.dependencies] -# Error handling for the pipe client and real VSS-backed CandidateSource -# (`?`/`anyhow::bail!`/`anyhow::anyhow!`) โ€” not used anywhere else in -# this crate today, so scoped alongside the rest of this Windows-only -# group rather than added unconditionally. -anyhow.workspace = true -# `SnapshotManagerRequest`/`SnapshotManagerResponse` wire types and -# `SNAPSHOT_PIPE_NAME` for the Broker's Snapshot Manager pipe. -uffs-broker-protocol.workspace = true -# Spawns the ephemeral `uffsd` instance and queries it over the same -# daemon RPC protocol every other UFFS client uses. -uffs-client.workspace = true -# `detect_ntfs_drives()` only โ€” see the block comment above. -uffs-mft.workspace = true -# Wire types for the private Coordinator<->Snapshot Reader protocol -# (`src/job/reader_client.rs`, `src/job/content_source.rs`'s -# `VssContentSource`). -uffs-content-reader-protocol.workspace = true -# Installs the subscriber consuming this crate's `tracing` events (see -# the unconditional `tracing` dependency above), for the -# `--self-test-*`/`--serve` entry points (`src/main.rs::init_tracing`) โ€” -# matches `uffs-broker`/`uffs-content-reader`'s own `fmt()` init. Scoped -# to Windows only: every entry point that calls `init_tracing()` is -# itself Windows-only (real VSS/Broker/ephemeral-daemon machinery). -tracing-subscriber.workspace = true -# Named-pipe server for the two-pipe transport (`src/serve/`) โ€” every -# other feature this crate needs (`rt`/`sync`/`time`/`io-util`) is -# already unconditional at workspace scope; only `net` (named pipes) -# needs enabling here, matching `uffs-daemon`'s own declaration. -tokio = { workspace = true, features = ["net"] } -# Owner-only pipe DACL + `PipeName` validation for the two-pipe -# transport server (`src/serve/pipe_io.rs`) โ€” the same primitives -# `uffs-daemon`'s and `uffs-content-reader`'s named-pipe servers use. -uffs-security.workspace = true - -[build-dependencies] -uffs-version = { workspace = true, features = ["build"] } -winresource.workspace = true - -[dev-dependencies] -tempfile.workspace = true -# Independent oracle digest for the E2E dir-walk parity harness -# (`tests/support/plain_walk.rs`) โ€” deliberately calls the `blake3` crate -# directly rather than going through `uffs-content-protocol::codec::digest`, -# so a bug in that wrapper can't hide from the parity test. -blake3.workspace = true - -[lints] -workspace = true diff --git a/crates/uffs-content/build.rs b/crates/uffs-content/build.rs deleted file mode 100644 index 1c7cc106d..000000000 --- a/crates/uffs-content/build.rs +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -// Build scripts run on the build host, not the shipping binary's target, so the -// workspace `deny(expect_used)` / `deny(unwrap_used)` runtime lints do not -// apply here; panicking on a build-host failure (missing icon / no resource -// compiler) is the idiomatic shape for a build script. -#![allow( - clippy::expect_used, - reason = "build scripts may panic on build-host failure; workspace deny-expect targets runtime code" -)] - -//! Build script for `uffs-content`. -//! -//! Two jobs, mirroring `uffs-daemon`/`uffs-content-reader`/`uffs-broker`: -//! -//! 1. Emits `UFFS_GIT_SHA` + build metadata (commit date, rustc, target, -//! profile) via [`uffs_version::emit_build_env`], so `uffs-content --version -//! --verbose` reports the real commit instead of `"unknown"`. Before this -//! script existed, `uffs-content.exe` was the only UFFS binary whose version -//! banner never carried a real build stamp โ€” exactly the "ran the -//! wrong/stale binary" trap the other binaries' build scripts exist to -//! close. -//! 2. On MSVC-Windows, embeds PE resources (UFFS icon, version info, -//! shared `app.manifest`) into `uffs-content.exe` via -//! [`winresource`](https://crates.io/crates/winresource), so the shipped -//! binary carries proper metadata instead of shipping bare. - -fn main() { - uffs_version::emit_build_env(); - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-changed=../../assets/brand/icons/uffs.ico"); - println!("cargo:rerun-if-changed=../../assets/brand/app.manifest"); - - let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); - let target_env = std::env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); - if target_os != "windows" || target_env != "msvc" { - return; - } - - let mut res = winresource::WindowsResource::new(); - res.set_icon("../../assets/brand/icons/uffs.ico") - .set("ProductName", "UltraFastFileSearch") - .set( - "FileDescription", - "UFFS Content Service (VSS-snapshot-scoped file content export)", - ) - .set("CompanyName", "SKY, LLC.") - .set("LegalCopyright", "(c) 2025-2026 SKY, LLC. MPL-2.0.") - .set("OriginalFilename", "uffs-content.exe") - .set_manifest_file("../../assets/brand/app.manifest"); - res.compile() - .expect("winresource: failed to embed uffs-content resources"); -} diff --git a/crates/uffs-content/src/job/candidate_source.rs b/crates/uffs-content/src/job/candidate_source.rs deleted file mode 100644 index 39cafc48e..000000000 --- a/crates/uffs-content/src/job/candidate_source.rs +++ /dev/null @@ -1,350 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Candidate enumeration: turns a job's root directory into the flat list -//! of files that will become manifest candidates. - -use std::path::{Path, PathBuf}; -use std::{fs, io}; - -/// One enumerated candidate, before it's assigned a `candidate_id` and -/// turned into a `CandidateRecord` (see [`super::manifest_builder`]). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CandidateEntry { - /// Path relative to the job's root. - pub relative_path: PathBuf, - /// Absolute path a [`super::content_source::ContentSource`] can open. - pub absolute_path: PathBuf, - /// Logical file size in bytes. - pub logical_size: u64, - /// Modification time, Unix milliseconds. - pub mtime_unix_ms: i64, - /// Filesystem-assigned unique identity for this file. In production - /// this is the NTFS file reference; this crate's cross-platform - /// source uses the OS's native per-volume file identifier, which is - /// stable across hard links the same way an NTFS file reference is. - pub file_reference: u64, - /// Which VSS snapshot lease (see `super::snapshot_client::SnapshotLease`) - /// this candidate's device path/file reference resolve against โ€” a - /// job may lease more than one drive. `0` (never a real lease id, - /// which the Broker assigns starting from 1) for - /// [`DirWalkCandidateSource`], which has no snapshot at all. - pub snapshot_lease_id: u64, -} - -/// Produces the candidate list for a job. -/// -/// The production implementation (not yet built โ€” UFI.1/UFI.2) evaluates -/// the job's UFFS query against an `MftIndex` built from a VSS snapshot. -/// [`DirWalkCandidateSource`] is a real, correct, but non-privileged -/// stand-in used until that lands: it walks the live filesystem directly, -/// which is exactly right for testing the Coordinator's own logic (this -/// is `uffs-ingest-implementation-plan.md` ยง9.5's "fast" harness) but is -/// not how a shipped job runs against NTFS. -/// -/// `Sync`: `run_job` enumerates every root concurrently (one thread per -/// root via `std::thread::scope`, mirroring the same shape as -/// [`super::content_source::ContentSource`]'s own `Sync` bound) โ€” real-hardware -/// benchmarking found root-by-root enumeration strictly sequential today, -/// even though each `enumerate` call opens its own independent connection -/// to the daemon and shares no mutable state with any other call. -pub trait CandidateSource: Sync { - /// Enumerate every regular file under `root`. - /// - /// # Errors - /// Propagates the underlying [`io::Error`] from directory traversal. - fn enumerate(&self, root: &Path) -> io::Result>; -} - -/// Enumerates candidates by walking the live filesystem with `std::fs`. -#[derive(Debug, Clone, Copy, Default)] -pub struct DirWalkCandidateSource; - -impl CandidateSource for DirWalkCandidateSource { - fn enumerate(&self, root: &Path) -> io::Result> { - let mut entries = Vec::new(); - walk(root, root, &mut entries)?; - Ok(entries) - } -} - -/// Recursively walks `dir` (rooted at `root`), appending one -/// [`CandidateEntry`] per regular file found, in deterministic -/// (path-sorted) order. -fn walk(root: &Path, dir: &Path, out: &mut Vec) -> io::Result<()> { - let mut dir_entries: Vec = fs::read_dir(dir)?.collect::>()?; - dir_entries.sort_by_key(fs::DirEntry::path); - - for entry in dir_entries { - let path = entry.path(); - let metadata = entry.metadata()?; - if metadata.is_dir() { - walk(root, &path, out)?; - } else if metadata.is_file() { - let relative_path = path.strip_prefix(root).unwrap_or(&path).to_path_buf(); - out.push(CandidateEntry { - relative_path, - absolute_path: path, - logical_size: metadata.len(), - mtime_unix_ms: mtime_unix_ms(&metadata), - file_reference: file_identity(&metadata), - snapshot_lease_id: 0, - }); - } - } - Ok(()) -} - -/// Extracts a file's modification time as Unix milliseconds, defaulting -/// to `0` if the platform can't report one or it predates the epoch. -fn mtime_unix_ms(metadata: &fs::Metadata) -> i64 { - metadata - .modified() - .ok() - .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok()) - .map_or(0, |duration| { - i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) - }) -} - -/// The file's inode number โ€” stable across hard links to the same file. -#[cfg(unix)] -fn file_identity(metadata: &fs::Metadata) -> u64 { - use std::os::unix::fs::MetadataExt as _; - metadata.ino() -} - -/// The file's NTFS file index โ€” stable across hard links to the same -/// file, the Windows analogue of a Unix inode number. -#[cfg(windows)] -fn file_identity(metadata: &fs::Metadata) -> u64 { - use std::os::windows::fs::MetadataExt as _; - metadata.file_index().unwrap_or(0) -} - -/// No native per-volume file identity is available on this platform; -/// hard-link detection simply won't apply here. -#[cfg(not(any(unix, windows)))] -const fn file_identity(_metadata: &fs::Metadata) -> u64 { - 0 -} - -/// Evaluates a job's query against the ephemeral, VSS-snapshot-backed -/// `uffsd` instance `prepare_ephemeral_daemon_for_roots` -/// ([`super::vss_orchestrator`]) spawned โ€” the real production -/// `CandidateSource`. -/// -/// Windows-only: VSS snapshots, and the ephemeral daemon that queries -/// them, don't exist on any other platform โ€” matching -/// [`super::ephemeral_daemon`]'s own scoping. -#[cfg(windows)] -pub struct VssCandidateSource<'a> { - /// UFFS name/path pattern (`JobRequest::query`), forwarded verbatim - /// to the daemon as `SearchParams::pattern`. - pattern: String, - // Remaining filter fields, forwarded verbatim to the matching - // `SearchParams` field โ€” see `JobRequest`'s doc comment for why - // this is a deliberately curated subset, not the full daemon - // filter surface. - /// Mirrors `SearchParams::ext`. - ext: Option, - /// Mirrors `SearchParams::min_size`. - min_size: Option, - /// Mirrors `SearchParams::max_size`. - max_size: Option, - /// Mirrors `SearchParams::newer`. - newer: Option, - /// Mirrors `SearchParams::older`. - older: Option, - /// Mirrors `SearchParams::exclude`. - exclude: Option, - /// Mirrors `SearchParams::attr`. - attr: Option, - /// The already-spawned, already-`Ready` ephemeral daemon covering - /// every drive this job leased. - daemon: &'a super::ephemeral_daemon::EphemeralDaemon, - /// Drive letter -> lease id, so each result row (which only carries - /// a drive letter) can be tagged with the lease - /// [`super::content_source::VssContentSource`] will need to read it - /// back afterward. - drive_to_lease: std::collections::HashMap, -} - -#[cfg(windows)] -impl<'a> VssCandidateSource<'a> { - /// Wrap an already-spawned, already-`Ready` ephemeral daemon, - /// copying every filter field off `request`. - #[must_use] - pub(crate) fn new( - request: &super::intake::JobRequest, - daemon: &'a super::ephemeral_daemon::EphemeralDaemon, - drive_to_lease: std::collections::HashMap, - ) -> Self { - Self { - pattern: request.query.clone(), - ext: request.ext.clone(), - min_size: request.min_size, - max_size: request.max_size, - newer: request.newer.clone(), - older: request.older.clone(), - exclude: request.exclude.clone(), - attr: request.attr.clone(), - daemon, - drive_to_lease, - } - } -} - -#[cfg(windows)] -impl CandidateSource for VssCandidateSource<'_> { - fn enumerate(&self, root: &Path) -> io::Result> { - tracing::info!(root = %root.display(), "candidate enumeration: connecting to ephemeral daemon"); - let mut client = self.daemon.connect().map_err(|err| { - tracing::warn!(root = %root.display(), error = %err, "candidate enumeration: connect failed"); - io::Error::other(err.to_string()) - })?; - - // Scope the search to this job's root: `path_contains` is a - // directory-path glob matched against each record's directory - // portion โ€” `*` restricts results to the subtree without - // needing this crate to walk anything itself. - let root_glob = format!("{}*", root.display()); - let params = uffs_client::protocol::SearchParams { - pattern: self.pattern.clone(), - filter_mode: Some(uffs_client::protocol::SearchFilterMode::Files), - path_contains: Some(root_glob), - limit: None, - ext: self.ext.clone(), - min_size: self.min_size, - max_size: self.max_size, - newer: self.newer.clone(), - older: self.older.clone(), - exclude: self.exclude.clone(), - attr: self.attr.clone(), - // Real-hardware benchmarking found reading candidates in - // match order (or even ascending-FRS order) leaves most of - // the achievable seek-distance reduction on the table for a - // volume that's been reorganized over years -- see - // docs/architecture/content-stream-tool-design.md. Bulk - // content reads are exactly the workload that benefits from - // this; interactive CLI searches never set it. - resolve_lcn_order: true, - ..Default::default() - }; - tracing::info!( - root = %root.display(), - params = %serde_json::to_string(¶ms).unwrap_or_else(|err| format!("")), - "candidate enumeration: sending search request" - ); - let response = client.search(¶ms).map_err(|err| { - tracing::warn!(root = %root.display(), error = %err, "candidate enumeration: search request failed"); - io::Error::other(err.to_string()) - })?; - - let rows = resolve_rows(response.payload)?; - let row_count = rows.len(); - log_first_rows(root, &rows); - let deduped = dedup_rows_by_file_reference_and_path(rows); - tracing::info!( - root = %root.display(), - rows = row_count, - deduped = deduped.len(), - "candidate enumeration: search response received" - ); - - let mut entries = Vec::with_capacity(deduped.len()); - for row in deduped { - let letter = row.drive.as_char(); - let Some(&lease_id) = self.drive_to_lease.get(&letter) else { - return Err(io::Error::other(format!( - "search result on drive {letter} has no matching lease for this job" - ))); - }; - let path = PathBuf::from(&row.path); - let relative_path = path.strip_prefix(root).unwrap_or(&path).to_path_buf(); - entries.push(CandidateEntry { - relative_path, - absolute_path: path, - logical_size: row.size, - // `SearchRow::modified` is Unix *microseconds*; - // `CandidateEntry::mtime_unix_ms` is Unix milliseconds. - mtime_unix_ms: row.modified / 1000, - file_reference: row.file_reference, - snapshot_lease_id: lease_id, - }); - } - Ok(entries) - } -} - -/// Log the first 20 raw rows exactly as the daemon returned them โ€” -/// *before* [`dedup_rows_by_file_reference_and_path`] touches anything โ€” -/// one full JSON row per log line so duplicates/corruption are visible -/// by eye without reconstructing them from a single giant blob. -/// Diagnostic aid for the row-count-inflation investigation (real -/// hardware has shown search returning far more rows than the true file -/// count for some drives, in ways the existing exact-duplicate dedup -/// doesn't catch). -#[cfg(windows)] -fn log_first_rows(root: &Path, rows: &[uffs_client::protocol::response::SearchRow]) { - for (index, row) in rows.iter().take(20).enumerate() { - tracing::info!( - root = %root.display(), - index, - row = %serde_json::to_string(row).unwrap_or_else(|err| format!("")), - "candidate enumeration: raw row" - ); - } -} - -/// Resolve a `SearchPayload` into its `SearchRow` list, reading a -/// shmem-backed result set from disk if the daemon chose that delivery -/// channel โ€” a job-scoped query is usually small enough to stay inline, -/// but a job matching many files could still cross the daemon's shmem -/// threshold. -#[cfg(windows)] -fn resolve_rows( - payload: uffs_client::protocol::response::SearchPayload, -) -> io::Result> { - use uffs_client::protocol::response::SearchPayload; - match payload { - SearchPayload::Empty => Ok(Vec::new()), - SearchPayload::InlineRows(rows) => Ok(rows), - SearchPayload::ShmemRows { path, .. } => { - uffs_client::shmem::read_search_results(Path::new(&path)) - .map(|response| response.payload.into_inline_rows().unwrap_or_default()) - } - SearchPayload::InlineBlob(_) | SearchPayload::ShmemBlob(_) => Err(io::Error::other( - "daemon returned a pre-formatted text blob instead of structured rows", - )), - } -} - -/// Collapse exact-duplicate rows the daemon's search occasionally returns -/// for the same physical file. Seen on real hardware: a fresh, uncached -/// VSS-device MFT parse returned two byte-for-byte identical rows (same -/// `file_reference`, same path, same everything) for files that have no -/// NTFS extension record and (for 3 of the 4 observed) only a single -/// on-disk `$FILE_NAME` attribute โ€” ruling out extension-record/attribute- -/// list double-counting as the cause. Root cause not yet pinned down -/// (suspected: a chunk-scheduling double-read in the sliding-window I/O -/// reader); this dedup is a defensive guard on this crate's own -/// consumption of the daemon's response, independent of wherever the -/// duplication actually originates. -/// -/// Keyed on `(file_reference, path)`, not `file_reference` alone: a -/// genuine hard link shares the same `file_reference` across multiple -/// *different* paths, and those must stay as separate candidates (that's -/// the whole point of `expand_links`). Only a row that is identical in -/// both file identity *and* path โ€” which is what a spurious double-count -/// produces โ€” gets collapsed. Preserves input order (first occurrence of -/// each key wins). -#[cfg(windows)] -fn dedup_rows_by_file_reference_and_path( - rows: Vec, -) -> Vec { - let mut seen: std::collections::HashSet<(u64, String)> = std::collections::HashSet::new(); - rows.into_iter() - .filter(|row| seen.insert((row.file_reference, row.path.clone()))) - .collect() -} diff --git a/crates/uffs-content/src/job/content_source.rs b/crates/uffs-content/src/job/content_source.rs deleted file mode 100644 index ab0d45ead..000000000 --- a/crates/uffs-content/src/job/content_source.rs +++ /dev/null @@ -1,176 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Content reading: turns a candidate + byte range into logical bytes. - -use std::fs::File; -use std::io::{self, Read as _, Seek as _, SeekFrom}; - -use super::candidate_source::CandidateEntry; - -/// Reads a bounded range of a candidate's logical content. -/// -/// The production implementation is `uffs-content`'s IPC client to -/// `uffs-content-reader` (`VssContentSource`, Windows-only โ€” not in -/// scope on this platform's rustdoc build), which resolves and reads -/// against a VSS snapshot device, never the live volume. -/// [`FsContentSource`] is a real, correct, but unprivileged stand-in: it -/// reads the live file directly with `std::fs`. See -/// [`super::candidate_source::CandidateSource`] for why that's the right -/// trade-off for this crate's own fast, cross-platform test harness. -/// -/// `Sync`: `workflow::run_job` reads several candidates' content -/// concurrently (one `std::thread::scope` thread each, sharing one -/// `&dyn ContentSource` โ€” see that module's "Concurrent reads, -/// sequential emission" doc section), so any implementation must -/// tolerate concurrent `begin_read` calls from different threads. The -/// [`ReadSession`] a single `begin_read` call produces is used from -/// exactly one thread for its whole lifetime, so it carries no such -/// bound itself. -pub trait ContentSource: Sync { - /// Begin a session for reading one candidate's *entire* content, - /// pinning whatever connection/handle state that read needs for the - /// session's whole lifetime rather than re-establishing it per - /// chunk. - /// - /// This exists because `VssContentSource`'s production - /// counterpart, `uffs-content-reader`, caches its open NTFS file - /// handle per connection across consecutive requests for the same - /// file (real-hardware benchmarking found the alternative โ€” a fresh - /// `OpenFileById` on every chunk โ€” dominates read time on large - /// files far more than actual disk throughput does). That cache - /// only helps if a candidate's chunks all land on the *same* - /// connection, which is exactly what pinning one session to one - /// connection for the read's whole duration guarantees. - /// - /// `candidate_id` is the same id `manifest_builder::build_manifest` - /// assigned this candidate (the caller already has it โ€” see - /// `workflow::run_job`'s `entries.iter().zip(&built.candidate_ids)`) - /// โ€” the production implementation needs it to correlate every read - /// in the session against the finalized manifest over the Reader's - /// wire protocol; [`FsContentSource`] ignores it entirely. - /// - /// # Errors - /// Propagates the underlying [`io::Error`] from whatever - /// establishing a session requires (e.g. opening the file, or - /// checking out a pooled connection). - fn begin_read( - &self, - candidate: &CandidateEntry, - candidate_id: u64, - ) -> io::Result>; -} - -/// One candidate's whole sequential read session, opened by -/// [`ContentSource::begin_read`]. Ends (releases whatever connection/ -/// handle it pinned) when dropped. -pub trait ReadSession { - /// Read up to `max_len` bytes starting at `offset`, continuing this - /// session. - /// - /// Returns fewer than `max_len` bytes only at EOF (matching a normal - /// [`std::io::Read::read`] short-read contract at end of file); an - /// empty result means `offset` was at or past EOF. - /// - /// # Errors - /// Propagates the underlying [`io::Error`] from seeking/reading. - fn read_at(&mut self, offset: u64, max_len: u32) -> io::Result>; -} - -/// Reads content directly from the live filesystem. -#[derive(Debug, Clone, Copy, Default)] -pub struct FsContentSource; - -impl ContentSource for FsContentSource { - fn begin_read( - &self, - candidate: &CandidateEntry, - _candidate_id: u64, - ) -> io::Result> { - let file = File::open(&candidate.absolute_path)?; - Ok(Box::new(FsReadSession { file })) - } -} - -/// [`FsContentSource`]'s session: just the one already-open file, kept -/// open for the candidate's whole read instead of reopened per chunk โ€” -/// mirrors `VssContentSource`'s real optimization even though a local -/// `std::fs::File` open is cheap enough that it wouldn't matter much on -/// its own; keeping the two implementations' shapes symmetric is the -/// point. -struct FsReadSession { - /// The candidate's already-open file handle. - file: File, -} - -impl ReadSession for FsReadSession { - fn read_at(&mut self, offset: u64, max_len: u32) -> io::Result> { - self.file.seek(SeekFrom::Start(offset))?; - - let capacity = usize::try_from(max_len).unwrap_or(usize::MAX); - let mut buffer = vec![0_u8; capacity]; - let mut total_read = 0_usize; - while total_read < buffer.len() { - let remaining = buffer.get_mut(total_read..).unwrap_or(&mut []); - let read = self.file.read(remaining)?; - if read == 0 { - break; - } - total_read += read; - } - buffer.truncate(total_read); - Ok(buffer) - } -} - -/// Reads content from a VSS snapshot via the privileged -/// `uffs-content-reader` process (see [`super::reader_client`]). -/// -/// Windows-only: VSS snapshots, and the Reader that reads them, don't -/// exist on any other platform โ€” matching [`super::ephemeral_daemon`]'s -/// and [`super::vss_orchestrator`]'s own scoping. -#[cfg(windows)] -pub struct VssContentSource { - /// The spawned Reader process + its live connection for this job. - reader: super::reader_client::ContentReader, -} - -#[cfg(windows)] -impl VssContentSource { - /// Wrap an already-spawned [`super::reader_client::ContentReader`]. - #[must_use] - pub(crate) const fn new(reader: super::reader_client::ContentReader) -> Self { - Self { reader } - } - - /// Tear down the wrapped Reader process. Explicit (rather than - /// relying on `Drop`) so a failed teardown is observable, mirroring - /// how [`super::vss_orchestrator::EphemeralJobResources::teardown`] - /// handles the ephemeral daemon. - /// - /// # Errors - /// Returns an error if the Reader process couldn't be killed. - pub(crate) fn shutdown(self) -> anyhow::Result<()> { - self.reader.shutdown() - } -} - -#[cfg(windows)] -impl ContentSource for VssContentSource { - fn begin_read( - &self, - candidate: &CandidateEntry, - candidate_id: u64, - ) -> io::Result> { - let session = self - .reader - .begin_read( - candidate.snapshot_lease_id, - candidate_id, - candidate.file_reference, - candidate.logical_size, - ) - .map_err(|err| io::Error::other(err.to_string()))?; - Ok(Box::new(session)) - } -} diff --git a/crates/uffs-content/src/job/ephemeral_daemon.rs b/crates/uffs-content/src/job/ephemeral_daemon.rs deleted file mode 100644 index 5b81a09bd..000000000 --- a/crates/uffs-content/src/job/ephemeral_daemon.rs +++ /dev/null @@ -1,179 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Spawns a single ephemeral `uffsd` instance covering every VSS -//! snapshot device this job's drives were leased for, connects to it -//! over the standard daemon RPC protocol, and tears it down when done. -//! -//! This is the revised design behind -//! `docs/dev/architecture/uffs-ingest-implementation-plan.md` ยง6.2 -//! (supersedes a literal reading of that section, per direct user -//! clarification): target selection is answered by an ephemeral daemon -//! instance loaded from the leased snapshot device(s) via `uffsd -//! --device =`, not by this crate reading or querying the -//! MFT directly โ€” the daemon already owns all `uffs-mft`/`uffs-core` -//! usage (see [`super::snapshot_client`]'s doc comment for the parallel -//! rationale on the lease side). - -use core::time::Duration; -use std::process::{Child, Command, Stdio}; -use std::time::Instant; - -use anyhow::{Context as _, Result}; -use uffs_client::connect_sync::UffsClientSync; -use uffs_client::daemon_ctl::{ephemeral_endpoint, find_daemon_exe}; - -/// How long [`EphemeralDaemon::spawn`] waits for the daemon to finish -/// loading every device source before giving up. -const READY_TIMEOUT: Duration = Duration::from_secs(120); - -/// How long [`EphemeralDaemon::spawn`] retries connecting to the -/// freshly spawned daemon's pipe/socket before treating it as dead on -/// arrival. -const CONNECT_RETRY_BUDGET: Duration = Duration::from_secs(10); - -/// Delay between connect retries while the daemon finishes binding its -/// endpoint. -const CONNECT_RETRY_INTERVAL: Duration = Duration::from_millis(50); - -/// A running ephemeral `uffsd` instance covering one or more VSS -/// snapshot devices. -pub(crate) struct EphemeralDaemon { - /// The spawned `uffsd` child process. Killed directly on - /// [`Self::shutdown`]/[`Drop`] rather than via the RPC `shutdown` - /// method โ€” see [`Self::shutdown`]'s doc comment for why. - child: Child, - /// This instance's IPC endpoint (Unix socket path, or Windows named - /// pipe path), from [`uffs_client::daemon_ctl::ephemeral_endpoint`]. - endpoint: String, -} - -impl EphemeralDaemon { - /// Spawn `uffsd --ephemeral-id --device - /// = ...` for every `(device_path, - /// drive_letter)` pair in `devices`, and wait until it reports - /// every device loaded. - /// - /// # Errors - /// Returns an error if `devices` is empty, the `uffsd` binary can't - /// be spawned, the pipe/socket never comes up, or the daemon - /// doesn't reach `Ready` within [`READY_TIMEOUT`]. - pub(crate) fn spawn(ephemeral_id: &str, devices: &[(String, char)]) -> Result { - anyhow::ensure!( - !devices.is_empty(), - "at least one device source is required to spawn an ephemeral daemon" - ); - - let exe = find_daemon_exe(); - // `--stdout/--stderr(Stdio::null())` used to discard every one of - // uffsd's own `tracing::info!` events (it defaults to logging to - // stdout at `info` level โ€” see `uffs-daemon::log_init`) โ€” meaning - // none of its internal timing (e.g. a slow `search` against a - // freshly-loaded, uncached device source) was ever visible, - // however much this crate's own logging improved. `--log-file` - // routes it to a discoverable file instead, and `--log-level` - // pins the level explicitly rather than relying on uffsd's own - // default matching ours. - let log_file = std::env::temp_dir().join(format!("uffsd-ephemeral-{ephemeral_id}.log")); - let mut command = Command::new(&exe); - command - .arg("--ephemeral-id") - .arg(ephemeral_id) - .arg("--log-level") - .arg("info") - .arg("--log-file") - .arg(&log_file) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - for (device_path, letter) in devices { - command - .arg("--device") - .arg(format!("{device_path}={letter}")); - } - - tracing::info!( - exe = %exe.display(), - device_count = devices.len(), - log_file = %log_file.display(), - "ephemeral daemon: spawning uffsd" - ); - let child = command - .spawn() - .with_context(|| format!("failed to spawn {}", exe.display()))?; - let pid = child.id(); - - let instance = Self { - child, - endpoint: ephemeral_endpoint(ephemeral_id), - }; - tracing::info!(pid, endpoint = %instance.endpoint, "ephemeral daemon: waiting for Ready"); - instance.await_ready()?; - tracing::info!(pid, "ephemeral daemon: Ready"); - Ok(instance) - } - - /// Connect to this instance's pipe/socket, retrying briefly while - /// the daemon finishes binding it, then block until it reports - /// `Ready` (every device source loaded). - fn await_ready(&self) -> Result<()> { - let connect_deadline = Instant::now() + CONNECT_RETRY_BUDGET; - let last_err = loop { - match UffsClientSync::connect_at(&self.endpoint) { - Ok(mut client) => { - return client - .await_ready(READY_TIMEOUT) - .context("ephemeral daemon did not become ready"); - } - Err(err) => { - if Instant::now() >= connect_deadline { - break err; - } - std::thread::sleep(CONNECT_RETRY_INTERVAL); - } - } - }; - Err(anyhow::anyhow!( - "could not connect to ephemeral daemon at {}: {last_err}", - self.endpoint - )) - } - - /// Open a fresh RPC connection to this running instance. - /// - /// # Errors - /// Returns an error if the connection can't be established. - pub(crate) fn connect(&self) -> Result { - UffsClientSync::connect_at(&self.endpoint) - .with_context(|| format!("failed to connect to ephemeral daemon at {}", self.endpoint)) - } - - /// Tear down this instance. - /// - /// Kills the process directly rather than using the RPC `shutdown` - /// method: that method reads the *resident* daemon's well-known PID - /// file for its shutdown nonce, which is meaningless (and unsafe to - /// reuse) for an ephemeral instance's own, differently located PID - /// file. Since this process spawned the child itself, a direct kill - /// is simpler and correct. - /// - /// # Errors - /// Returns an error if the process couldn't be killed. The process - /// is still waited on best-effort even on error. - pub(crate) fn shutdown(mut self) -> Result<()> { - self.child - .kill() - .context("failed to kill ephemeral daemon process")?; - drop(self.child.wait()); - Ok(()) - } -} - -impl Drop for EphemeralDaemon { - /// Best-effort safety net: if [`Self::shutdown`] was never called - /// explicitly (e.g. an earlier step returned an error), don't leak - /// the child process. A no-op if it was already reaped. - fn drop(&mut self) { - drop(self.child.kill()); - } -} diff --git a/crates/uffs-content/src/job/intake.rs b/crates/uffs-content/src/job/intake.rs deleted file mode 100644 index 76c0b3a69..000000000 --- a/crates/uffs-content/src/job/intake.rs +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Job intake: the structured request that starts a content-ingest run. - -use std::path::PathBuf; - -/// A request to ingest content under `roots`. -/// -/// This is the local job-submission format โ€” ordinary JSON, unlike the -/// Docenta-facing frame protocol, which uses the explicit binary codec -/// (addendum ยง5.4). -/// -/// `query` carries the UFFS name/path pattern (glob, regex with a `>` -/// prefix, or substring โ€” e.g. `"*.txt"`, or `"*"` to match everything). -/// The remaining fields mirror a narrow, deliberately curated subset of -/// the daemon's own `SearchParams` filter surface (`uffs-client`'s -/// `search` method โ€” the same one the CLI's `--ext`/`--min-size`/etc. -/// flags and `scripts/windows/api-validation.rs` exercise) so a job can -/// express the size/extension/date-bounded queries a real content-ingest -/// consumer (e.g. Docenta) actually needs, without this crate -/// re-implementing query parsing. All are forwarded verbatim to an -/// ephemeral `uffsd` instance by the real, -/// VSS+MFT-query-backed `super::candidate_source::VssCandidateSource`. -/// [`super::candidate_source::DirWalkCandidateSource`] (the fake, -/// cross-platform backend) ignores every filter field โ€” it always -/// matches every regular file under a root, equivalent to `query: "*"` -/// with no other filters set. -#[derive(Debug, Clone, PartialEq, Eq, Default, serde::Deserialize)] -pub struct JobRequest { - /// Identifier for the source this job's candidates came from. - /// `ManifestHeader::source_id` is derived deterministically from this - /// string (see [`super::workflow::run_job`]). - pub source_id: String, - /// Root directories to enumerate candidates under โ€” one job may span - /// multiple drives (`super::vss_orchestrator` already leases one VSS - /// snapshot per distinct drive letter among these and serves them all - /// from a single combined ephemeral daemon). Empty means "every local - /// NTFS drive" โ€” see `super::vss_job::run_vss_job`'s own doc - /// comment for how that default is resolved (Windows-only; the - /// cross-platform fake `DirWalkCandidateSource` path requires an - /// explicit, non-empty list, since "every drive" isn't a concept a - /// plain directory walk has). - #[serde(default)] - pub roots: Vec, - /// UFFS name/path pattern to evaluate against the snapshot's MFT - /// (e.g. `"*.txt"`); `"*"` matches every regular file. - pub query: String, - /// Comma-separated extension filter (e.g. `"txt"` or `"rs,toml,md"`). - /// Mirrors `SearchParams::ext`. - #[serde(default)] - pub ext: Option, - /// Minimum file size in bytes. Mirrors `SearchParams::min_size`. - #[serde(default)] - pub min_size: Option, - /// Maximum file size in bytes. Mirrors `SearchParams::max_size`. - #[serde(default)] - pub max_size: Option, - /// Modified-time lower bound (e.g. `"7d"`, `"24h"`, `"2026-01-15"`). - /// Mirrors `SearchParams::newer`. - #[serde(default)] - pub newer: Option, - /// Modified-time upper bound. Mirrors `SearchParams::older`. - #[serde(default)] - pub older: Option, - /// Exclude glob pattern (e.g. `"backup*"`). Mirrors - /// `SearchParams::exclude`. - #[serde(default)] - pub exclude: Option, - /// Attribute filter spec (e.g. `"hidden,compressed,!system"`). - /// Mirrors `SearchParams::attr`. - #[serde(default)] - pub attr: Option, - /// Content-delivery ceiling: a candidate whose `logical_size` exceeds - /// this is still enumerated in the manifest (so reap/tombstone - /// completeness holds โ€” see - /// [`uffs_content_protocol::frame::ReadMode::MetadataOnly`]'s own doc - /// comment) but its body is not streamed. `None` means no ceiling โ€” - /// every matched candidate's content is delivered regardless of size. - /// - /// Independent of `query`/`ext`/`min_size`/etc.: those decide which - /// files become candidates at all; this decides which already- - /// matched candidates are worth paying to stream, e.g. so a consumer - /// doesn't wait on a 100 GB file it has no intention of extracting - /// text from. Forwarded verbatim into - /// `JOB_BEGIN.max_content_delivery_bytes` - /// (see [`super::workflow::run_job`]). - #[serde(default)] - pub max_content_delivery_bytes: Option, -} diff --git a/crates/uffs-content/src/job/manifest_builder.rs b/crates/uffs-content/src/job/manifest_builder.rs deleted file mode 100644 index b17fc56fd..000000000 --- a/crates/uffs-content/src/job/manifest_builder.rs +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Builds a finalized candidate manifest from an enumerated candidate -//! list (design-doc ยง4.1 step 8: checksummed and finalized before any -//! candidate is processed). - -use uffs_content_protocol::codec::Digest; -use uffs_content_protocol::manifest::{ - AuthorizationMode, CandidateFlags, CandidateRecord, ManifestError, ManifestHeader, - ManifestTrailer, -}; -use uffs_content_protocol::path_encoding::WindowsPath; - -use super::candidate_source::CandidateEntry; - -/// A finalized manifest: encoded bytes plus the metadata a job needs to -/// build its `JOB_BEGIN`/`JOB_END` frames. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BuiltManifest { - /// Header + record section + trailer, exactly as they appear on disk. - pub bytes: Vec, - /// BLAKE3 digest over the header + record section โ€” the trailer's - /// `manifest_digest`, and what `JOB_BEGIN.manifest_digest` repeats. - pub manifest_digest: Digest, - /// `candidate_id` assigned to each input entry, in the same order as - /// the `entries` slice given to [`build_manifest`]. - pub candidate_ids: Vec, -} - -/// Assigns sequential `candidate_id`s and builds a finalized manifest for -/// `entries`. -/// -/// # Errors -/// Propagates [`ManifestError`] from encoding any header/record (only -/// possible for implausibly large fields โ€” see -/// [`CandidateRecord::encode`]). -pub fn build_manifest( - job_id: [u8; 16], - source_id: [u8; 16], - query_digest: Digest, - entries: &[CandidateEntry], -) -> Result { - let mut record_bytes = Vec::new(); - let mut candidate_ids = Vec::with_capacity(entries.len()); - for (index, entry) in entries.iter().enumerate() { - let candidate_id = index_to_candidate_id(index); - candidate_ids.push(candidate_id); - let record = CandidateRecord { - candidate_id, - file_reference: entry.file_reference, - logical_size: entry.logical_size, - valid_data_length: entry.logical_size, - mtime_unix_ms: entry.mtime_unix_ms, - candidate_flags: CandidateFlags::empty(), - path: WindowsPath::from_str_lossless(&entry.relative_path.to_string_lossy()), - }; - record_bytes.extend_from_slice(&record.encode()?); - } - - let candidate_count = u64::try_from(entries.len()).unwrap_or(u64::MAX); - let header = ManifestHeader { - format_version: 2, - job_id, - source_id, - volume_serial: 0, - volume_guid: Vec::new(), - snapshot_id: Vec::new(), - snapshot_created_unix_ms: 0, - query_digest, - authorization_mode: AuthorizationMode::AdminExport, - candidate_count, - record_section_length: u64::try_from(record_bytes.len()).unwrap_or(u64::MAX), - }; - - let mut bytes = header.encode()?; - bytes.extend_from_slice(&record_bytes); - let manifest_digest = ManifestTrailer::compute_digest(&bytes); - let trailer = ManifestTrailer { - candidate_count_repeat: candidate_count, - manifest_digest, - }; - bytes.extend_from_slice(&trailer.encode()); - - Ok(BuiltManifest { - bytes, - manifest_digest, - candidate_ids, - }) -} - -/// `candidate_id` assignment policy: sequential, 1-based (`0` is left -/// unused as a future not-a-candidate sentinel might want it). -fn index_to_candidate_id(index: usize) -> u64 { - u64::try_from(index).unwrap_or(u64::MAX).saturating_add(1) -} diff --git a/crates/uffs-content/src/job/mod.rs b/crates/uffs-content/src/job/mod.rs deleted file mode 100644 index 39d8bbd87..000000000 --- a/crates/uffs-content/src/job/mod.rs +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Job intake and execution: the real Coordinator workflow described in -//! `docs/dev/architecture/uffs-ingest-implementation-plan.md` ยง6. -//! -//! Built against swappable [`candidate_source::CandidateSource`] / -//! [`content_source::ContentSource`] backends so it can be exercised -//! today โ€” via this crate's own -//! [`candidate_source::DirWalkCandidateSource`] / -//! [`content_source::FsContentSource`] โ€” ahead of the real -//! Broker/Reader-backed implementations landing (UFI.1/UFI.2). This is -//! also what powers the plan's ยง9.5 "fast" end-to-end dir-walk parity -//! harness (`crates/uffs-content/tests/e2e_dir_walk_parity_fake_reader.rs`). - -pub mod candidate_source; -pub mod content_source; -pub mod intake; -pub mod manifest_builder; -// In-memory per-job resume state (which candidates a reconnecting -// consumer still needs streamed). Cross-platform: pure logic, no VSS/ -// pipe dependency of its own. `pub(crate)` so `crate::serve` (the -// two-pipe transport server, a sibling of this module) can reach it. -pub(crate) mod registry; -// Credit-based backpressure tracker (design-doc ยง13). Cross-platform: -// pure logic, no VSS/pipe dependency of its own. `pub(crate)` for the -// same reason as `registry`. -pub(crate) mod window; -// Coordinator-side client for the Broker's Snapshot Manager pipe โ€” the -// real VSS lease backend `candidate_source`'s VSS-backed implementation -// calls into. Windows-only: no VSS, no Broker to talk to elsewhere, -// matching the `[target.'cfg(windows)'.dependencies]` scoping in -// Cargo.toml this module's own dependency (`uffs-broker-protocol`) -// requires. -#[cfg(windows)] -pub mod snapshot_client; -// Spawns/connects/tears down the ephemeral `uffsd` instance that -// answers target-selection queries against a leased VSS snapshot. -// Windows-only for the same reason as `snapshot_client`. -#[cfg(windows)] -pub mod ephemeral_daemon; -// Ties `snapshot_client` and `ephemeral_daemon` together: one lease per -// distinct drive, one combined daemon. Windows-only for the same reason -// as its two dependents. -#[cfg(windows)] -pub mod vss_orchestrator; -// Coordinator-side client for `uffs-content-reader-protocol` โ€” spawns -// and talks to the privileged `uffs-content-reader` process -// `content_source::VssContentSource` reads through. Windows-only for -// the same reason as its siblings. -#[cfg(windows)] -pub mod reader_client; -pub mod workflow; -// End-to-end VSS-backed job execution โ€” ties every piece above together -// into the real production entry point. Windows-only for the same -// reason as its dependencies. -#[cfg(windows)] -pub mod vss_job; -// Elevated smoke test: real VSS + real Reader playback, reused by the -// `--self-test-vss-playback` CLI flag and the `#[ignore]` cargo test. -// Windows-only for the same reason as `vss_job`. -#[cfg(windows)] -pub mod self_test; - -#[cfg(test)] -mod tests; diff --git a/crates/uffs-content/src/job/reader_client.rs b/crates/uffs-content/src/job/reader_client.rs deleted file mode 100644 index bfff41666..000000000 --- a/crates/uffs-content/src/job/reader_client.rs +++ /dev/null @@ -1,480 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Coordinator-side client for `uffs-content-reader-protocol`. -//! -//! Spawns `uffs-content-reader --device = ...` -//! once per job โ€” mirrors [`super::ephemeral_daemon`]'s spawn model, but -//! for the content-reading phase rather than target selection โ€” and -//! opens **a pool of persistent connections per leased drive** to its -//! fixed `READER_PIPE_NAME`, sending framed `ReadRequest`/`ReadResponse` -//! messages over whichever pool matches a read's `snapshot_lease_id`. -//! -//! Each pool's *size* is chosen by the caller (`vss_job.rs`), not fixed -//! here โ€” see `CONNECTIONS_PER_DRIVE`'s doc comment for why an -//! HDD-backed drive gets exactly one connection (real read concurrency -//! of 1, preserving the enumeration/MFT order candidates were collected -//! in) while an NVMe/SSD-backed drive gets many. Keying pools by -//! `snapshot_lease_id` means reads for different drives never contend -//! on each other's pool regardless of size. -//! -//! Each pool is a bounded [`crossbeam_channel`] of already-open -//! connections. A candidate's whole sequential read pins exactly one -//! connection for its entire duration โ€” see `ContentReader::begin_read`/ -//! `ReaderSession` โ€” rather than checking one out fresh per chunk: -//! real-hardware benchmarking found `uffs-content-reader` caches its -//! open NTFS file handle per connection across consecutive requests for -//! the same file (see that crate's `reader/logical.rs`), so consecutive -//! chunks of one candidate landing on *different* connections (which a -//! per-chunk checkout would do, round-robin) would defeat that cache -//! entirely. Checking out a connection is a blocking `recv` (waits for -//! one to free up rather than erroring); a connection that survives its -//! session unscathed is returned for reuse when the session drops. A -//! connection that errors mid-round-trip (frame desync, pipe reset) is -//! deliberately *not* returned โ€” better to shrink that drive's pool by -//! one than serve subsequent reads over a connection in an unknown -//! framing state. -//! -//! Mirrors [`super::snapshot_client`]'s connect style (plain -//! `std::fs::OpenOptions` + `Read`/`Write`) and wire framing -//! (`[u32 LE length][payload]`) exactly โ€” see that module's doc comment -//! for the rationale. - -use alloc::sync::Arc; -use core::sync::atomic::{AtomicU64, Ordering}; -use core::time::Duration; -use std::collections::HashMap; -use std::io::{Read as _, Write as _}; -use std::process::{Child, Command, Stdio}; -use std::time::Instant; - -use anyhow::{Context as _, Result}; -use crossbeam_channel::{Receiver, Sender}; -use uffs_content_reader_protocol::codec::Reader as WireReader; -use uffs_content_reader_protocol::{ - MAX_RESPONSE_PAYLOAD_BYTES, READER_PIPE_NAME, ReadRequest, ReadResponse, RequestedReadMode, - StreamKind, VolumeIdentity, -}; - -use super::content_source::ReadSession; - -/// How long to retry connecting to the freshly spawned Reader's pipe -/// while it finishes binding it. -const CONNECT_RETRY_BUDGET: Duration = Duration::from_secs(10); - -/// Delay between connect retries. -const CONNECT_RETRY_INTERVAL: Duration = Duration::from_millis(50); - -/// Connections given to an NVMe/SSD-backed drive's pool. -/// -/// Each connection is a plain, unpipelined request/response round trip -/// over a named pipe, so this is that drive's real read concurrency โ€” -/// see the module doc comment. Reads here are small files (an -/// IPC-round-trip-bound workload, not a bytes/sec-bound one), so a -/// value well above typical disk queue depth is appropriate for a -/// no-seek-penalty medium designed around deep concurrent queues. -/// -/// An HDD (or removable/virtual/unknown โ€” anything -/// [`uffs_mft::platform::DriveType::is_high_performance`] doesn't claim) -/// gets exactly 1 connection instead, chosen by the caller -/// (`vss_job.rs`) โ€” not this constant. Racing multiple concurrent reads -/// against the same spinning disk scatters its head across every -/// in-flight read's location instead of letting it sweep through -/// candidates in the order they were enumerated (which, since -/// candidates come off the MFT roughly in on-disk order, approximates -/// sequential access) โ€” pure seek-time waste for a medium where seeks, -/// not bandwidth, are the bottleneck. -pub(crate) const CONNECTIONS_PER_DRIVE: usize = 8; - -/// A running `uffs-content-reader` process + its live pipe connection -/// pools (one per leased drive, sized by the caller โ€” see -/// `CONNECTIONS_PER_DRIVE`), held for the whole job's content-reading -/// phase. -pub(crate) struct ContentReader { - /// The spawned `uffs-content-reader` child process. Killed on - /// [`Self::shutdown`]/[`Drop`] โ€” this process spawned it, so a - /// direct kill is simplest and correct (mirrors - /// [`super::ephemeral_daemon::EphemeralDaemon::shutdown`]). - child: Child, - /// One connection pool per leased drive, keyed by - /// `snapshot_lease_id` โ€” see the module doc comment. - connections: HashMap, - /// This job's id, echoed into every `ReadRequest`. - job_id: [u8; 16], - /// Monotonically increasing nonce for request/response correlation. - /// Shared (not per-session) via `Arc` so every session drawn from - /// every drive's pool still produces globally unique nonces. - next_nonce: Arc, -} - -/// A bounded pool of already-open pipe connections for one drive. -/// -/// `checkout`/`checkin` are the two ends of the same bounded -/// [`crossbeam_channel`], pre-filled at construction with as many -/// connections as the caller asked for โ€” see the module doc comment for -/// the checkout/checkin/drop-on-error contract. -struct ConnectionPool { - /// Checked-in (idle) connections, ready to be checked out. - checkout: Receiver, - /// The other end of the same channel โ€” returns a connection after a - /// successful round trip. - checkin: Sender, -} - -impl ConnectionPool { - /// Open `pool_size` fresh connections and fill a new pool with them - /// (clamped to at least 1 โ€” a pool can never be usefully empty). - fn connect(lease_id: u64, pool_size: usize) -> Result { - let clamped_pool_size = pool_size.max(1); - let (checkin, checkout) = crossbeam_channel::bounded(clamped_pool_size); - for _ in 0..clamped_pool_size { - let pipe = connect_with_retry() - .with_context(|| format!("failed to open a connection for lease {lease_id}"))?; - // Never blocks: the channel's capacity is exactly - // clamped_pool_size and we send exactly that many. - checkin.try_send(pipe).map_err(|err| { - anyhow::anyhow!("connection pool for lease {lease_id} overfilled: {err}") - })?; - } - Ok(Self { checkout, checkin }) - } -} - -impl ContentReader { - /// Spawn `uffs-content-reader --device = - /// ...` for every `(device_path, lease_id, _)` in `devices`, and open - /// a connection pool of the given size for each โ€” see - /// `CONNECTIONS_PER_DRIVE` for how the caller should choose that - /// size per drive. - /// - /// # Errors - /// Returns an error if `devices` is empty, the binary can't be - /// spawned, or any connection never comes up within - /// [`CONNECT_RETRY_BUDGET`]. - pub(crate) fn spawn(job_id: [u8; 16], devices: &[(String, u64, usize)]) -> Result { - anyhow::ensure!( - !devices.is_empty(), - "at least one device is required to spawn a content reader" - ); - - let exe = find_reader_exe(); - // uffs-content-reader's own tracing subscriber writes to its - // process's stderr with no level cap (see its main.rs), but - // stderr used to be piped to Stdio::null() โ€” discarding every - // one of its events, including the per-phase read timing this - // crate's own `logical.rs` can emit at debug level. Redirecting - // to a discoverable file (mirroring `ephemeral_daemon`'s - // `--log-file` for uffsd) makes that timing data actually - // retrievable for a real-hardware investigation instead of - // silently vanishing. - let job_id_str = uuid::Uuid::from_bytes(job_id).simple().to_string(); - let log_file = std::env::temp_dir().join(format!("uffs-content-reader-{job_id_str}.log")); - let log_file_handle = std::fs::File::create(&log_file) - .with_context(|| format!("failed to create {}", log_file.display()))?; - let mut command = Command::new(&exe); - command - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::from(log_file_handle)); - for (device_path, lease_id, _pool_size) in devices { - command - .arg("--device") - .arg(format!("{device_path}={lease_id}")); - } - tracing::info!( - exe = %exe.display(), - device_count = devices.len(), - log_file = %log_file.display(), - "content reader: spawning uffs-content-reader" - ); - let child = command - .spawn() - .with_context(|| format!("failed to spawn {}", exe.display()))?; - tracing::info!(pid = child.id(), "content reader: process spawned"); - - let mut connections = HashMap::with_capacity(devices.len()); - for (_device_path, lease_id, pool_size) in devices { - let pool = ConnectionPool::connect(*lease_id, *pool_size) - .with_context(|| format!("failed to build connection pool for lease {lease_id}"))?; - tracing::info!( - lease_id, - connections = pool_size, - "content reader: connection pool established" - ); - connections.insert(*lease_id, pool); - } - - Ok(Self { - child, - connections, - job_id, - next_nonce: Arc::new(AtomicU64::new(1)), - }) - } - - /// Begin a session for reading one candidate's entire content, - /// checking out one of `snapshot_lease_id`'s pooled connections and - /// pinning it for the session's whole lifetime โ€” see the module doc - /// comment for why pinning (rather than checking a connection out - /// fresh per chunk) matters. Blocks until a connection is available - /// if every connection in this drive's pool is currently checked - /// out. - /// - /// `known_logical_size` is the candidate's size as the manifest - /// already knows it โ€” forwarded to the Reader so it can skip its own - /// `GetFileSizeEx` re-resolution; see - /// `uffs-content-reader-protocol::ReadRequest::known_logical_size`'s - /// doc comment for the trust reasoning. - /// - /// # Errors - /// Returns an error if `snapshot_lease_id` has no pool, or every - /// connection in that pool has already failed and been dropped. - pub(crate) fn begin_read( - &self, - snapshot_lease_id: u64, - candidate_id: u64, - full_file_reference: u64, - known_logical_size: u64, - ) -> Result { - let pool = self.connections.get(&snapshot_lease_id).ok_or_else(|| { - anyhow::anyhow!( - "no content reader connection pool for snapshot_lease_id {snapshot_lease_id}" - ) - })?; - let pipe = pool.checkout.recv().map_err(|err| { - anyhow::anyhow!( - "connection pool for lease {snapshot_lease_id} is exhausted \ - (every connection failed): {err}" - ) - })?; - Ok(ReaderSession { - pipe: Some(pipe), - checkin: pool.checkin.clone(), - job_id: self.job_id, - snapshot_lease_id, - candidate_id, - full_file_reference, - known_logical_size, - next_nonce: Arc::clone(&self.next_nonce), - }) - } - - /// Tear down this instance: kill the spawned process. The pipe - /// connection is closed when `self` drops. - /// - /// # Errors - /// Returns an error if the process couldn't be killed. - pub(crate) fn shutdown(mut self) -> Result<()> { - self.child - .kill() - .context("failed to kill content reader process")?; - drop(self.child.wait()); - Ok(()) - } -} - -impl Drop for ContentReader { - /// Best-effort safety net: if [`Self::shutdown`] was never called - /// explicitly, don't leak the child process. - fn drop(&mut self) { - drop(self.child.kill()); - } -} - -/// One candidate's whole sequential read session: a pinned pooled -/// connection plus the fields every one of that candidate's requests -/// shares โ€” see [`ContentReader::begin_read`]'s doc comment and the -/// module doc comment for why pinning one connection for the whole -/// session (rather than checking one out fresh per chunk) matters. -/// -/// Checks the connection back in on drop if it's still framing-aligned -/// (`pipe` is `Some`); a connection that failed mid-round-trip is left -/// as `None` and simply not returned, shrinking that drive's pool by -/// one โ€” same contract [`ContentReader`]'s old per-call `round_trip` -/// used to implement. -pub(crate) struct ReaderSession { - /// The pinned connection, or `None` once a round trip on it has - /// failed (see the struct doc comment). - pipe: Option, - /// Returns `pipe` to its pool's checkout queue on drop. - checkin: Sender, - /// This job's id, echoed into every `ReadRequest`. - job_id: [u8; 16], - /// This session's drive, echoed into every `ReadRequest`. - snapshot_lease_id: u64, - /// This session's candidate, echoed into every `ReadRequest`. - candidate_id: u64, - /// This session's file, echoed into every `ReadRequest`. - full_file_reference: u64, - /// This candidate's manifest-known logical size, forwarded as - /// `ReadRequest::known_logical_size` on every request so the Reader - /// can skip its own `GetFileSizeEx` re-resolution on a cache miss โ€” - /// see that field's own doc comment for the trust reasoning. - known_logical_size: u64, - /// Shared with every other live session (see - /// [`ContentReader::next_nonce`]'s own doc comment). - next_nonce: Arc, -} - -impl ReaderSession { - /// Send one framed [`ReadRequest`] for this session's candidate/file - /// and read back one framed [`ReadResponse`], over this session's - /// pinned connection. - fn round_trip( - &mut self, - logical_offset: u64, - maximum_logical_length: u32, - ) -> Result { - let Some(pipe) = self.pipe.as_mut() else { - anyhow::bail!( - "read session for candidate {} already lost its connection to an earlier \ - round-trip failure", - self.candidate_id - ); - }; - let request = ReadRequest { - job_id: self.job_id, - snapshot_lease_id: self.snapshot_lease_id, - candidate_id: self.candidate_id, - // Presently inert on the Reader side โ€” v1's `OpenFileById` - // locates the file by `full_file_reference` alone, no - // volume cross-check. See - // `uffs-content-reader/src/reader/logical.rs`'s module doc. - volume_identity: VolumeIdentity { - volume_serial: 0, - volume_guid: Vec::new(), - }, - full_file_reference: self.full_file_reference, - known_logical_size: Some(self.known_logical_size), - stream_kind: StreamKind::UnnamedData, - logical_offset, - maximum_logical_length, - requested_mode: RequestedReadMode::Logical, - request_nonce: self.next_nonce.fetch_add(1, Ordering::Relaxed), - }; - - let result = (|| -> Result { - write_framed_message(pipe, &request.encode())?; - let response_bytes = read_framed_message(pipe)?; - let mut wire_reader = WireReader::new(&response_bytes); - ReadResponse::decode(&mut wire_reader, MAX_RESPONSE_PAYLOAD_BYTES) - .map_err(|err| anyhow::anyhow!("malformed Reader response: {err}")) - })(); - if result.is_err() { - // Framing state is now unknown โ€” see the struct doc comment - // for why this session's connection must not be reused - // again, by this or any future chunk. - self.pipe = None; - } - result - } -} - -impl ReadSession for ReaderSession { - fn read_at(&mut self, offset: u64, max_len: u32) -> std::io::Result> { - let snapshot_lease_id = self.snapshot_lease_id; - let candidate_id = self.candidate_id; - match self.round_trip(offset, max_len) { - Ok(ReadResponse::Bytes { payload, .. }) => Ok(payload), - Ok(ReadResponse::Error { code, message }) => { - tracing::warn!( - snapshot_lease_id, - candidate_id, - offset, - ?code, - message = %message, - "content reader: read rejected" - ); - Err(std::io::Error::other(format!( - "Reader rejected read: {code:?}: {message}" - ))) - } - Err(err) => { - tracing::warn!( - snapshot_lease_id, - candidate_id, - offset, - error = %err, - "content reader: round trip failed" - ); - Err(std::io::Error::other(err.to_string())) - } - } - } -} - -impl Drop for ReaderSession { - fn drop(&mut self) { - if let Some(pipe) = self.pipe.take() { - // Best-effort, matching the pool's own established contract - // (see the module doc comment): this can't actually - // overflow since the pool never holds more than its - // original connection count, and a failure here just means - // one fewer pooled connection. - drop(self.checkin.try_send(pipe)); - } - } -} - -/// Open [`READER_PIPE_NAME`], retrying briefly while the freshly -/// spawned process finishes binding it. -fn connect_with_retry() -> Result { - let deadline = Instant::now() + CONNECT_RETRY_BUDGET; - let last_err = loop { - match std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(READER_PIPE_NAME) - { - Ok(pipe) => return Ok(pipe), - Err(err) => { - if Instant::now() >= deadline { - break err; - } - std::thread::sleep(CONNECT_RETRY_INTERVAL); - } - } - }; - Err(anyhow::anyhow!( - "could not connect to content reader at {READER_PIPE_NAME}: {last_err}" - )) -} - -/// Find the `uffs-content-reader` executable: prefer a sibling of the -/// current binary, falling back to the platform binary name on `$PATH`. -fn find_reader_exe() -> std::path::PathBuf { - if let Ok(exe) = std::env::current_exe() - && let Some(parent) = exe.parent() - { - let sibling = parent.join("uffs-content-reader.exe"); - if sibling.exists() { - return sibling; - } - } - std::path::PathBuf::from("uffs-content-reader.exe") -} - -/// Write `payload` as `[u32 LE length][payload]`, flushing immediately. -fn write_framed_message(pipe: &mut std::fs::File, payload: &[u8]) -> Result<()> { - let length = u32::try_from(payload.len()) - .map_err(|err| anyhow::anyhow!("request payload too large to frame: {err}"))?; - pipe.write_all(&length.to_le_bytes())?; - pipe.write_all(payload)?; - pipe.flush()?; - Ok(()) -} - -/// Read one `[u32 LE length][payload]`-framed message. -fn read_framed_message(pipe: &mut std::fs::File) -> Result> { - let mut length_bytes = [0_u8; 4]; - pipe.read_exact(&mut length_bytes)?; - let length = u32::from_le_bytes(length_bytes); - anyhow::ensure!( - length <= MAX_RESPONSE_PAYLOAD_BYTES, - "response length {length} exceeds maximum {MAX_RESPONSE_PAYLOAD_BYTES}" - ); - - let mut payload = vec![0_u8; length as usize]; - pipe.read_exact(&mut payload)?; - Ok(payload) -} diff --git a/crates/uffs-content/src/job/registry.rs b/crates/uffs-content/src/job/registry.rs deleted file mode 100644 index 8dbab1cab..000000000 --- a/crates/uffs-content/src/job/registry.rs +++ /dev/null @@ -1,288 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! In-memory, per-process job registry: the resume state a `JOB_RESUME` -//! reconnect consults to skip candidates the consumer already -//! acknowledged, instead of re-streaming the whole job. -//! -//! Deliberately **not** durable. This is scoped to the specific gap -//! between two already-decided positions: -//! -//! - [`crate::run`]'s own doc comment: no durable per-candidate ledger, no -//! crash-recovery reconciliation โ€” a producer-*process* crash means a fresh -//! job attempt with a fresh VSS snapshot, relying on the consumer's own -//! content-hash dedup to make re-streaming already- ingested content a no-op. -//! - A live producer process losing its *connection* to the consumer (a -//! transport blip, the consumer process restarting) is a much more common -//! event than a producer crash, and re-streaming everything already streamed -//! and acknowledged before the blip (potentially most of a large job) is -//! real, avoidable waste โ€” not a correctness requirement, since the -//! consumer's dedup would absorb it either way. -//! -//! So: while the producer *process* is alive, it keeps this registry in -//! memory; a reconnecting consumer names the `job_id` it wants to -//! resume, and the registry reports which candidates still need -//! streaming. If the producer process itself has died, the registry -//! (and the job with it) is gone โ€” that falls through to the existing, -//! already-decided "start a fresh job" path, unchanged. - -use std::collections::{HashMap, HashSet}; -use std::sync::Mutex; - -/// One job's resume-relevant state: which candidate ids exist, and -/// which of them the consumer has already acknowledged. -#[cfg_attr( - not(any(windows, test)), - expect( - dead_code, - reason = "only constructed by the Windows-only `serve` module's streaming \ - task in production; exercised cross-platform by this module's own \ - unit tests, which is why the type still lives here rather than \ - behind `#[cfg(windows)]`" - ) -)] -struct ActiveJob { - /// Every candidate id this job's manifest assigned, in enumeration - /// order. - candidate_ids: Vec, - /// Candidate ids the consumer has sent `FILE_ACK` for. - acked: HashSet, -} - -/// Registry of jobs the current producer process is actively serving. -/// -/// Cheap to hold for a job's whole lifetime: `acked` is a `HashSet`, -/// a few bytes per candidate โ€” negligible even for a job with hundreds -/// of thousands of candidates, and nothing here is written to disk. -pub(crate) struct JobRegistry { - /// Active jobs keyed by `job_id`. - jobs: Mutex>, -} - -impl JobRegistry { - /// An empty registry. - pub(crate) fn new() -> Self { - Self { - jobs: Mutex::new(HashMap::new()), - } - } - - /// Register a freshly started job with its full candidate id list. - /// Replaces any prior registration under the same `job_id` (there - /// shouldn't be one โ€” job ids are fresh UUIDs per job โ€” but a - /// pathological duplicate submission overwrites rather than panics). - #[cfg_attr( - not(any(windows, test)), - expect(dead_code, reason = "see the `ActiveJob` doc comment above") - )] - pub(crate) fn register(&self, job_id: [u8; 16], candidate_ids: Vec) { - let mut jobs = self - .jobs - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - jobs.insert(job_id, ActiveJob { - candidate_ids, - acked: HashSet::new(), - }); - } - - /// Record a `FILE_ACK` for `candidate_id` under `job_id`. - /// - /// Returns `true` if `job_id` is a known, still-registered job - /// (regardless of whether `candidate_id` was already acked โ€” acking - /// twice is a harmless no-op, matching the wire protocol's own - /// idempotency contract). - #[cfg_attr( - not(any(windows, test)), - expect(dead_code, reason = "see the `ActiveJob` doc comment above") - )] - pub(crate) fn ack(&self, job_id: [u8; 16], candidate_id: u64) -> bool { - let mut jobs = self - .jobs - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let Some(job) = jobs.get_mut(&job_id) else { - return false; - }; - job.acked.insert(candidate_id); - drop(jobs); - true - } - - /// Candidate ids for `job_id` that have **not** yet been acked, in - /// their original enumeration order โ€” what a fresh connection or a - /// `JOB_RESUME` reconnect should stream. `None` if `job_id` isn't a - /// currently-registered job (producer restarted, job finished and - /// was removed, or it was never this producer's job). - #[cfg_attr( - not(any(windows, test)), - expect(dead_code, reason = "see the `ActiveJob` doc comment above") - )] - pub(crate) fn pending(&self, job_id: [u8; 16]) -> Option> { - let jobs = self - .jobs - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let job = jobs.get(&job_id)?; - let pending_ids: Vec = job - .candidate_ids - .iter() - .copied() - .filter(|id| !job.acked.contains(id)) - .collect(); - drop(jobs); - Some(pending_ids) - } - - /// Whether every candidate registered for `job_id` has been acked. - /// `None` if `job_id` isn't currently registered. - #[cfg_attr( - not(any(windows, test)), - expect(dead_code, reason = "see the `ActiveJob` doc comment above") - )] - pub(crate) fn is_complete(&self, job_id: [u8; 16]) -> Option { - let jobs = self - .jobs - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let job = jobs.get(&job_id)?; - let complete = job.candidate_ids.iter().all(|id| job.acked.contains(id)); - drop(jobs); - Some(complete) - } - - /// Drop `job_id`'s state โ€” once a job is fully acked (or explicitly - /// cancelled), there is nothing left to resume. - #[cfg_attr( - not(any(windows, test)), - expect(dead_code, reason = "see the `ActiveJob` doc comment above") - )] - pub(crate) fn remove(&self, job_id: [u8; 16]) { - let mut jobs = self - .jobs - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - jobs.remove(&job_id); - } - - /// Whether `job_id` is currently registered (alive in this - /// producer process). - #[cfg_attr( - not(test), - expect( - dead_code, - reason = "exercised by this module's own unit tests only; no production \ - call site needs it yet (JOB_RESUME handling keys off \ - `ServerState::active` instead) โ€” kept because it is the natural \ - complement to `pending`/`is_complete` and cheap to maintain" - ) - )] - pub(crate) fn contains(&self, job_id: [u8; 16]) -> bool { - let jobs = self - .jobs - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - jobs.contains_key(&job_id) - } -} - -impl Default for JobRegistry { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::JobRegistry; - - const JOB_A: [u8; 16] = [1; 16]; - const JOB_B: [u8; 16] = [2; 16]; - - #[test] - fn unregistered_job_reports_no_pending_and_not_complete() { - let registry = JobRegistry::new(); - assert_eq!(registry.pending(JOB_A), None); - assert_eq!(registry.is_complete(JOB_A), None); - assert!(!registry.contains(JOB_A)); - assert!(!registry.ack(JOB_A, 1)); - } - - #[test] - fn fresh_registration_has_every_candidate_pending() { - let registry = JobRegistry::new(); - registry.register(JOB_A, vec![1, 2, 3]); - assert!(registry.contains(JOB_A)); - assert_eq!(registry.pending(JOB_A), Some(vec![1, 2, 3])); - assert_eq!(registry.is_complete(JOB_A), Some(false)); - } - - #[test] - fn acking_a_candidate_removes_it_from_pending() { - let registry = JobRegistry::new(); - registry.register(JOB_A, vec![1, 2, 3]); - assert!(registry.ack(JOB_A, 2)); - assert_eq!(registry.pending(JOB_A), Some(vec![1, 3])); - assert_eq!(registry.is_complete(JOB_A), Some(false)); - } - - #[test] - fn acking_every_candidate_marks_the_job_complete() { - let registry = JobRegistry::new(); - registry.register(JOB_A, vec![1, 2]); - assert!(registry.ack(JOB_A, 1)); - assert!(registry.ack(JOB_A, 2)); - assert_eq!(registry.pending(JOB_A), Some(vec![])); - assert_eq!(registry.is_complete(JOB_A), Some(true)); - } - - #[test] - fn acking_the_same_candidate_twice_is_a_harmless_no_op() { - let registry = JobRegistry::new(); - registry.register(JOB_A, vec![1, 2]); - assert!(registry.ack(JOB_A, 1)); - assert!(registry.ack(JOB_A, 1)); - assert_eq!(registry.pending(JOB_A), Some(vec![2])); - } - - #[test] - fn acking_an_unknown_candidate_id_is_recorded_but_never_appears_pending() { - // Defends against a malicious/buggy consumer acking an id that - // was never in the manifest: it's silently absorbed (the ack - // just never removes anything from `pending`, since `pending` - // is built from `candidate_ids`, not from `acked`), never - // fabricates a phantom pending entry. - let registry = JobRegistry::new(); - registry.register(JOB_A, vec![1, 2]); - assert!(registry.ack(JOB_A, 999)); - assert_eq!(registry.pending(JOB_A), Some(vec![1, 2])); - } - - #[test] - fn jobs_are_independent() { - let registry = JobRegistry::new(); - registry.register(JOB_A, vec![1, 2]); - registry.register(JOB_B, vec![10, 20]); - assert!(registry.ack(JOB_A, 1)); - assert_eq!(registry.pending(JOB_A), Some(vec![2])); - assert_eq!(registry.pending(JOB_B), Some(vec![10, 20])); - } - - #[test] - fn removing_a_job_drops_its_resume_state() { - let registry = JobRegistry::new(); - registry.register(JOB_A, vec![1, 2]); - registry.remove(JOB_A); - assert!(!registry.contains(JOB_A)); - assert_eq!(registry.pending(JOB_A), None); - } - - #[test] - fn re_registering_the_same_job_id_replaces_prior_state() { - let registry = JobRegistry::new(); - registry.register(JOB_A, vec![1, 2]); - assert!(registry.ack(JOB_A, 1)); - registry.register(JOB_A, vec![5, 6, 7]); - assert_eq!(registry.pending(JOB_A), Some(vec![5, 6, 7])); - } -} diff --git a/crates/uffs-content/src/job/self_test.rs b/crates/uffs-content/src/job/self_test.rs deleted file mode 100644 index f603b38d6..000000000 --- a/crates/uffs-content/src/job/self_test.rs +++ /dev/null @@ -1,548 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Elevated smoke test: real VSS snapshot + real privileged Reader, -//! creating a unique sample file and proving playback through -//! [`super::vss_job::run_vss_job`] reproduces its content exactly. -//! -//! Mirrors `uffs-broker`'s own `--self-test-vss` design -//! (`crates/uffs-broker/src/broker.rs`/`broker/snapshot_manager/ -//! vss_self_test.rs`): the round-trip logic lives once, here, in -//! production code โ€” reused by both the `--self-test-vss-playback` CLI -//! flag (`main.rs`) and `cargo test -p uffs-content -- --ignored` -//! (`tests/e2e_real_vss_content_reader.rs`), so none of the three ever -//! drift apart. - -use std::path::Path; - -use anyhow::{Context as _, Result}; -use uffs_content_protocol::codec::Reader as WireReader; -use uffs_content_protocol::frame::{ContentChunk, FileEnd, FrameEnvelope, FrameType}; -use uffs_content_protocol::manifest::{CandidateRecord, ManifestHeader}; - -use super::intake::JobRequest; -use super::vss_job::run_vss_job; - -/// Run the real create-snapshot -> select-target -> read-content round trip. -/// -/// Uses a freshly created, uniquely-named sample file under `test_dir`, -/// and verifies the streamed bytes exactly match what was written. -/// -/// # Errors -/// Returns an error if the sample file can't be created, `run_vss_job` -/// fails, the job doesn't find exactly the one sample file, or the -/// played-back content doesn't match what was written. -pub fn self_test_vss_playback(test_dir: &Path) -> Result<()> { - std::fs::create_dir_all(test_dir) - .with_context(|| format!("failed to create test dir {}", test_dir.display()))?; - - let unique_name = format!( - "uffs-content-self-test-{}.txt", - uuid::Uuid::new_v4().simple() - ); - let content = - b"UFFS content-reader self-test: real VSS snapshot + real Reader playback.\n".as_slice(); - let sample_path = test_dir.join(&unique_name); - std::fs::write(&sample_path, content) - .with_context(|| format!("failed to write sample file {}", sample_path.display()))?; - - let run_dir = test_dir.join("run"); - std::fs::create_dir_all(&run_dir) - .with_context(|| format!("failed to create run dir {}", run_dir.display()))?; - - let request = JobRequest { - source_id: "uffs-content-self-test".to_owned(), - roots: vec![test_dir.to_path_buf()], - query: unique_name, - ..Default::default() - }; - - let mut frames = Vec::new(); - let outcome = run_vss_job(&request, &run_dir, |frame| { - frames.push(frame); - Ok(()) - }) - .context("run_vss_job failed")?; - - anyhow::ensure!( - outcome.run_summary.candidate_count == 1, - "expected exactly 1 candidate (the unique sample file), found {}", - outcome.run_summary.candidate_count - ); - anyhow::ensure!( - outcome.run_summary.succeeded_count == 1, - "expected the sample file to succeed, got {} succeeded / {} failed-retryable / {} \ - failed-terminal / {} deferred", - outcome.run_summary.succeeded_count, - outcome.run_summary.failed_retryable_count, - outcome.run_summary.failed_terminal_count, - outcome.run_summary.deferred_manual_count - ); - - let played_back = decode_single_file_content(&outcome.manifest_bytes, &frames) - .context("failed to decode the job's own manifest/frame output")?; - anyhow::ensure!( - played_back == content, - "playback content does not match the original sample file (got {} bytes, expected {})", - played_back.len(), - content.len() - ); - - Ok(()) -} - -/// Run a real, extension-filtered query against an existing directory and -/// verify the pipeline's reported metadata/content totals against ground -/// truth. -/// -/// Runs against a real drive with real files already on it โ€” not a -/// synthetic sample. Unlike [`self_test_vss_playback`] (one synthetic -/// file, content-only), -/// this validates the pipeline against however many real files of -/// `extension` already exist under `root`: every candidate must succeed, -/// the candidate count must match the ground-truth walk's file count, the -/// manifest's own `logical_size` fields must sum to the ground-truth -/// total, and the bytes actually streamed over `CONTENT_CHUNK` frames -/// must also sum to that same total. Ground truth comes from -/// `walk_tolerating_denied` โ€” a permissive `std::fs` walker reading the -/// **live** volume rather than the job's VSS snapshot; on a quiescent -/// drive the two are expected to match exactly. -/// -/// # Errors -/// Returns an error if the ground-truth walk finds no matching files, -/// `run_vss_job` fails, any candidate doesn't succeed, or any of the -/// three totals (candidate count, manifest metadata bytes, streamed -/// content bytes) disagrees with ground truth. -pub fn self_test_vss_query_metadata(root: &Path, extension: &str) -> Result<()> { - let (ground_truth_count, ground_truth_bytes, skipped_dirs, ground_truth_paths) = - ground_truth_extension_totals(root, extension); - if !skipped_dirs.is_empty() { - tracing::warn!( - skipped_count = skipped_dirs.len(), - skipped = ?skipped_dirs, - "ground-truth walk skipped {} inaccessible director{} (e.g. OS-reserved \ - folders) โ€” the real MFT-based query engine reads these regardless, so a \ - mismatch caused by this is a ground-truth walker limitation, not a pipeline bug", - skipped_dirs.len(), - if skipped_dirs.len() == 1 { "y" } else { "ies" } - ); - } - anyhow::ensure!( - ground_truth_count > 0, - "no *.{extension} files found under {} โ€” nothing to validate", - root.display() - ); - - let run_dir = std::env::temp_dir().join(format!( - "uffs-content-query-metadata-{}", - uuid::Uuid::new_v4().simple() - )); - std::fs::create_dir_all(&run_dir) - .with_context(|| format!("failed to create run dir {}", run_dir.display()))?; - - let request = JobRequest { - source_id: "uffs-content-self-test-query".to_owned(), - roots: vec![root.to_path_buf()], - query: "*".to_owned(), - ext: Some(extension.to_owned()), - ..Default::default() - }; - - let mut frames = Vec::new(); - let outcome = run_vss_job(&request, &run_dir, |frame| { - frames.push(frame); - Ok(()) - }) - .context("run_vss_job failed")?; - - if outcome.run_summary.candidate_count != ground_truth_count { - let pipeline_paths = decode_candidate_paths(&outcome.manifest_bytes, root) - .context("failed to decode candidate paths for mismatch diagnostics")?; - anyhow::bail!( - "candidate count mismatch: pipeline found {}, ground-truth disk walk found {}\n\ - (path, pipeline_count, ground_truth_count) for every differing path:\n{:#?}", - outcome.run_summary.candidate_count, - ground_truth_count, - count_mismatches(&pipeline_paths, &ground_truth_paths), - ); - } - anyhow::ensure!( - outcome.run_summary.succeeded_count == outcome.run_summary.candidate_count, - "not every candidate succeeded: {} of {} (failed-retryable={}, failed-terminal={}, \ - deferred={})", - outcome.run_summary.succeeded_count, - outcome.run_summary.candidate_count, - outcome.run_summary.failed_retryable_count, - outcome.run_summary.failed_terminal_count, - outcome.run_summary.deferred_manual_count - ); - - let summary = summarize_query_outcome(&outcome.manifest_bytes, &frames) - .context("failed to decode the job's own manifest/frame output")?; - anyhow::ensure!( - summary.metadata_total_bytes == ground_truth_bytes, - "manifest metadata size total mismatch: pipeline reported {} bytes, ground-truth {} bytes", - summary.metadata_total_bytes, - ground_truth_bytes - ); - anyhow::ensure!( - summary.content_total_bytes == ground_truth_bytes, - "streamed content byte total mismatch: pipeline streamed {} bytes, ground-truth {} bytes", - summary.content_total_bytes, - ground_truth_bytes - ); - - Ok(()) -} - -/// One [`self_test_reader_benchmark`] run's measured results. -#[derive(Debug, Clone, Copy)] -pub struct ReaderBenchmarkReport { - /// Total candidates the manifest committed to. - pub candidate_count: u64, - /// Candidates that reached a successful terminal outcome. - pub succeeded_count: u64, - /// Sum of every `CONTENT_CHUNK.payload.len()` actually streamed. - pub content_bytes: u64, - /// Wall-clock time from job start to the first `CONTENT_CHUNK` frame: - /// VSS lease + ephemeral daemon spawn + enumeration + manifest - /// finalization, milliseconds. - pub enumeration_ms: u128, - /// Wall-clock time from the first `CONTENT_CHUNK` frame to the job - /// finishing โ€” the number this benchmark exists to measure, - /// milliseconds. - pub content_read_ms: u128, - /// `content_bytes` / `content_read_ms`, in MiB/s. `0.0` if - /// `content_read_ms` is `0` (nothing to divide by โ€” e.g. a job with - /// no content-bearing candidates). - pub throughput_mib_per_sec: f64, -} - -/// Run a real VSS-backed job against `roots` (empty = every local NTFS -/// drive โ€” see [`super::vss_job::run_vss_job`]) evaluating `query`, and -/// report content-read wall-clock time and throughput. -/// -/// This is the baseline-measurement tool for judging Reader-parallelism -/// work (see the local-only content-engine architecture doc): it -/// deliberately isolates the *content-read phase* from VSS-lease/ -/// ephemeral-daemon/enumeration overhead by using `emit_frame` itself as -/// the observation point โ€” the moment the first `CONTENT_CHUNK` frame -/// arrives marks the enumeration/content-read phase boundary โ€” rather -/// than adding timing instrumentation to `run_job`/`workflow` itself. -/// -/// # Errors -/// Returns an error if the run directory can't be created or -/// `run_vss_job` fails. -pub fn self_test_reader_benchmark( - roots: &[std::path::PathBuf], - query: &str, -) -> Result { - let run_dir = std::env::temp_dir().join(format!( - "uffs-content-reader-benchmark-{}", - uuid::Uuid::new_v4().simple() - )); - std::fs::create_dir_all(&run_dir) - .with_context(|| format!("failed to create run dir {}", run_dir.display()))?; - - let request = JobRequest { - source_id: "uffs-content-reader-benchmark".to_owned(), - roots: roots.to_vec(), - query: query.to_owned(), - ..Default::default() - }; - - let start = std::time::Instant::now(); - let mut first_content_chunk_at: Option = None; - let mut content_bytes: u64 = 0; - - let outcome = run_vss_job(&request, &run_dir, |frame_bytes| { - let mut reader = WireReader::new(&frame_bytes); - if let Ok((envelope, payload)) = FrameEnvelope::decode(&mut reader, u64::MAX) - && envelope.frame_type == FrameType::ContentChunk - { - first_content_chunk_at.get_or_insert_with(std::time::Instant::now); - let mut payload_reader = WireReader::new(&payload); - if let Ok(chunk) = ContentChunk::decode(&mut payload_reader, u32::MAX) { - content_bytes += u64::try_from(chunk.payload.len()).unwrap_or(u64::MAX); - } - } - Ok(()) - }) - .context("run_vss_job failed")?; - - let end = std::time::Instant::now(); - let content_start = first_content_chunk_at.unwrap_or(end); - let enumeration_ms = content_start.duration_since(start).as_millis(); - let content_read_ms = end.duration_since(content_start).as_millis(); - #[expect( - clippy::cast_precision_loss, - reason = "diagnostic-only throughput number for a benchmark report, not a value \ - anything downstream computes against โ€” losing precision above 2^53 bytes \ - (8+ petabytes) or milliseconds is not a real concern here" - )] - #[expect( - clippy::float_arithmetic, - reason = "diagnostic-only throughput ratio for a benchmark report โ€” same precision \ - posture as uffs-daemon's own EMA rate arithmetic (drive_stats.rs)" - )] - let throughput_mib_per_sec = if content_read_ms > 0 { - (content_bytes as f64 / (1_024.0_f64 * 1_024.0_f64)) - / (content_read_ms as f64 / 1_000.0_f64) - } else { - 0.0_f64 - }; - - Ok(ReaderBenchmarkReport { - candidate_count: outcome.run_summary.candidate_count, - succeeded_count: outcome.run_summary.succeeded_count, - content_bytes, - enumeration_ms, - content_read_ms, - throughput_mib_per_sec, - }) -} - -/// Independent ground truth for [`self_test_vss_query_metadata`]: walk -/// `root` live via `std::fs` (bypassing VSS/the daemon entirely) and sum -/// the size of every regular file whose extension case-insensitively -/// matches `extension`. -/// -/// Deliberately **not** [`super::candidate_source::DirWalkCandidateSource`] -/// (used elsewhere in this crate for synthetic test fixtures, where an -/// access-denied error is itself a bug worth failing loud on): a real, -/// pre-existing drive -/// routinely has OS-reserved, ACL-locked directories (`System Volume -/// Information`, `$RECYCLE.BIN`) that plain `std::fs::read_dir` can't -/// enter but that the real MFT-based query engine reads regardless (it -/// never goes through filesystem permission checks). This walker treats -/// a directory it can't enter as "skip, not fail" and reports how many -/// were skipped, so a real discrepancy is still visible rather than -/// silently swallowed. -/// -/// Returns `(matching_file_count, total_logical_bytes, skipped_dirs, -/// matching_paths)`. -fn ground_truth_extension_totals( - root: &Path, - extension: &str, -) -> (u64, u64, Vec, Vec) { - let mut count: u64 = 0; - let mut total_bytes: u64 = 0; - let mut skipped_dirs = Vec::new(); - let mut matching_paths = Vec::new(); - walk_tolerating_denied( - root, - extension, - &mut count, - &mut total_bytes, - &mut skipped_dirs, - &mut matching_paths, - ); - (count, total_bytes, skipped_dirs, matching_paths) -} - -/// Recursive worker for [`ground_truth_extension_totals`]. A directory -/// that can't be listed (permission denied, or any other `read_dir` -/// error) is appended to `skipped_dirs` and skipped, rather than -/// propagated โ€” see that function's doc comment for why. -fn walk_tolerating_denied( - dir: &Path, - extension: &str, - count: &mut u64, - total_bytes: &mut u64, - skipped_dirs: &mut Vec, - matching_paths: &mut Vec, -) { - let Ok(read_dir) = std::fs::read_dir(dir) else { - skipped_dirs.push(dir.to_path_buf()); - return; - }; - for entry in read_dir.flatten() { - let path = entry.path(); - let Ok(metadata) = entry.metadata() else { - continue; - }; - if metadata.is_dir() { - walk_tolerating_denied( - &path, - extension, - count, - total_bytes, - skipped_dirs, - matching_paths, - ); - } else if metadata.is_file() { - let matches = path - .extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| ext.eq_ignore_ascii_case(extension)); - if matches { - *count += 1; - *total_bytes += metadata.len(); - matching_paths.push(path); - } - } - } -} - -/// Decode every `CandidateRecord::path` out of a manifest, re-joined onto -/// `root` for the candidate-count-mismatch diagnostic in -/// [`self_test_vss_query_metadata`]. -/// -/// `CandidateRecord::path` is root-relative by design (see -/// `CandidateEntry::relative_path`'s doc comment), while the ground-truth -/// walker's paths are absolute โ€” rejoining here puts both sides in the -/// same representation so the diff isn't swamped by a spurious -/// "every path differs" noise from the root prefix alone. -fn decode_candidate_paths(manifest_bytes: &[u8], root: &Path) -> Result> { - let mut manifest_reader = WireReader::new(manifest_bytes); - let header = ManifestHeader::decode(&mut manifest_reader) - .map_err(|err| anyhow::anyhow!("decode manifest header: {err}"))?; - let mut paths = Vec::with_capacity(usize::try_from(header.candidate_count).unwrap_or(0)); - for _ in 0..header.candidate_count { - let record = CandidateRecord::decode(&mut manifest_reader) - .map_err(|err| anyhow::anyhow!("decode candidate record: {err}"))?; - paths.push(root.join(record.path.display_lossy())); - } - Ok(paths) -} - -/// For every path whose occurrence count differs between `left` and -/// `right`, `(path, left_count, right_count)` โ€” for the candidate-count- -/// mismatch diagnostic in [`self_test_vss_query_metadata`]. Counts a path -/// appearing twice in one side but once in the other (a literal duplicate -/// row), not just paths missing entirely from one side, since that's -/// exactly the shape a merge/dedup bug would produce. -fn count_mismatches( - left: &[std::path::PathBuf], - right: &[std::path::PathBuf], -) -> Vec<(std::path::PathBuf, usize, usize)> { - let mut counts: alloc::collections::BTreeMap<&Path, (usize, usize)> = - alloc::collections::BTreeMap::new(); - for path in left { - counts.entry(path.as_path()).or_default().0 += 1; - } - for path in right { - counts.entry(path.as_path()).or_default().1 += 1; - } - counts - .into_iter() - .filter(|(_, (left_count, right_count))| left_count != right_count) - .map(|(path, (left_count, right_count))| (path.to_path_buf(), left_count, right_count)) - .collect() -} - -/// Aggregate totals decoded from a job's own manifest + frame output, for -/// [`self_test_vss_query_metadata`]. -struct QueryOutcomeSummary { - /// Sum of every `CandidateRecord::logical_size` in the manifest. - metadata_total_bytes: u64, - /// Sum of every `CONTENT_CHUNK.payload.len()` actually streamed. - content_total_bytes: u64, -} - -/// Decode a manifest describing `header.candidate_count` candidates plus -/// their frame stream, returning both the manifest's own metadata-size -/// total and the total bytes actually streamed over `CONTENT_CHUNK` -/// frames โ€” the two independent numbers [`self_test_vss_query_metadata`] -/// cross-checks against ground truth. -/// -/// Deliberately duplicated from [`decode_single_file_content`] rather than -/// generalizing that one: this decoder sums across an arbitrary number of -/// candidates and never buffers content bytes, while that one is scoped -/// to exactly one candidate and returns its buffered content โ€” different -/// enough shapes that a shared abstraction would obscure both. -fn summarize_query_outcome( - manifest_bytes: &[u8], - frames: &[Vec], -) -> Result { - let mut manifest_reader = WireReader::new(manifest_bytes); - let header = ManifestHeader::decode(&mut manifest_reader) - .map_err(|err| anyhow::anyhow!("decode manifest header: {err}"))?; - - let mut metadata_total_bytes: u64 = 0; - for _ in 0..header.candidate_count { - let record = CandidateRecord::decode(&mut manifest_reader) - .map_err(|err| anyhow::anyhow!("decode candidate record: {err}"))?; - metadata_total_bytes += record.logical_size; - } - - let mut content_total_bytes: u64 = 0; - for frame_bytes in frames { - let mut frame_reader = WireReader::new(frame_bytes); - let (envelope, payload) = FrameEnvelope::decode(&mut frame_reader, u64::MAX) - .map_err(|err| anyhow::anyhow!("decode frame envelope: {err}"))?; - if envelope.frame_type != FrameType::ContentChunk { - continue; - } - let mut payload_reader = WireReader::new(&payload); - let chunk = ContentChunk::decode(&mut payload_reader, u32::MAX) - .map_err(|err| anyhow::anyhow!("decode CONTENT_CHUNK: {err}"))?; - content_total_bytes += chunk.payload.len() as u64; - } - - Ok(QueryOutcomeSummary { - metadata_total_bytes, - content_total_bytes, - }) -} - -/// Decode a manifest + frame stream that is known to describe exactly -/// one candidate, returning the bytes its `CONTENT_CHUNK` frames -/// carried. -/// -/// A narrow, self-test-only decoder โ€” see -/// `tests/support/test_consumer.rs` for the fuller, general-purpose -/// version the parity harness uses; duplicated here (not shared) -/// because this one is production code (compiled into the shipped -/// binary), matching `uffs-content-reader-protocol`'s own "small, -/// independent duplicate" precedent for the same reason. -fn decode_single_file_content(manifest_bytes: &[u8], frames: &[Vec]) -> Result> { - let mut manifest_reader = WireReader::new(manifest_bytes); - let header = ManifestHeader::decode(&mut manifest_reader) - .map_err(|err| anyhow::anyhow!("decode manifest header: {err}"))?; - anyhow::ensure!( - header.candidate_count == 1, - "expected exactly 1 candidate in the manifest, found {}", - header.candidate_count - ); - - let mut buffered = Vec::new(); - let mut saw_file_end = false; - for frame_bytes in frames { - let mut frame_reader = WireReader::new(frame_bytes); - let (envelope, payload) = FrameEnvelope::decode(&mut frame_reader, u64::MAX) - .map_err(|err| anyhow::anyhow!("decode frame envelope: {err}"))?; - let mut payload_reader = WireReader::new(&payload); - match envelope.frame_type { - FrameType::ContentChunk => { - let chunk = ContentChunk::decode(&mut payload_reader, u32::MAX) - .map_err(|err| anyhow::anyhow!("decode CONTENT_CHUNK: {err}"))?; - buffered.extend_from_slice(&chunk.payload); - } - FrameType::FileEnd => { - FileEnd::decode(&mut payload_reader) - .map_err(|err| anyhow::anyhow!("decode FILE_END: {err}"))?; - saw_file_end = true; - } - FrameType::FileFailed | FrameType::FileDeferred => { - anyhow::bail!("candidate did not succeed (saw {:?})", envelope.frame_type); - } - FrameType::JobBegin - | FrameType::FileBegin - | FrameType::FileAck - | FrameType::Progress - | FrameType::Heartbeat - | FrameType::JobEnd - | FrameType::JobCancel - | FrameType::WindowUpdate - | FrameType::JobResume - | FrameType::JobSubmit => {} - } - } - anyhow::ensure!( - saw_file_end, - "never saw a FILE_END frame for the sample file" - ); - - Ok(buffered) -} diff --git a/crates/uffs-content/src/job/snapshot_client.rs b/crates/uffs-content/src/job/snapshot_client.rs deleted file mode 100644 index cb2ac30f5..000000000 --- a/crates/uffs-content/src/job/snapshot_client.rs +++ /dev/null @@ -1,219 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Coordinator-side client for the Broker's Snapshot Manager pipe -//! (`uffs_broker_protocol::snapshot_manager::SNAPSHOT_PIPE_NAME`). -//! -//! Mirrors `uffs-daemon::broker_client`'s connect style (plain -//! `std::fs::OpenOptions` + `Read`/`Write`, no raw Win32 FFI, no -//! pipe-existence probe before the real request โ€” see that module's -//! doc comment for why a probe would starve the real request). The -//! wire shape here is different, though: the MFT-handle protocol is a -//! fixed-length exchange, but this one is a variable-length -//! `[u32 LE length][payload]`-framed request/response, one per -//! connection โ€” this module implements the client side of exactly the -//! framing `uffs-broker`'s `read_framed_message`/`write_framed_message` -//! implement server-side (`crates/uffs-broker/src/broker/ -//! snapshot_manager/mod.rs`). -//! -//! The Broker only replies at all if this process's own image passes -//! its Coordinator identity check (`uffs-content*.exe` + Authenticode โ€” -//! see `verify_coordinator_identity` in the Broker module above): a -//! connection from any other binary gets no response and the Broker -//! closes the pipe. - -use std::io::{Read as _, Write as _}; - -use anyhow::Context as _; -use uffs_broker_protocol::snapshot_manager::{ - CreateSnapshotLease, CreateSnapshotLeaseResult, ReleaseSnapshotLease, SNAPSHOT_PIPE_NAME, - SnapshotManagerErrorCode, SnapshotManagerRequest, SnapshotManagerResponse, VolumeIdentity, -}; - -/// A structured `Create` rejection from the Broker, as opposed to a -/// transport-level failure (pipe unreachable, malformed response, โ€ฆ). -/// -/// Kept separate from a plain `anyhow::bail!` string so callers (see -/// [`super::vss_orchestrator::prepare_ephemeral_daemon_for_roots`]) can -/// `downcast_ref` and branch on `code`/`hresult` โ€” e.g. skipping a -/// drive VSS permanently refuses (`VSS_E_VOLUME_NOT_SUPPORTED` for -/// removable media) instead of string-matching `message`. -#[derive(Debug)] -pub(crate) struct BrokerRejectedCreate { - /// Stable error code the Broker reported. - pub(crate) code: SnapshotManagerErrorCode, - /// The underlying `HRESULT`, when the Broker's failure came from a - /// VSS call and one was available. - pub(crate) hresult: Option, - /// Human-readable diagnostic message. - pub(crate) message: String, -} - -impl core::fmt::Display for BrokerRejectedCreate { - #[expect( - clippy::use_debug, - reason = "SnapshotManagerErrorCode has no Display impl (it's a wire enum, not \ - user-facing text) โ€” Debug is the only formatting available, and this \ - is itself a diagnostic-only error message" - )] - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!( - f, - "Broker rejected Create: {:?}: {}", - self.code, self.message - ) - } -} - -impl core::error::Error for BrokerRejectedCreate {} - -/// Matches the Broker's own `MAX_REQUEST_BYTES` โ€” a response this large -/// would indicate a protocol desync, not a legitimate reply. -const MAX_RESPONSE_BYTES: u32 = 64 * 1024; - -/// A live snapshot lease this process holds โ€” the Coordinator-side -/// counterpart of the Broker's `CreateSnapshotLeaseResult`. -#[expect( - clippy::struct_field_names, - reason = "field names deliberately mirror CreateSnapshotLeaseResult's own \ - wire field names for clarity when converting between the two" -)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct SnapshotLease { - /// Lease identifier, used in every subsequent call for this lease. - pub(crate) snapshot_lease_id: u64, - /// Opaque VSS snapshot identifier. - pub(crate) snapshot_id: Vec, - /// Device path the snapshot is reachable at (e.g. - /// `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopyN`). - pub(crate) snapshot_device_identity: String, - /// Snapshot creation time, Unix milliseconds. - pub(crate) snapshot_created_at_unix_ms: i64, - /// Lease expiry, Unix milliseconds. - pub(crate) expires_at_unix_ms: i64, -} - -/// Request a new snapshot lease from the Broker. -/// -/// # Errors -/// Returns an error if the pipe can't be opened (e.g. the Broker isn't -/// running, or this process fails its identity check), the request -/// can't be sent, the response can't be read/decoded, or the Broker -/// reports failure (`SnapshotManagerResponse::Error`). -pub(crate) fn create_lease( - authenticated_job_id: [u8; 16], - source_volume_identity: VolumeIdentity, - requested_root: Vec, - maximum_lifetime_secs: u64, - policy_id: u32, -) -> anyhow::Result { - let request = SnapshotManagerRequest::Create(CreateSnapshotLease { - authenticated_job_id, - source_volume_identity, - requested_root, - maximum_lifetime_secs, - policy_id, - }); - match round_trip(&request)? { - SnapshotManagerResponse::Created(CreateSnapshotLeaseResult { - snapshot_lease_id, - snapshot_id, - snapshot_device_identity, - snapshot_created_at_unix_ms, - expires_at_unix_ms, - }) => Ok(SnapshotLease { - snapshot_lease_id, - snapshot_id, - snapshot_device_identity, - snapshot_created_at_unix_ms, - expires_at_unix_ms, - }), - SnapshotManagerResponse::Error { - code, - hresult, - message, - } => Err(BrokerRejectedCreate { - code, - hresult, - message, - } - .into()), - other @ (SnapshotManagerResponse::Duplicated - | SnapshotManagerResponse::Renewed { .. } - | SnapshotManagerResponse::Released - | SnapshotManagerResponse::Status(_)) => { - anyhow::bail!("unexpected response to Create: {other:?}") - } - } -} - -/// Release a previously created lease. -/// -/// # Errors -/// Returns an error if the pipe round trip fails or the Broker reports -/// failure. -pub(crate) fn release_lease(snapshot_lease_id: u64) -> anyhow::Result<()> { - let request = SnapshotManagerRequest::Release(ReleaseSnapshotLease { snapshot_lease_id }); - match round_trip(&request)? { - SnapshotManagerResponse::Released => Ok(()), - SnapshotManagerResponse::Error { code, message, .. } => { - anyhow::bail!("Broker rejected Release: {code:?}: {message}") - } - other @ (SnapshotManagerResponse::Created(_) - | SnapshotManagerResponse::Duplicated - | SnapshotManagerResponse::Renewed { .. } - | SnapshotManagerResponse::Status(_)) => { - anyhow::bail!("unexpected response to Release: {other:?}") - } - } -} - -/// Open the Snapshot Manager pipe, send one framed request, and read -/// back one framed response. -/// -/// # Errors -/// Returns an error if the pipe can't be opened, the write/read fails, -/// the response exceeds [`MAX_RESPONSE_BYTES`], or the payload doesn't -/// decode as a valid [`SnapshotManagerResponse`]. -fn round_trip(request: &SnapshotManagerRequest) -> anyhow::Result { - let mut pipe = std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(std::path::Path::new(SNAPSHOT_PIPE_NAME)) - .map_err(|err| anyhow::anyhow!("opening Snapshot Manager pipe: {err}"))?; - - write_framed_message(&mut pipe, &request.encode()) - .context("writing request to Snapshot Manager pipe")?; - let response_bytes = - read_framed_message(&mut pipe).context("reading response from Snapshot Manager pipe")?; - SnapshotManagerResponse::decode(&response_bytes) - .map_err(|err| anyhow::anyhow!("malformed Snapshot Manager response: {err}")) -} - -/// Write `payload` as `[u32 LE length][payload]`, flushing immediately โ€” -/// the client-side mirror of the Broker's `write_framed_message`. -fn write_framed_message(pipe: &mut std::fs::File, payload: &[u8]) -> anyhow::Result<()> { - let length = u32::try_from(payload.len()) - .map_err(|err| anyhow::anyhow!("request payload too large to frame: {err}"))?; - pipe.write_all(&length.to_le_bytes())?; - pipe.write_all(payload)?; - pipe.flush()?; - Ok(()) -} - -/// Read one `[u32 LE length][payload]`-framed message โ€” the -/// client-side mirror of the Broker's `read_framed_message`. -fn read_framed_message(pipe: &mut std::fs::File) -> anyhow::Result> { - let mut length_bytes = [0_u8; 4]; - pipe.read_exact(&mut length_bytes) - .context("reading response length prefix")?; - let length = u32::from_le_bytes(length_bytes); - if length > MAX_RESPONSE_BYTES { - anyhow::bail!("response length {length} exceeds maximum {MAX_RESPONSE_BYTES}"); - } - - let mut payload = vec![0_u8; usize::try_from(length).unwrap_or(0)]; - pipe.read_exact(&mut payload) - .context("reading response payload")?; - Ok(payload) -} diff --git a/crates/uffs-content/src/job/tests.rs b/crates/uffs-content/src/job/tests.rs deleted file mode 100644 index e6dceb0c7..000000000 --- a/crates/uffs-content/src/job/tests.rs +++ /dev/null @@ -1,673 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Unit tests for job intake, candidate/content sources, manifest -//! building, and the end-to-end workflow. The full directory-walk -//! parity check against an independent oracle lives in -//! `crates/uffs-content/tests/e2e_dir_walk_parity_fake_reader.rs` โ€” these -//! tests instead cover this module's own internals in isolation. - -use alloc::sync::Arc; -use core::time::Duration; -use std::fs; -use std::path::{Path, PathBuf}; -use std::sync::Mutex; -use std::time::Instant; - -use uffs_content_protocol::codec::Reader; -use uffs_content_protocol::frame::{ - ContentChunk, FileBegin, FileEnd, FrameEnvelope, FrameType, ReadMode, -}; -use uffs_content_protocol::manifest::{CandidateRecord, ManifestHeader, ManifestTrailer}; - -use super::candidate_source::{CandidateEntry, CandidateSource, DirWalkCandidateSource}; -use super::content_source::{ContentSource, FsContentSource, ReadSession}; -use super::intake::JobRequest; -use super::manifest_builder::build_manifest; -use super::workflow::{ReadConcurrency, run_job}; - -#[test] -fn dir_walk_candidate_source_enumerates_files_not_directories() { - let dir = tempfile::tempdir().expect("create temp dir"); - fs::create_dir_all(dir.path().join("nested")).expect("create nested dir"); - fs::write(dir.path().join("a.txt"), b"a").expect("write a.txt"); - fs::write(dir.path().join("nested/b.txt"), b"bb").expect("write nested/b.txt"); - - let entries = DirWalkCandidateSource - .enumerate(dir.path()) - .expect("enumerate must succeed"); - - let mut relative_paths: Vec<_> = entries - .iter() - .map(|entry| entry.relative_path.clone()) - .collect(); - relative_paths.sort(); - assert_eq!(relative_paths, vec![ - PathBuf::from("a.txt"), - PathBuf::from("nested/b.txt"), - ]); -} - -#[test] -fn dir_walk_candidate_source_gives_hard_links_the_same_file_reference() { - let dir = tempfile::tempdir().expect("create temp dir"); - let original = dir.path().join("original.txt"); - let linked = dir.path().join("linked.txt"); - fs::write(&original, b"shared content").expect("write original"); - fs::hard_link(&original, &linked).expect("create hard link"); - - let entries = DirWalkCandidateSource - .enumerate(dir.path()) - .expect("enumerate must succeed"); - assert_eq!(entries.len(), 2); - - let mut file_references: Vec = entries.iter().map(|entry| entry.file_reference).collect(); - file_references.sort_unstable(); - let [first, second] = file_references.as_slice() else { - panic!("expected exactly two entries"); - }; - assert_eq!( - first, second, - "two directory entries for the same inode must share file_reference" - ); -} - -#[test] -fn fs_content_source_reads_bounded_ranges_and_reports_eof() { - let dir = tempfile::tempdir().expect("create temp dir"); - let path = dir.path().join("data.bin"); - fs::write(&path, b"0123456789").expect("write data.bin"); - - let entries = DirWalkCandidateSource - .enumerate(dir.path()) - .expect("enumerate must succeed"); - let entry = entries.first().expect("one entry expected"); - - let mut session = FsContentSource - .begin_read(entry, 0) - .expect("begin_read must succeed"); - - let first_half = session.read_at(0, 5).expect("read first half"); - assert_eq!(first_half, b"01234"); - - let second_half = session.read_at(5, 5).expect("read second half"); - assert_eq!(second_half, b"56789"); - - let past_eof = session - .read_at(10, 5) - .expect("read past EOF must not error"); - assert!(past_eof.is_empty(), "read at EOF must return no bytes"); -} - -#[test] -fn build_manifest_round_trips_through_the_wire_codec() { - let dir = tempfile::tempdir().expect("create temp dir"); - fs::write(dir.path().join("one.txt"), b"one").expect("write one.txt"); - fs::write(dir.path().join("two.txt"), b"two!!").expect("write two.txt"); - let entries = DirWalkCandidateSource - .enumerate(dir.path()) - .expect("enumerate must succeed"); - - let built = build_manifest([1_u8; 16], [2_u8; 16], [3_u8; 32], &entries) - .expect("build_manifest must succeed"); - assert_eq!(built.candidate_ids.len(), entries.len()); - - let mut reader = Reader::new(&built.bytes); - let header = ManifestHeader::decode(&mut reader).expect("decode header"); - assert_eq!(header.candidate_count, entries.len() as u64); - - let mut decoded_records = Vec::new(); - for _ in 0..header.candidate_count { - decoded_records.push(CandidateRecord::decode(&mut reader).expect("decode record")); - } - let trailer = ManifestTrailer::decode(&mut reader).expect("decode trailer"); - assert_eq!(reader.remaining(), 0, "trailer must be the last thing"); - assert_eq!(trailer.manifest_digest, built.manifest_digest); - - let mut decoded_ids: Vec = decoded_records - .iter() - .map(|record| record.candidate_id) - .collect(); - decoded_ids.sort_unstable(); - let mut expected_ids = built.candidate_ids.clone(); - expected_ids.sort_unstable(); - assert_eq!(decoded_ids, expected_ids); -} - -#[test] -fn run_job_produces_a_well_formed_frame_sequence_with_no_failures() { - let source_dir = tempfile::tempdir().expect("create source temp dir"); - fs::write(source_dir.path().join("hello.txt"), b"hello world").expect("write hello.txt"); - fs::create_dir_all(source_dir.path().join("sub")).expect("create sub dir"); - fs::write(source_dir.path().join("sub/empty.txt"), b"").expect("write empty.txt"); - - let run_dir = tempfile::tempdir().expect("create run temp dir"); - let request = JobRequest { - source_id: "test-source".to_owned(), - roots: vec![source_dir.path().to_path_buf()], - query: "*".to_owned(), - ..Default::default() - }; - - let mut frames = Vec::new(); - let outcome = run_job( - &request, - &DirWalkCandidateSource, - &FsContentSource, - run_dir.path(), - // >1 so this test also exercises the sliding-window concurrent- - // read path (`read_lease_run_pipelined`), not just the fully- - // sequential (`concurrency == 1`) case. - &ReadConcurrency::flat(4), - &[], - 0, - |frame| { - frames.push(frame); - Ok(()) - }, - ) - .expect("run_job must succeed"); - - assert_eq!(outcome.run_summary.candidate_count, 2); - assert_eq!(outcome.run_summary.succeeded_count, 2); - assert_eq!(outcome.run_summary.failed_retryable_count, 0); - assert_eq!(outcome.run_summary.failed_terminal_count, 0); - assert_eq!(outcome.run_summary.deferred_manual_count, 0); - assert_eq!(outcome.run_summary.logical_bytes_succeeded, 11); - - // Decode every emitted frame and assert the expected type sequence: - // JOB_BEGIN, then (FILE_BEGIN, [CONTENT_CHUNK]*, FILE_END) per - // candidate, then JOB_END. - let mut decoded_types = Vec::new(); - for frame_bytes in &frames { - let mut reader = Reader::new(frame_bytes); - let (envelope, _payload) = - FrameEnvelope::decode(&mut reader, u64::MAX).expect("decode frame envelope"); - assert_eq!(envelope.job_id, outcome.job_id); - decoded_types.push(envelope.frame_type); - } - - assert_eq!(decoded_types.first(), Some(&FrameType::JobBegin)); - assert_eq!(decoded_types.last(), Some(&FrameType::JobEnd)); - let file_end_count = decoded_types - .iter() - .filter(|frame_type| **frame_type == FrameType::FileEnd) - .count(); - assert_eq!(file_end_count, 2, "both candidates must reach FILE_END"); -} - -#[test] -fn a_run_larger_than_the_credit_window_still_completes_correctly() { - let source_dir = tempfile::tempdir().expect("create source temp dir"); - // concurrency 2 -> credit_window = 2 * 4 = 8 (see pipeline:: - // read_lease_run_pipelined), so a run of 40 files forces the feeder - // to actually exhaust and wait on credits several times over, not - // just exercise the never-full-window happy path every other test - // in this file takes. - let file_count: u64 = 40; - for index in 0..file_count { - fs::write( - source_dir.path().join(format!("file_{index:03}.txt")), - format!("content for file {index}").into_bytes(), - ) - .expect("write fixture file"); - } - - let run_dir = tempfile::tempdir().expect("create run temp dir"); - let request = JobRequest { - source_id: "credit-window-source".to_owned(), - roots: vec![source_dir.path().to_path_buf()], - query: "*".to_owned(), - ..Default::default() - }; - - let mut frames = Vec::new(); - let outcome = run_job( - &request, - &DirWalkCandidateSource, - &FsContentSource, - run_dir.path(), - &ReadConcurrency::flat(2), - &[], - 0, - |frame| { - frames.push(frame); - Ok(()) - }, - ) - .expect("run_job must succeed"); - - assert_eq!(outcome.run_summary.candidate_count, file_count); - assert_eq!(outcome.run_summary.succeeded_count, file_count); - assert_eq!(outcome.run_summary.failed_retryable_count, 0); - assert_eq!(outcome.run_summary.failed_terminal_count, 0); - - // FILE_END frames must appear in strict, gapless candidate_id order - // -- confirms the credit-window backpressure never disturbs the - // pipeline's strict-order emission guarantee, even when the feeder - // is repeatedly forced to block waiting for a credit. - let mut file_end_candidate_ids = Vec::new(); - for frame_bytes in &frames { - let mut reader = Reader::new(frame_bytes); - let (envelope, payload) = - FrameEnvelope::decode(&mut reader, u64::MAX).expect("decode frame envelope"); - if envelope.frame_type == FrameType::FileEnd { - let mut payload_reader = Reader::new(&payload); - let file_end = FileEnd::decode(&mut payload_reader).expect("decode file end"); - file_end_candidate_ids.push(file_end.candidate_id); - } - } - let expected_ids: Vec = (1..=file_count).collect(); - assert_eq!( - file_end_candidate_ids, expected_ids, - "FILE_END frames must appear in strict candidate_id order even when the run exceeds \ - the credit window" - ); -} - -#[test] -fn candidates_over_the_delivery_ceiling_are_reported_metadata_only() { - let source_dir = tempfile::tempdir().expect("create source temp dir"); - fs::write(source_dir.path().join("small.txt"), b"tiny").expect("write small.txt"); - fs::write(source_dir.path().join("big.bin"), vec![0_u8; 64]).expect("write big.bin"); - - let run_dir = tempfile::tempdir().expect("create run temp dir"); - let request = JobRequest { - source_id: "ceiling-source".to_owned(), - roots: vec![source_dir.path().to_path_buf()], - query: "*".to_owned(), - max_content_delivery_bytes: Some(10), - ..Default::default() - }; - - let mut frames = Vec::new(); - let outcome = run_job( - &request, - &DirWalkCandidateSource, - &FsContentSource, - run_dir.path(), - &ReadConcurrency::flat(2), - &[], - 0, - |frame| { - frames.push(frame); - Ok(()) - }, - ) - .expect("run_job must succeed"); - - assert_eq!(outcome.run_summary.candidate_count, 2); - assert_eq!(outcome.run_summary.succeeded_count, 2); - - let mut file_ends = Vec::new(); - for frame_bytes in &frames { - let mut reader = Reader::new(frame_bytes); - let (envelope, payload) = - FrameEnvelope::decode(&mut reader, u64::MAX).expect("decode frame envelope"); - if envelope.frame_type == FrameType::FileEnd { - let mut payload_reader = Reader::new(&payload); - file_ends.push(FileEnd::decode(&mut payload_reader).expect("decode file end")); - } - } - assert_eq!(file_ends.len(), 2); - - let big_end = file_ends - .iter() - .find(|end| end.total_logical_bytes == 0) - .expect("big file's FILE_END must report zero delivered bytes"); - assert_eq!(big_end.read_mode, ReadMode::MetadataOnly); - assert!(big_end.content_digest.is_none()); - assert_eq!(big_end.chunk_count, 0); - - let small_end = file_ends - .iter() - .find(|end| end.total_logical_bytes == 4) - .expect("small file's FILE_END must report its actual byte count"); - assert_eq!(small_end.read_mode, ReadMode::LogicalSnapshot); - assert!(small_end.content_digest.is_some()); -} - -/// Test-only [`CandidateSource`] that fabricates a fixed small candidate -/// set per root, tagging each with a `snapshot_lease_id` parsed from the -/// root itself (`"lease:"`) โ€” lets a single test drive multiple -/// concurrent lease runs without touching the filesystem or a real VSS -/// snapshot. -struct MultiLeaseCandidateSource { - /// Candidates to synthesize per lease. - per_lease: usize, -} - -impl CandidateSource for MultiLeaseCandidateSource { - fn enumerate(&self, root: &Path) -> std::io::Result> { - let root_str = root.to_string_lossy(); - let lease_id: u64 = root_str - .strip_prefix("lease:") - .and_then(|suffix| suffix.parse().ok()) - .unwrap_or(0); - Ok((0..self.per_lease) - .map(|i| { - let name = format!("file-{lease_id}-{i}.txt"); - CandidateEntry { - relative_path: PathBuf::from(&name), - absolute_path: PathBuf::from(&name), - logical_size: 4, - mtime_unix_ms: 0, - file_reference: lease_id * 1000 + i as u64, - snapshot_lease_id: lease_id, - } - }) - .collect()) - } -} - -/// Asserts that at least two of `intervals` overlap in wall-clock time โ€” -/// direct proof that two of the recorded operations actually ran at the -/// same time, rather than inferring concurrency from a coarse -/// elapsed-time-vs-threshold check. A fixed millisecond threshold is -/// only ever true *relative to how fast the test machine happens to be -/// that run*; on a loaded/throttled CI runner, `thread::sleep(100ms)` -/// itself can take several hundred milliseconds of wall-clock time -/// (real GitHub-hosted-Windows-runner behavior, not hypothetical โ€” see -/// the `concurrent_lease_runs_...` test's git history), which pushes -/// *both* the sequential and concurrent paths past any fixed absolute -/// threshold and produces a false failure. Two intervals overlapping is -/// true or false independent of how slow the machine is: it only asks -/// whether two things happened during the same stretch of time. -fn assert_any_two_intervals_overlap(intervals: &[(Instant, Instant)], what: &str) { - for (i, &(a_start, a_end)) in intervals.iter().enumerate() { - for &(b_start, b_end) in intervals.get(i + 1..).unwrap_or_default() { - if a_start < b_end && b_start < a_end { - return; - } - } - } - panic!("no two {what} intervals overlap in wall-clock time: {intervals:?}"); -} - -/// Test-only [`ContentSource`] whose every candidate read sleeps -/// `per_candidate_delay` before returning a fixed 4-byte payload โ€” -/// simulates real per-candidate I/O latency without touching a real -/// disk. Records each read's (start, end) so a test can prove two -/// lease runs' reads genuinely overlapped in wall-clock time (see -/// [`assert_any_two_intervals_overlap`]) rather than inferring it from -/// an elapsed-time-vs-threshold check. -struct SlowContentSource { - /// How long each candidate's one real read takes. - per_candidate_delay: Duration, - /// (start, end) of every `read_at` call across every thread. `Arc` - /// (not a borrow of `&self`) because `ContentSource::begin_read` - /// returns a `'static` `Box`. - intervals: Arc>>, -} - -impl SlowContentSource { - fn new(per_candidate_delay: Duration) -> Self { - Self { - per_candidate_delay, - intervals: Arc::new(Mutex::new(Vec::new())), - } - } -} - -impl ContentSource for SlowContentSource { - fn begin_read( - &self, - _candidate: &CandidateEntry, - _candidate_id: u64, - ) -> std::io::Result> { - Ok(Box::new(SlowReadSession { - delay: self.per_candidate_delay, - served: false, - intervals: Arc::clone(&self.intervals), - })) - } -} - -/// [`SlowContentSource`]'s session: sleeps once, returns 4 bytes, then -/// signals EOF on every subsequent call. -struct SlowReadSession { - /// How long the one real read takes. - delay: Duration, - /// Whether the 4-byte payload has already been served. - served: bool, - /// Shared back-reference to record this read's (start, end) into. - intervals: Arc>>, -} - -impl ReadSession for SlowReadSession { - fn read_at(&mut self, _offset: u64, _max_len: u32) -> std::io::Result> { - let start = Instant::now(); - std::thread::sleep(self.delay); - let end = Instant::now(); - self.intervals - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .push((start, end)); - if self.served { - return Ok(Vec::new()); - } - self.served = true; - Ok(b"data".to_vec()) - } -} - -/// Multiple lease runs (drives) must run concurrently, not one fully -/// finishing before the next starts โ€” real-hardware benchmarking found -/// a slow HDD-backed lease holding up every other drive's candidates, -/// including a fast SSD-backed lease sitting idle in queue, even though -/// they share no connection pool or physical device. Proves this two -/// ways: wall-clock time must reflect the *slowest single lease*, not -/// the *sum* of every lease's time, and the emitted frame stream must -/// never let one candidate's frame group be split apart by another's โ€” -/// the one atomicity guarantee concurrent lease runs must still uphold -/// (see `workflow`'s "Concurrent reads, concurrent drives, atomic -/// per-candidate emission" doc section). -#[test] -fn concurrent_lease_runs_actually_overlap_and_never_interleave_a_candidates_frames() { - const CANDIDATES_PER_LEASE: usize = 3; - const PER_CANDIDATE_DELAY: Duration = Duration::from_millis(100); - - let run_dir = tempfile::tempdir().expect("create run temp dir"); - let request = JobRequest { - source_id: "test-source".to_owned(), - roots: vec![PathBuf::from("lease:1"), PathBuf::from("lease:2")], - query: "*".to_owned(), - ..Default::default() - }; - - let candidate_source = MultiLeaseCandidateSource { - per_lease: CANDIDATES_PER_LEASE, - }; - let content_source = SlowContentSource::new(PER_CANDIDATE_DELAY); - - let mut frames = Vec::new(); - let outcome = run_job( - &request, - &candidate_source, - &content_source, - run_dir.path(), - &ReadConcurrency::flat(1), - &[], - 0, - |frame| { - frames.push(frame); - Ok(()) - }, - ) - .expect("run_job must succeed"); - - let total_candidates = 2 * CANDIDATES_PER_LEASE; - assert_eq!(outcome.run_summary.candidate_count, total_candidates as u64); - assert_eq!(outcome.run_summary.succeeded_count, total_candidates as u64); - assert_eq!(outcome.run_summary.failed_retryable_count, 0); - - // Direct proof of concurrency: two different leases' reads must - // genuinely overlap in wall-clock time (each lease runs its - // CANDIDATES_PER_LEASE reads sequentially within itself, at - // concurrency = 1, so an overlap can only come from two *different* - // leases' single-connection reads proceeding at the same time). - let intervals = content_source - .intervals - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - assert_any_two_intervals_overlap(&intervals, "lease-run read_at"); - - // Correctness: decode every frame in emission order and confirm no - // candidate's frame group (FILE_BEGIN..FILE_END) is ever split apart - // by another candidate's frames -- the one atomicity guarantee that - // must hold regardless of how many lease runs execute concurrently. - let mut open_candidate: Option = None; - for frame_bytes in &frames { - let mut reader = Reader::new(frame_bytes); - let Ok((envelope, payload)) = FrameEnvelope::decode(&mut reader, u64::MAX) else { - panic!("every emitted frame must decode"); - }; - match envelope.frame_type { - FrameType::FileBegin => { - assert_eq!( - open_candidate, None, - "a new FILE_BEGIN must never arrive while another candidate is still open" - ); - let file_begin = FileBegin::decode(&mut Reader::new(&payload)) - .expect("decode FILE_BEGIN payload"); - open_candidate = Some(file_begin.candidate_id); - } - FrameType::ContentChunk => { - let chunk = ContentChunk::decode(&mut Reader::new(&payload), u32::MAX) - .expect("decode CONTENT_CHUNK payload"); - assert_eq!( - open_candidate, - Some(chunk.candidate_id), - "a CONTENT_CHUNK must belong to the currently-open candidate" - ); - } - FrameType::FileEnd => { - let file_end = - FileEnd::decode(&mut Reader::new(&payload)).expect("decode FILE_END payload"); - assert_eq!( - open_candidate, - Some(file_end.candidate_id), - "FILE_END must close the currently-open candidate" - ); - open_candidate = None; - } - FrameType::JobBegin | FrameType::JobEnd => {} - other @ (FrameType::FileFailed - | FrameType::FileDeferred - | FrameType::FileAck - | FrameType::Progress - | FrameType::Heartbeat - | FrameType::JobCancel - | FrameType::WindowUpdate - | FrameType::JobResume - | FrameType::JobSubmit) => { - panic!("unexpected frame type in this test's stream: {other:?}") - } - } - } - assert_eq!( - open_candidate, None, - "every candidate must be closed by the end of the stream" - ); -} - -/// Test-only [`CandidateSource`] whose every `enumerate` call sleeps -/// `per_root_delay` before returning one fixed candidate for `root` โ€” -/// simulates real per-root search latency (a synchronous round trip to -/// the daemon) without touching a real daemon. Records each call's -/// (start, end) so a test can prove two roots' enumeration genuinely -/// overlapped in wall-clock time (see -/// [`assert_any_two_intervals_overlap`]) rather than inferring it from -/// an elapsed-time-vs-threshold check. -struct SlowEnumerateCandidateSource { - /// How long each root's `enumerate` call takes. - per_root_delay: Duration, - /// (start, end) of every `enumerate` call across every thread. - intervals: Mutex>, -} - -impl SlowEnumerateCandidateSource { - fn new(per_root_delay: Duration) -> Self { - Self { - per_root_delay, - intervals: Mutex::new(Vec::new()), - } - } -} - -impl CandidateSource for SlowEnumerateCandidateSource { - fn enumerate(&self, root: &Path) -> std::io::Result> { - let start = Instant::now(); - std::thread::sleep(self.per_root_delay); - let end = Instant::now(); - self.intervals - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .push((start, end)); - let root_str = root.to_string_lossy(); - let root_index: u64 = root_str - .strip_prefix("root:") - .and_then(|suffix| suffix.parse().ok()) - .unwrap_or(0); - let name = format!("file-{root_index}.txt"); - Ok(vec![CandidateEntry { - relative_path: PathBuf::from(&name), - absolute_path: PathBuf::from(&name), - logical_size: 4, - mtime_unix_ms: 0, - file_reference: root_index, - snapshot_lease_id: 0, - }]) - } -} - -/// Enumerating multiple roots must run concurrently, not one root's -/// whole search-and-collect cycle blocking the next โ€” real-hardware -/// benchmarking found a two-drive job's enumeration costing ~15s + ~13s -/// back to back (~28s total) even though each root's `enumerate` call -/// opens its own independent connection to the daemon and shares no -/// mutable state with any other call (see -/// `workflow::enumerate_all_roots_concurrently`'s own doc comment). -#[test] -fn root_enumeration_actually_overlaps_across_roots() { - const ROOT_COUNT: usize = 4; - const PER_ROOT_DELAY: Duration = Duration::from_millis(100); - - let run_dir = tempfile::tempdir().expect("create run temp dir"); - let roots: Vec = (0..ROOT_COUNT) - .map(|i| PathBuf::from(format!("root:{i}"))) - .collect(); - let request = JobRequest { - source_id: "test-source".to_owned(), - roots, - query: "*".to_owned(), - ..Default::default() - }; - - let candidate_source = SlowEnumerateCandidateSource::new(PER_ROOT_DELAY); - let content_source = SlowContentSource::new(Duration::ZERO); - - let outcome = run_job( - &request, - &candidate_source, - &content_source, - run_dir.path(), - &ReadConcurrency::flat(1), - &[], - 0, - |_frame| Ok(()), - ) - .expect("run_job must succeed"); - - assert_eq!(outcome.run_summary.candidate_count, ROOT_COUNT as u64); - assert_eq!(outcome.run_summary.succeeded_count, ROOT_COUNT as u64); - assert_eq!(outcome.run_summary.failed_retryable_count, 0); - - // Direct proof of concurrency: two different roots' `enumerate` - // calls must genuinely overlap in wall-clock time. - let intervals = candidate_source - .intervals - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - assert_any_two_intervals_overlap(&intervals, "root enumeration"); -} diff --git a/crates/uffs-content/src/job/vss_job.rs b/crates/uffs-content/src/job/vss_job.rs deleted file mode 100644 index dd514c0a4..000000000 --- a/crates/uffs-content/src/job/vss_job.rs +++ /dev/null @@ -1,185 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! End-to-end VSS-backed job execution โ€” the real production path. -//! -//! Ties every piece built for UFI.1/UFI.2 together: lease the drive(s) -//! a job's root touches, spin up the ephemeral target-selection daemon, -//! enumerate candidates against it, spawn the privileged content Reader, -//! stream content through [`super::workflow::run_job`], and tear -//! everything down โ€” in the right order (content Reader/leases outlive -//! candidate enumeration; see `EphemeralJobResources` -//! ([`super::vss_orchestrator`]) for why daemon and leases -//! are bundled into one teardown step). -//! -//! Windows-only: every piece this wires together already is. - -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -use anyhow::{Context as _, Result}; - -use super::candidate_source::VssCandidateSource; -use super::content_source::VssContentSource; -use super::intake::JobRequest; -use super::reader_client::{CONNECTIONS_PER_DRIVE, ContentReader}; -use super::vss_orchestrator; -use super::workflow::{JobOutcome, ReadConcurrency, run_job}; - -/// Run `request` end to end against a real VSS snapshot. -/// -/// Every encoded frame is passed to `emit_frame` as soon as it's -/// produced โ€” see [`run_job`]'s own doc comment for why this is a -/// callback rather than a returned `Vec`. -/// -/// `request.roots` is used as given if non-empty; if empty, this job -/// defaults to every local NTFS drive (`uffs_mft::detect_ntfs_drives`) โ€” -/// the same auto-discovery `uffsd` itself falls back to when started -/// with no `--drive` flag. -/// -/// # Errors -/// Returns an error if root resolution finds no local NTFS drives to -/// default to, any VSS lease, ephemeral daemon spawn, or content Reader -/// spawn step fails, or if the underlying `run_job` call fails. Every -/// resource successfully acquired before a failure is released -/// best-effort before returning. -pub fn run_vss_job(request: &JobRequest, run_dir: &Path, emit_frame: F) -> Result -where - F: FnMut(Vec) -> std::io::Result<()> + Send, -{ - let job_id = *uuid::Uuid::new_v4().as_bytes(); - let ephemeral_id = uuid::Uuid::new_v4().simple().to_string(); - - let roots = resolve_roots(request)?; - let root_paths: Vec<&Path> = roots.iter().map(PathBuf::as_path).collect(); - - let resources = - vss_orchestrator::prepare_ephemeral_daemon_for_roots(job_id, &root_paths, &ephemeral_id) - .context("failed to lease VSS snapshot(s) and spawn the target-selection daemon")?; - - let drive_to_lease: HashMap = resources - .leases - .iter() - .map(|lease| (lease.drive_letter, lease.lease_id)) - .collect(); - // The resolved (never-empty) root list is what `run_job`'s own - // enumeration loop must iterate, not whatever `request.roots` - // originally said (which may have been empty, relying on the - // default-to-all-drives resolution above). - let mut resolved_request = request.clone(); - resolved_request.roots = roots; - let candidate_source = - VssCandidateSource::new(&resolved_request, &resources.daemon, drive_to_lease); - - // Per-drive read concurrency: an HDD gets exactly one connection - // (read its candidates strictly one at a time, in the order they - // were enumerated โ€” approximating sequential disk access instead of - // seek-thrashing between concurrent reads on a spinning disk), an - // NVMe/SSD gets `CONNECTIONS_PER_DRIVE` (no seek penalty to protect, - // and it benefits from many reads in flight). See - // `reader_client::CONNECTIONS_PER_DRIVE` and - // `workflow::ReadConcurrency`'s doc comments for both sides of this. - let mut devices_for_reader: Vec<(String, u64, usize)> = - Vec::with_capacity(resources.leases.len()); - let mut read_concurrency = ReadConcurrency::new(1); - for lease in &resources.leases { - let connections = drive_read_connections(lease.drive_letter); - tracing::info!( - drive = %lease.drive_letter, - connections, - "content read: per-drive connection count" - ); - read_concurrency.set(lease.lease_id, connections); - devices_for_reader.push((lease.device_path.clone(), lease.lease_id, connections)); - } - - let content_reader = ContentReader::spawn(job_id, &devices_for_reader) - .context("failed to spawn the content reader")?; - let content_source = VssContentSource::new(content_reader); - - // JOB_BEGIN carries only one job-level snapshot_id/snapshot_created_at - // pair (see `JobBegin`'s own doc comment), so a multi-drive job's - // provenance is necessarily a representative one, not one per drive. - // The first leased drive is as good a choice as any: every lease for - // a job is taken back-to-back at job start (see - // `vss_orchestrator::prepare_ephemeral_daemon_for_roots`), so their - // snapshot_created_at values differ by at most the lease loop's own - // wall-clock time, not something a consumer's temporal-memory use case - // would notice. - let (snapshot_id, snapshot_created_at) = resources - .leases - .first() - .map(|lease| (lease.snapshot_id.clone(), lease.snapshot_created_at_unix_ms)) - .unwrap_or_default(); - - let result = run_job( - &resolved_request, - &candidate_source, - &content_source, - run_dir, - &read_concurrency, - &snapshot_id, - snapshot_created_at, - emit_frame, - ) - .context("run_job failed"); - - // Drop the candidate source first (releases its borrow of - // `resources.daemon`, which `resources.teardown()` below needs to - // consume by value), then tear down the Reader and the - // daemon/leases explicitly so a failed teardown is observable - // rather than silently swallowed by `Drop`. - drop(candidate_source); - if let Err(err) = content_source.shutdown() { - tracing::warn!(error = %err, "failed to fully tear down content reader"); - } - if let Err(err) = resources.teardown() { - tracing::warn!(error = %err, "failed to fully tear down VSS job resources"); - } - - result -} - -/// How many concurrent content-read connections `drive_letter`'s lease -/// should get: [`CONNECTIONS_PER_DRIVE`] for a high-performance medium -/// (NVMe/SSD โ€” no seek penalty, benefits from many reads in flight), or -/// exactly `1` for anything else (HDD, removable, virtual, or a type -/// that couldn't be determined) โ€” see [`CONNECTIONS_PER_DRIVE`]'s doc -/// comment for why concurrency `1` is the correct choice for a -/// seek-bound medium, not just a conservative fallback. -fn drive_read_connections(drive_letter: char) -> usize { - let Ok(letter) = uffs_mft::platform::DriveLetter::parse(drive_letter) else { - // Unreachable in practice: `drive_letter` came from a lease VSS - // already accepted for this exact letter. Fall back to the safe - // (sequential) choice rather than panicking. - return 1; - }; - if uffs_mft::platform::detect_drive_type(letter).is_high_performance() { - CONNECTIONS_PER_DRIVE - } else { - 1 - } -} - -/// Resolve `request.roots`: as given if non-empty, else one root per -/// local NTFS drive on this machine โ€” the consumer's "search everything" -/// default, matching `uffsd`'s own no-`--drive`-flag fallback -/// (`uffs_mft::detect_ntfs_drives`). -/// -/// # Errors -/// Returns an error if `request.roots` is empty and no local NTFS drive -/// is found to default to. -fn resolve_roots(request: &JobRequest) -> Result> { - if !request.roots.is_empty() { - return Ok(request.roots.clone()); - } - let drives = uffs_mft::detect_ntfs_drives(); - anyhow::ensure!( - !drives.is_empty(), - "no roots given and no local NTFS drive found to default to" - ); - Ok(drives - .into_iter() - .map(|letter| PathBuf::from(format!("{}:\\", letter.as_char()))) - .collect()) -} diff --git a/crates/uffs-content/src/job/vss_orchestrator.rs b/crates/uffs-content/src/job/vss_orchestrator.rs deleted file mode 100644 index ca8e70587..000000000 --- a/crates/uffs-content/src/job/vss_orchestrator.rs +++ /dev/null @@ -1,278 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Ties [`super::snapshot_client`] and [`super::ephemeral_daemon`] together. -//! -//! Leases one VSS snapshot per distinct drive a job's roots touch, then -//! spawns a single ephemeral `uffsd` instance covering all of them โ€” per -//! the user's direct design decision: "create a VSS for each drive -//! (multiple) ... then when all done have the UFFS-content tool spin up -//! one instance of the daemon covering all the VSS MFT copies." - -use std::collections::HashSet; -use std::path::Path; - -use anyhow::{Context as _, Result}; -use uffs_broker_protocol::snapshot_manager::{SnapshotManagerErrorCode, VolumeIdentity}; - -use super::ephemeral_daemon::EphemeralDaemon; -use super::snapshot_client::{self, BrokerRejectedCreate}; - -/// `VSS_E_VOLUME_NOT_SUPPORTED` โ€” VSS permanently refuses to snapshot -/// this volume (observed in practice on removable/USB media). Distinct -/// from `uffs-vss-requestor`'s `RETRYABLE_HRESULTS`: this is -/// deliberately *not* in that list, and a job should skip the drive -/// rather than fail outright, since it will never become supported by -/// retrying. -const VSS_E_VOLUME_NOT_SUPPORTED: i32 = 0x8004_230C_u32.cast_signed(); - -/// Default VSS snapshot lease lifetime. -/// -/// Generous relative to a single ingest job's expected wall-clock time; -/// revisit once real job-duration telemetry exists (no policy schema -/// for this yet โ€” see [`DEFAULT_POLICY_ID`]). -const DEFAULT_LEASE_LIFETIME_SECS: u64 = 3600; - -/// Placeholder policy id โ€” the Broker's authorization-policy schema -/// doesn't exist yet; every lease request uses this until it does. -const DEFAULT_POLICY_ID: u32 = 0; - -/// One leased drive: the snapshot device path, the drive letter it was -/// leased from, and the lease id โ€” everything a caller needs to build -/// either the daemon's `--device =` args or the Reader's -/// `--device =` args, or to correlate a query result -/// row's drive letter back to the lease that produced it. -pub(crate) struct LeasedDrive { - /// VSS snapshot device path (e.g. - /// `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopyN`). - pub(crate) device_path: String, - /// Drive letter the snapshot was taken from. - pub(crate) drive_letter: char, - /// This drive's lease id. - pub(crate) lease_id: u64, - /// Opaque VSS snapshot identifier, as reported by the Broker at - /// lease time. Carried through to `JOB_BEGIN.snapshot_id` (see - /// `super::vss_job::run_vss_job`) โ€” one drive's worth of real - /// snapshot provenance, since the wire protocol has only one - /// job-level `snapshot_id`/`snapshot_created_at` pair even though a - /// job may lease several drives. - pub(crate) snapshot_id: Vec, - /// This drive's snapshot creation time, Unix milliseconds. - pub(crate) snapshot_created_at_unix_ms: i64, -} - -/// Every live resource this orchestration step produced. -/// -/// The daemon and the leases have different intended lifetimes: the -/// daemon is only needed for target-selection queries (`enumerate`) and -/// can be torn down as soon as that's done, while the leases must stay -/// alive for the whole job (content reading happens against the same -/// snapshots afterward). This struct bundles both anyway and tears them -/// down together via [`Self::teardown`] โ€” a v1 simplification (the -/// daemon's memory stays resident through content-reading unnecessarily) -/// documented here rather than silently accepted; revisit if daemon -/// memory footprint during long content-reading phases matters in -/// practice. -pub(crate) struct EphemeralJobResources { - /// The running ephemeral daemon covering every leased drive. - pub(crate) daemon: EphemeralDaemon, - /// Every drive this job leased, in lease order. - pub(crate) leases: Vec, -} - -impl EphemeralJobResources { - /// Tear down the daemon, then release every lease. Daemon teardown - /// runs first โ€” releasing a lease out from under a still-running - /// daemon would pull the volume out from under its loaded index. - /// Lease release is best-effort per-lease (a single failed release - /// is logged, not fatal โ€” see [`release_all_leases`]). - /// - /// # Errors - /// Returns an error if the daemon couldn't be killed. Lease-release - /// failures are logged, not propagated. - pub(crate) fn teardown(self) -> Result<()> { - self.daemon.shutdown()?; - let lease_ids: Vec = self.leases.iter().map(|lease| lease.lease_id).collect(); - release_all_leases(&lease_ids); - Ok(()) - } -} - -/// Lease one VSS snapshot per distinct drive letter across `roots`, -/// then spawn one combined ephemeral daemon covering all of them. -/// -/// Drive letters are read directly off each root path's own prefix -/// (`drive_letter_from_path`) โ€” never inferred from the MFT: the -/// Coordinator already knows which drive it's snapshotting, per the -/// user's explicit correction during design. -/// -/// A drive VSS permanently refuses to snapshot (`VSS_E_VOLUME_NOT_SUPPORTED` -/// โ€” seen in practice on removable/USB media) is skipped, not fatal: it is -/// warn-logged and left out of the returned leases/daemon devices, so the -/// rest of a multi-drive "all drives" job still completes. Any other lease -/// failure still aborts the whole job. -/// -/// # Errors -/// Returns an error if any root has no drive-letter prefix, a lease -/// request fails for a reason other than `VSS_E_VOLUME_NOT_SUPPORTED`, -/// or the ephemeral daemon fails to spawn or become ready. On error, -/// any leases already taken out are released best-effort before -/// returning. -pub(crate) fn prepare_ephemeral_daemon_for_roots( - job_id: [u8; 16], - roots: &[&Path], - ephemeral_id: &str, -) -> Result { - let mut leases: Vec = Vec::new(); - let mut seen_letters = HashSet::new(); - - for root in roots { - let letter = drive_letter_from_path(root) - .with_context(|| format!("job root {} has no drive-letter prefix", root.display()))?; - if !seen_letters.insert(letter) { - continue; // already leased this drive for an earlier root - } - - match lease_one_drive(job_id, letter) { - Ok(Some(lease)) => leases.push(lease), - Ok(None) => {} // VSS_E_VOLUME_NOT_SUPPORTED โ€” skip, already warn-logged - Err(err) => { - release_all_leases(&lease_ids(&leases)); - return Err(err); - } - } - } - - let devices: Vec<(String, char)> = leases - .iter() - .map(|lease| (lease.device_path.clone(), lease.drive_letter)) - .collect(); - - tracing::info!(drive_count = devices.len(), "spawning ephemeral daemon"); - match EphemeralDaemon::spawn(ephemeral_id, &devices) { - Ok(daemon) => { - tracing::info!("ephemeral daemon ready"); - Ok(EphemeralJobResources { daemon, leases }) - } - Err(err) => { - tracing::warn!(error = %err, "ephemeral daemon spawn failed"); - release_all_leases(&lease_ids(&leases)); - Err(err) - } - } -} - -/// Lease a VSS snapshot for drive `letter`, split out of -/// [`prepare_ephemeral_daemon_for_roots`]'s loop to keep that function's -/// cognitive complexity down. -/// -/// `Ok(None)` means VSS specifically reported -/// `VSS_E_VOLUME_NOT_SUPPORTED` for this volume โ€” skip it, not fatal -/// (already warn-logged here) โ€” see the caller's doc comment. -/// -/// # Errors -/// Returns any other lease failure reason. -fn lease_one_drive(job_id: [u8; 16], letter: char) -> Result> { - tracing::info!(drive = %letter, "leasing VSS snapshot"); - let requested_root = utf16le_bytes(&format!("{letter}:\\")); - let lease_result = snapshot_client::create_lease( - job_id, - VolumeIdentity { - // Presently inert: the Broker's real `create_snapshot` path - // derives the volume to snapshot from `requested_root`, not - // this struct (confirmed via direct source read) โ€” populate - // a real serial/GUID once the Broker actually validates - // against it. - volume_serial: 0, - volume_guid: Vec::new(), - }, - requested_root, - DEFAULT_LEASE_LIFETIME_SECS, - DEFAULT_POLICY_ID, - ); - - let lease = match lease_result { - Ok(lease) => lease, - Err(err) if is_volume_not_supported(&err) => { - tracing::warn!( - drive = %letter, - "skipping drive: VSS does not support snapshotting this volume \ - (VSS_E_VOLUME_NOT_SUPPORTED โ€” typically removable/USB media)" - ); - return Ok(None); - } - Err(err) => { - return Err(err.context(format!("failed to lease a VSS snapshot for drive {letter}"))); - } - }; - tracing::info!( - drive = %letter, - lease_id = lease.snapshot_lease_id, - device = %lease.snapshot_device_identity, - "VSS snapshot leased" - ); - Ok(Some(LeasedDrive { - device_path: lease.snapshot_device_identity, - drive_letter: letter, - lease_id: lease.snapshot_lease_id, - snapshot_id: lease.snapshot_id, - snapshot_created_at_unix_ms: lease.snapshot_created_at_unix_ms, - })) -} - -/// Whether `err` is a [`BrokerRejectedCreate`] specifically reporting -/// `VSS_E_VOLUME_NOT_SUPPORTED` for a snapshot-creation failure โ€” the -/// one lease-failure reason a multi-drive job should skip past rather -/// than abort on (see [`prepare_ephemeral_daemon_for_roots`]'s doc -/// comment). -fn is_volume_not_supported(err: &anyhow::Error) -> bool { - err.downcast_ref::() - .is_some_and(|rejected| { - rejected.code == SnapshotManagerErrorCode::SnapshotCreateFailed - && rejected.hresult == Some(VSS_E_VOLUME_NOT_SUPPORTED) - }) -} - -/// Extract just the lease ids from `leases`, for [`release_all_leases`]. -fn lease_ids(leases: &[LeasedDrive]) -> Vec { - leases.iter().map(|lease| lease.lease_id).collect() -} - -/// Release every lease in `lease_ids`. Best-effort: a single failed -/// release is warn-logged, not propagated โ€” teardown must proceed even -/// if one lease is already gone or the Broker is unreachable. -fn release_all_leases(lease_ids: &[u64]) { - for &lease_id in lease_ids { - if let Err(err) = snapshot_client::release_lease(lease_id) { - tracing::warn!(lease_id, error = %err, "failed to release VSS snapshot lease"); - } - } -} - -/// Extract the drive letter a Windows path is rooted on (e.g. -/// `C:\Users\x` -> `'C'`). -/// -/// Cheap and purely syntactic: reads the path's own `Prefix` component. -/// Never touches the filesystem or the MFT โ€” the whole point is that -/// the Coordinator already knows which drive it's snapshotting from the -/// job's own root path. -fn drive_letter_from_path(path: &Path) -> Option { - use std::path::{Component, Prefix}; - - let Component::Prefix(prefix) = path.components().next()? else { - return None; - }; - let (Prefix::Disk(byte) | Prefix::VerbatimDisk(byte)) = prefix.kind() else { - return None; - }; - Some(byte.to_ascii_uppercase() as char) -} - -/// Encode `text` as raw UTF-16LE bytes (no null terminator) โ€” the wire -/// format [`snapshot_client::create_lease`]'s `requested_root` expects, -/// mirroring the Broker's own `utf16le_bytes` helper -/// (`crates/uffs-broker/src/broker/snapshot_manager/vss_helper.rs`, -/// `pub(crate)` there so not reusable directly from this crate). -fn utf16le_bytes(text: &str) -> Vec { - text.encode_utf16().flat_map(u16::to_le_bytes).collect() -} diff --git a/crates/uffs-content/src/job/window.rs b/crates/uffs-content/src/job/window.rs deleted file mode 100644 index 63ff4eabd..000000000 --- a/crates/uffs-content/src/job/window.rs +++ /dev/null @@ -1,136 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Credit-based backpressure (design-doc ยง13.1 "Byte-based limits" / -//! ยง13.2 "Slow consumer"). -//! -//! Same mechanism as HTTP/2 stream-level flow control (RFC 7540 ยง6.9) -//! and TCP's receive-window advertisement: the consumer grants the -//! producer a byte budget up front; the producer consumes budget as it -//! sends `CONTENT_CHUNK` bytes and must stop admitting new read work -//! once the budget is exhausted; a `WINDOW_UPDATE` frame from the -//! consumer raises the ceiling as it frees buffer space. Deliberately -//! independent of `FILE_ACK` (a separate, file-granularity, digest- -//! verified concern โ€” see `crate::job::registry`) โ€” a consumer may grant -//! window credit as soon as it has buffer room, without having verified -//! or durably persisted any specific file yet. - -/// Tracks how many bytes the producer may still send before it must -/// pause and wait for a `WINDOW_UPDATE`. -#[cfg_attr( - not(any(windows, test)), - expect( - dead_code, - reason = "only constructed by the Windows-only `serve` module's streaming \ - task in production; exercised cross-platform by this module's own \ - unit tests, which is why the type still lives here rather than \ - behind `#[cfg(windows)]`" - ) -)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct WindowTracker { - /// Total bytes ever granted: the initial negotiated - /// `max_unacknowledged_bytes` plus every `WINDOW_UPDATE` grant since. - granted_bytes: u64, - /// Total bytes sent so far. - sent_bytes: u64, -} - -#[cfg_attr( - not(any(windows, test)), - expect(dead_code, reason = "see the `WindowTracker` doc comment above") -)] -impl WindowTracker { - /// A new tracker starting with `initial_window_bytes` of budget - /// (the negotiated `max_unacknowledged_bytes`). - pub(crate) const fn new(initial_window_bytes: u64) -> Self { - Self { - granted_bytes: initial_window_bytes, - sent_bytes: 0, - } - } - - /// Bytes still available to send before the window is exhausted. - pub(crate) const fn available(&self) -> u64 { - self.granted_bytes.saturating_sub(self.sent_bytes) - } - - /// Whether `bytes` more may be sent without exceeding the current - /// window. - pub(crate) const fn can_admit(&self, bytes: u64) -> bool { - bytes <= self.available() - } - - /// Record that `bytes` were just sent (now counted against the - /// window until a matching `WINDOW_UPDATE` arrives). - pub(crate) const fn record_sent(&mut self, bytes: u64) { - self.sent_bytes = self.sent_bytes.saturating_add(bytes); - } - - /// Apply a `WINDOW_UPDATE { additional_window_bytes }` frame, - /// raising the ceiling. - pub(crate) const fn grant(&mut self, additional_window_bytes: u64) { - self.granted_bytes = self.granted_bytes.saturating_add(additional_window_bytes); - } -} - -#[cfg(test)] -mod tests { - use super::WindowTracker; - - #[test] - fn fresh_tracker_has_the_full_initial_window_available() { - let tracker = WindowTracker::new(1000); - assert_eq!(tracker.available(), 1000); - assert!(tracker.can_admit(1000)); - assert!(!tracker.can_admit(1001)); - } - - #[test] - fn sending_bytes_reduces_availability() { - let mut tracker = WindowTracker::new(1000); - tracker.record_sent(400); - assert_eq!(tracker.available(), 600); - assert!(tracker.can_admit(600)); - assert!(!tracker.can_admit(601)); - } - - #[test] - fn exhausting_the_window_admits_nothing_further() { - let mut tracker = WindowTracker::new(500); - tracker.record_sent(500); - assert_eq!(tracker.available(), 0); - assert!(!tracker.can_admit(1)); - assert!(tracker.can_admit(0)); - } - - #[test] - fn window_update_raises_the_ceiling() { - let mut tracker = WindowTracker::new(500); - tracker.record_sent(500); - assert_eq!(tracker.available(), 0); - tracker.grant(300); - assert_eq!(tracker.available(), 300); - assert!(tracker.can_admit(300)); - assert!(!tracker.can_admit(301)); - } - - #[test] - fn sent_bytes_never_underflow_available_below_zero() { - // Sending exactly up to the ceiling, never past it (can_admit is - // the caller's contract to honor), leaves available() at exactly - // zero rather than wrapping. - let mut tracker = WindowTracker::new(100); - tracker.record_sent(100); - assert_eq!(tracker.available(), 0); - } - - #[test] - fn multiple_grants_accumulate() { - let mut tracker = WindowTracker::new(0); - assert_eq!(tracker.available(), 0); - tracker.grant(100); - tracker.grant(50); - assert_eq!(tracker.available(), 150); - } -} diff --git a/crates/uffs-content/src/job/workflow.rs b/crates/uffs-content/src/job/workflow.rs deleted file mode 100644 index f17db37eb..000000000 --- a/crates/uffs-content/src/job/workflow.rs +++ /dev/null @@ -1,520 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Drives one job end to end: enumerate candidates, finalize the -//! manifest, stream framed content, and finalize the run summary. -//! -//! This is the real Coordinator workflow โ€” only the -//! [`CandidateSource`]/[`ContentSource`] it's given are swappable; see -//! those traits' docs for what "swappable" means today (a real vs. fake -//! backing). -//! -//! # Concurrent reads, concurrent drives, atomic per-candidate emission -//! -//! Candidates are read through a bounded pipeline, one per contiguous -//! `snapshot_lease_id` run (see `lease_runs`), sized by that lease's -//! own [`ReadConcurrency`] โ€” routed to that drive's own connection pool -//! by `reader_client::ContentReader` (see that module's doc comment). -//! `read_lease_run_pipelined` is a genuine sliding window, not a -//! fixed-size batch that waits for its slowest member before starting -//! the next one: as soon as any of the `concurrency` worker threads -//! finishes a candidate, it immediately claims the next not-yet-started -//! one from the same run, regardless of whether earlier candidates are -//! still in flight. This matters in practice โ€” a real-hardware run -//! mixing a handful of multi-gigabyte files into tens of thousands of -//! tiny ones showed the earlier fixed-batch design stalling *every* -//! connection in a batch for as long as its one large straggler took, -//! since the next batch could never start until the current one's -//! `std::thread::scope` join completed. The sliding window instead keeps -//! the other `concurrency - 1` connections working through subsequent -//! candidates for the whole time the straggler is still streaming. -//! -//! **Every lease run (drive) also runs concurrently with every other -//! one**, each on its own thread โ€” not one full drive at a time. A -//! real-hardware full-drive-set job showed this matters just as much as -//! within-drive concurrency: with drives processed strictly sequentially, -//! one slow HDD-backed lease grinding through a heavily-fragmented legacy -//! archive held up every *other* drive's candidates โ€” including a fast -//! SSD-backed lease sitting fully idle in queue โ€” for as long as the slow -//! one took, even though the two share no connection pool, no volume -//! handle, and no physical device. -//! -//! Running drives concurrently means frames from *different* candidates -//! (possibly on different drives) may now interleave on the wire โ€” but -//! never frames belonging to the *same* candidate, and this is exactly -//! what the protocol allows: `JOB_BEGIN.ordering` is fixed to -//! [`FrameOrdering::None`] ("no cross-file ordering contract"), and the -//! one consumer that groups frames back up (`crate::serve::stream::Grouped`) -//! keys purely on each frame's own `candidate_id`, with no assumption -//! about which `candidate_id` shows up next โ€” it only requires that one -//! candidate's `FILE_BEGIN`, its `CONTENT_CHUNK`s, and its -//! `FILE_END`/`FILE_FAILED` never get split apart by another candidate's -//! frames. That per-candidate atomicity is what `EmitState`'s mutex -//! enforces: every lease run's own coordinator thread reads through -//! `read_lease_run_pipelined`'s existing per-drive reorder map exactly as -//! before (so within one drive, candidates are still handed back to -//! `on_ready` in strict order), but the actual *emission* step โ€” assembling -//! and writing one candidate's whole frame group, and updating -//! `frame_sequence`/`counters`/`failure_log` to match โ€” now happens under -//! a shared lock so two drives' coordinator threads can never do it at -//! the same time. The lock is held only for that short, in-memory -//! assembly-and-write step, never for the (comparatively slow, I/O-bound) -//! disk read that produces a candidate's content, so the actual -//! parallelism this exists to unlock is untouched by the lock's presence. -//! -//! The reorder map's size โ€” and therefore how far the sliding window can -//! run ahead of a straggler โ€” is self-bounding to a small constant -//! multiple of `concurrency`, never the run's total length: candidates -//! are dispatched to workers through an input channel bounded to -//! `concurrency` slots, and completed results flow back through an -//! output channel of the same bound, so a worker that finishes early -//! blocks on its next claim (or its result send) once the pipeline is -//! full, rather than racing arbitrarily far ahead and buffering the rest -//! of the job's content in memory behind one slow file. -//! -//! A run never spans two different `snapshot_lease_id`s โ€” candidates -//! are collected one root/drive at a time (see `run_job`'s enumeration -//! loop), so they already arrive grouped contiguously by drive, and -//! `lease_runs` splits at each lease boundary rather than letting one -//! drive's run absorb another's candidates under the wrong concurrency -//! setting. This is what makes an HDD-backed lease's concurrency-`1` -//! setting actually mean "read every candidate on *this drive* strictly -//! one at a time, in the order they were enumerated" โ€” the same order -//! the MFT (and therefore, roughly, on-disk position) produced them -//! in โ€” rather than being diluted by whatever other drives happen to be -//! in the same job; it says nothing about ordering relative to any -//! *other* drive's candidates, which is exactly the freedom running -//! drives concurrently needs. -//! -//! # Why `emit_frame` is a callback, not a returned `Vec` -//! -//! Earlier revisions of this function collected every emitted frame into -//! one `Vec>` and returned it once the whole job finished โ€” for a -//! job matching many/large files, that meant peak memory proportional to -//! the job's *entire* logical content, held before a single byte reached -//! any consumer. Emitting each frame through a caller-supplied callback as -//! soon as it's produced removes that ceiling: a caller that wants the old -//! all-in-memory behavior (this crate's own tests, `self_test`) can still -//! collect into a `Vec` via a trivial closure, while the real production -//! caller (`crate::serve::stream`) forwards frames onto a bounded channel -//! and paces them out under backpressure, so memory stays bounded near the -//! send-window size rather than the job size โ€” see that module's own doc -//! comment for the consumer side of this. - -use std::collections::HashMap; -use std::io; -use std::path::Path; - -use uffs_content_protocol::codec::{Digest, digest}; -use uffs_content_protocol::frame::{ - ContentSemantics, DigestAlgorithm, FrameEnvelope, FrameOrdering, FrameType, JobBegin, JobEnd, - JobStatus, PROTOCOL_VERSION, -}; -use uffs_content_protocol::manifest::AuthorizationMode; - -use super::candidate_source::{CandidateEntry, CandidateSource}; -use super::content_source::ContentSource; -use super::intake::JobRequest; -use super::manifest_builder::build_manifest; - -mod emit; -mod pipeline; -use crate::run::{FailureLogWriter, RunCounters, RunSummary}; - -/// One `CONTENT_CHUNK`'s maximum payload size for a job run. -/// -/// Real-hardware benchmarking found `uffs-content-reader`'s -/// `read_logical` (`crates/uffs-content-reader/src/reader/logical.rs`) -/// does a full `CreateFileW`+`OpenFileById`+`GetFileSizeEx`+`ReadFile`+ -/// close-both-handles cycle on *every single* `ReadRequest` โ€” i.e. once -/// per chunk, with no handle caching across a file's sequential reads. -/// At the previous `64 * 1024` default, a 2.76 GB file needed roughly -/// 42,000 of those full open/close cycles; per-`OpenFileById` cost -/// against a VSS snapshot device varied wildly file to file in a way -/// that didn't correlate with file size, which is exactly what you'd -/// expect from open/close overhead rather than genuine streaming -/// throughput. `1 MiB` cuts that count 16x with no protocol risk โ€” it -/// stays far under `uffs_content_reader_protocol::MAX_RESPONSE_PAYLOAD_BYTES` -/// (64 MiB), and `serve::pipe_io::MAX_MESSAGE_BYTES` is derived from -/// this constant, so it grows with it automatically. -/// -/// This does not fix the underlying per-chunk open/close cost, only -/// reduces how often it's paid; caching the open handle across a -/// candidate's whole read (a bigger, separate change) is the deeper fix. -pub const DEFAULT_MAX_CHUNK_BYTES: u32 = 1024 * 1024; - -/// Per-drive (per `snapshot_lease_id`) content-read concurrency. -/// -/// An HDD-backed lease should read its candidates one at a time, in the -/// order [`CandidateSource::enumerate`] returned them: candidates come -/// off the MFT roughly in on-disk order, so reading them strictly in -/// sequence approximates sequential disk access; racing several -/// concurrent reads against the same spinning disk instead scatters its -/// head across every in-flight read's location โ€” pure seek-time waste. -/// An NVMe/SSD-backed lease has no seek penalty to protect and benefits -/// from many candidates in flight at once (see `reader_client`'s -/// `ConnectionPool`, which is what actually backs this concurrency on -/// the wire โ€” this type only controls how many threads -/// [`run_job`] fans a given lease's reads out to). -/// -/// `snapshot_lease_id == 0` (never a real Broker-assigned lease id โ€” see -/// [`CandidateEntry::snapshot_lease_id`]) always falls through to -/// `default`, which is what the cross-platform fake/test harness (no -/// drive-type concept at all) relies on via [`Self::flat`]. -#[derive(Debug, Clone)] -pub struct ReadConcurrency { - /// Concurrency overrides, keyed by `snapshot_lease_id`. - per_lease: HashMap, - /// Concurrency for any lease not present in `per_lease`. - default: usize, -} - -impl ReadConcurrency { - /// A single flat concurrency for every lease โ€” for callers with no - /// per-drive concurrency to report (the cross-platform fake/test - /// harness). `1` gives the fully-sequential, deterministic-order - /// behavior some tests rely on. - #[must_use] - pub fn flat(concurrency: usize) -> Self { - Self { - per_lease: HashMap::new(), - default: concurrency.max(1), - } - } - - /// Start from `default` (used for any lease [`Self::set`] hasn't - /// overridden) with no per-lease overrides yet. - #[must_use] - pub fn new(default: usize) -> Self { - Self { - per_lease: HashMap::new(), - default: default.max(1), - } - } - - /// Override `lease_id`'s own concurrency. - pub fn set(&mut self, lease_id: u64, concurrency: usize) { - self.per_lease.insert(lease_id, concurrency.max(1)); - } - - /// The concurrency to use for a candidate from `lease_id`. - fn for_lease(&self, lease_id: u64) -> usize { - self.per_lease - .get(&lease_id) - .copied() - .unwrap_or(self.default) - } -} - -/// Everything one completed job produced, aside from the frames -/// themselves (see the module doc for why those are emitted through a -/// callback instead of collected here). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JobOutcome { - /// Job identifier assigned to this run. - pub job_id: [u8; 16], - /// The finalized manifest's encoded bytes. - pub manifest_bytes: Vec, - /// The finalized run summary. - pub run_summary: RunSummary, -} - -/// Run one job: enumerate every one of `request.roots` via -/// `candidate_source`. -/// -/// Finalize a manifest, stream every candidate's content via -/// `content_source`, and finalize the run's summary/failure log under -/// `run_dir`. -/// -/// Every encoded frame (`JOB_BEGIN`, then per-candidate frames, then -/// `JOB_END`) is passed to `emit_frame` in emission order as soon as it -/// exists โ€” see the module doc comment. -/// -/// `read_concurrency` is how many candidates are read concurrently per -/// batch, per drive (see the module doc's "Concurrent reads, sequential -/// emission" section and [`ReadConcurrency`]'s own doc comment for why -/// this varies by drive rather than being one flat number). Pass -/// [`ReadConcurrency::flat`] for the fully-sequential, deterministic- -/// order behavior tests rely on, or a per-lease-tuned one built from -/// each leased drive's actual `uffs_mft::platform::DriveType` (see -/// `super::vss_job::run_vss_job`, the real caller) โ€” this function has -/// no way to know drive types itself, since drive leasing happens in -/// the caller. -/// -/// # Errors -/// Returns an [`io::Error`] for any filesystem failure enumerating -/// candidates, writing the failure log, or finalizing the summary, or -/// propagated from `emit_frame` itself (e.g. a downstream transport -/// failure). A per-candidate content-read failure is *not* an error -/// return โ€” it's recorded as a `FAILED_RETRYABLE` outcome for that -/// candidate instead (a `FILE_FAILED` frame plus a `FailureRecord`). -#[expect( - clippy::too_many_arguments, - reason = "snapshot_id/snapshot_created_at are real VSS provenance the caller (run_vss_job) \ - already holds from its lease response; the fake/test callers pass empty/zero. \ - Bundling them into a struct purely to satisfy this lint would add indirection \ - for two fields that always travel together and change meaning together." -)] -pub fn run_job( - request: &JobRequest, - candidate_source: &dyn CandidateSource, - content_source: &dyn ContentSource, - run_dir: &Path, - read_concurrency: &ReadConcurrency, - snapshot_id: &[u8], - snapshot_created_at: i64, - mut emit_frame: F, -) -> io::Result -where - F: FnMut(Vec) -> io::Result<()> + Send, -{ - let job_id = *uuid::Uuid::new_v4().as_bytes(); - let source_id = source_id_bytes(&request.source_id); - // No query filtering is wired up yet (see `JobRequest` docs) โ€” every - // job is equivalent to a `"*"` query, so its digest is fixed. - let query_digest = digest(b"*"); - - tracing::info!( - job_id = %uuid::Uuid::from_bytes(job_id), - root_count = request.roots.len(), - "job: starting candidate enumeration" - ); - let entries = enumerate_all_roots_concurrently(candidate_source, &request.roots)?; - let candidate_count = len_as_u64(entries.len()); - tracing::info!( - candidate_count, - "job: enumeration complete, building manifest" - ); - - let built = build_manifest(job_id, source_id, query_digest, &entries) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; - - let mut frame_sequence: u64 = 0; - - let job_begin = JobBegin { - job_id, - source_id, - snapshot_id: snapshot_id.to_vec(), - snapshot_created_at, - manifest_digest: built.manifest_digest, - candidate_count, - authorization_mode: AuthorizationMode::AdminExport, - ordering: FrameOrdering::None, - content_semantics: ContentSemantics::UnnamedLogicalStream, - digest_algorithm: DigestAlgorithm::Blake3, - max_chunk_bytes: DEFAULT_MAX_CHUNK_BYTES, - max_content_delivery_bytes: request.max_content_delivery_bytes, - }; - emit_frame(encode_frame( - job_id, - &mut frame_sequence, - FrameType::JobBegin, - &job_begin.encode(), - ))?; - - let mut counters = RunCounters::new(candidate_count); - let run_id = uuid::Uuid::from_bytes(job_id).to_string(); - let failures_path = run_dir.join(format!("run-{run_id}.failures.jsonl")); - let mut failure_log = FailureLogWriter::open(&failures_path)?; - - let candidates: Vec<(&CandidateEntry, u64)> = entries - .iter() - .zip(built.candidate_ids.iter().copied()) - .collect(); - emit::read_and_emit_all_candidates( - &candidates, - read_concurrency, - content_source, - request.max_content_delivery_bytes, - job_id, - &mut counters, - &mut failure_log, - &mut frame_sequence, - &mut emit_frame, - )?; - drop(failure_log); - - tracing::info!( - succeeded = counters.succeeded_count, - failed_retryable = counters.failed_retryable_count, - failed_terminal = counters.failed_terminal_count, - deferred_manual = counters.deferred_manual_count, - "job: content reads complete, finalizing" - ); - emit_job_end( - &counters, - built.manifest_digest, - candidate_count, - &failures_path, - job_id, - &mut frame_sequence, - &mut emit_frame, - )?; - - let now_ms = unix_ms_now(); - let summary_path = run_dir.join(format!("run-{run_id}.summary.json")); - let run_summary = counters - .finalize(run_id, now_ms, now_ms) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; - run_summary.finalize_to_disk(&summary_path)?; - - Ok(JobOutcome { - job_id, - manifest_bytes: built.bytes, - run_summary, - }) -} - -/// Enumerate every root in `roots` concurrently (one thread per root) and -/// concatenate the results back in `roots`' own order. -/// -/// Real-hardware benchmarking found this step strictly sequential โ€” -/// root-by-root, one full search-and-collect cycle blocking the next โ€” -/// even though each [`CandidateSource::enumerate`] call opens its own -/// independent connection to the daemon (see -/// `VssCandidateSource::enumerate`) and shares no mutable state with any -/// other call. A two-root job showed ~15s + ~13s back to back (~28s -/// total) that this reduces to ~max(15s, 13s) by running both searches -/// at once โ€” and the effect compounds with every additional root. -/// -/// # Errors -/// Propagates the first error from any root's [`CandidateSource::enumerate`] -/// call, in `roots` order (matching the sequential loop this replaces). -#[expect( - clippy::needless_collect, - reason = "the intermediate `handles` collect is the whole point: every root's \ - scope.spawn must happen before any handle is joined, or this degenerates \ - back into spawn-then-immediately-join-one-at-a-time -- exactly the \ - sequential behavior this function exists to replace" -)] -fn enumerate_all_roots_concurrently( - candidate_source: &dyn CandidateSource, - roots: &[std::path::PathBuf], -) -> io::Result> { - let results: Vec>> = std::thread::scope(|scope| { - let handles: Vec<_> = roots - .iter() - .map(|root| scope.spawn(move || candidate_source.enumerate(root))) - .collect(); - handles - .into_iter() - .map(|handle| { - handle.join().unwrap_or_else(|panic_payload| { - Err(io::Error::other(format!( - "candidate enumeration thread panicked: {panic_payload:?}" - ))) - }) - }) - .collect() - }); - - let mut entries = Vec::new(); - for result in results { - entries.extend(result?); - } - Ok(entries) -} - -/// Wrap `payload` in a `FrameEnvelope` for `job_id`, assigning and -/// advancing the next `frame_sequence`. -fn encode_frame( - job_id: [u8; 16], - frame_sequence: &mut u64, - frame_type: FrameType, - payload: &[u8], -) -> Vec { - let envelope = FrameEnvelope { - protocol_version: PROTOCOL_VERSION, - frame_type, - flags: 0, - job_id, - frame_sequence: *frame_sequence, - }; - *frame_sequence += 1; - envelope.encode(payload) -} - -/// Build and emit `JOB_END`: `job_status` is derived from `counters`, -/// `failure_bucket_id`/`outcome_ledger_digest` from the failure log file -/// at `failures_path`. -fn emit_job_end( - counters: &RunCounters, - manifest_digest: Digest, - candidate_count: u64, - failures_path: &Path, - job_id: [u8; 16], - frame_sequence: &mut u64, - emit_frame: &mut dyn FnMut(Vec) -> io::Result<()>, -) -> io::Result<()> { - let job_status = if counters.failed_retryable_count == 0 - && counters.failed_terminal_count == 0 - && counters.deferred_manual_count == 0 - { - JobStatus::Completed - } else { - JobStatus::CompletedWithFailures - }; - - let failure_log_bytes = std::fs::read(failures_path).unwrap_or_default(); - let failure_bucket_id = failures_path - .file_name() - .map(|name| name.to_string_lossy().into_owned().into_bytes()) - .unwrap_or_default(); - let job_end = JobEnd { - candidate_count, - succeeded_count: counters.succeeded_count, - failed_retryable_count: counters.failed_retryable_count, - failed_terminal_count: counters.failed_terminal_count, - deferred_manual_count: counters.deferred_manual_count, - // No FILE_ACK loop is modeled by this fake-reader harness yet - // (UFI.2 scheduler work) โ€” every success is treated as - // immediately acknowledged. - acknowledged_success_count: counters.succeeded_count, - logical_bytes_succeeded: counters.logical_bytes_succeeded, - failure_bucket_id, - manifest_digest, - outcome_ledger_digest: digest(&failure_log_bytes), - job_status, - }; - let job_end_bytes = job_end - .encode() - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; - emit_frame(encode_frame( - job_id, - frame_sequence, - FrameType::JobEnd, - &job_end_bytes, - )) -} - -/// Deterministically derives a manifest `source_id` from an arbitrary -/// caller-supplied string, truncating a BLAKE3 digest to 16 bytes (this -/// avoids requiring the `uuid` crate's `v5` feature workspace-wide for -/// what both docs and every existing user only ever treat as an opaque -/// 16-byte identifier). -fn source_id_bytes(source_id: &str) -> [u8; 16] { - let full: Digest = digest(source_id.as_bytes()); - let mut out = [0_u8; 16]; - if let Some(prefix) = full.get(..16) { - out.copy_from_slice(prefix); - } - out -} - -/// Converts a byte length to `u64`, saturating instead of panicking (this -/// crate never handles files anywhere near `u64::MAX` bytes long, so -/// saturation is unobservable in practice and keeps every call site -/// infallible). -fn len_as_u64(len: usize) -> u64 { - u64::try_from(len).unwrap_or(u64::MAX) -} - -/// Current wall-clock time, Unix milliseconds, saturating to `0` if the -/// clock is somehow set before the epoch. -fn unix_ms_now() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |duration| { - i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) - }) -} diff --git a/crates/uffs-content/src/job/workflow/emit.rs b/crates/uffs-content/src/job/workflow/emit.rs deleted file mode 100644 index d8c393b07..000000000 --- a/crates/uffs-content/src/job/workflow/emit.rs +++ /dev/null @@ -1,379 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Per-candidate, per-lease-run content emission โ€” split out of -//! `workflow` itself purely to stay under this workspace's file-size -//! policy (mirroring `workflow`'s own existing `pipeline` split for the -//! same reason); every item here is still conceptually part of -//! `run_job`'s single content-reading step, just extracted so that one -//! file doesn't hold both the top-level job orchestration and the -//! concurrency machinery beneath it. -//! -//! See `super`'s "Concurrent reads, concurrent drives, atomic -//! per-candidate emission" doc section for why [`EmitState`]'s mutex -//! exists and what atomicity it's protecting. - -use std::io; - -use uffs_content_protocol::error::ErrorCode; -use uffs_content_protocol::frame::{ - FailedOutcome, FailureStage, FileBegin, FileEnd, FileFailed, FrameType, ReadMode, RetryClass, -}; -use uffs_content_protocol::path_encoding::WindowsPath; - -use super::pipeline::{CandidateContent, lease_runs, read_lease_run_pipelined}; -use super::{DEFAULT_MAX_CHUNK_BYTES, ReadConcurrency, encode_frame}; -use crate::job::candidate_source::CandidateEntry; -use crate::job::content_source::ContentSource; -use crate::run::{FailureLogWriter, FailureRecord, RunCounters}; - -/// Read and emit every candidate's content, one [`read_lease_run_pipelined`] -/// call per contiguous [`lease_runs`] group. Extracted from -/// `super::run_job` itself so that function stays under the workspace's -/// `too_many_lines` budget โ€” every parameter here is `run_job`'s own -/// local state, threaded through unchanged. -/// -/// # Errors -/// Propagates the first error from enumerating a lease run's content or -/// from `emit_frame` itself, exactly as `run_job`'s own doc comment -/// describes. -#[expect( - clippy::too_many_arguments, - reason = "the alternative is a bespoke context struct bundling counters/failure_log/ \ - frame_sequence/emit_frame purely to satisfy this lint, for a private helper \ - extracted from run_job with exactly one call site; not worth the indirection" -)] -#[expect( - clippy::significant_drop_tightening, - reason = "the lock IS the intended critical section, per lease-run closure below: it must \ - stay held for one candidate's whole emission (assembling + writing its frames, \ - updating counters/failure_log/frame_sequence together), not something to shrink \ - -- that's exactly what keeps two drives' candidates from interleaving on the wire" -)] -pub(super) fn read_and_emit_all_candidates( - candidates: &[(&CandidateEntry, u64)], - read_concurrency: &ReadConcurrency, - content_source: &dyn ContentSource, - max_content_delivery_bytes: Option, - job_id: [u8; 16], - counters: &mut RunCounters, - failure_log: &mut FailureLogWriter, - frame_sequence: &mut u64, - emit_frame: &mut F, -) -> io::Result<()> -where - F: FnMut(Vec) -> io::Result<()> + Send, -{ - let total_candidates = candidates.len(); - let run_started_at = std::time::Instant::now(); - let emit_state = std::sync::Mutex::new(EmitState { - counters, - failure_log, - frame_sequence, - emit_frame, - emitted_count: 0, - last_progress_log_at: run_started_at, - last_progress_log_bytes: 0, - }); - - let runs = lease_runs(candidates); - let results: Vec> = std::thread::scope(|scope| { - let handles: Vec<_> = runs - .iter() - .map(|run| { - let emit_state_ref = &emit_state; - scope.spawn(move || { - let Some(&(first_entry, _)) = run.first() else { - return Ok(()); - }; - let concurrency = read_concurrency.for_lease(first_entry.snapshot_lease_id); - read_lease_run_pipelined( - run, - concurrency, - content_source, - DEFAULT_MAX_CHUNK_BYTES, - max_content_delivery_bytes, - |index, read_result| { - let Some(&(entry, candidate_id)) = run.get(index) else { - return Ok(()); - }; - let mut guard = emit_state_ref - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - // Explicit reborrow: the guard's `DerefMut` hides - // field disjointness from the borrow checker, so - // every subsequent access goes through this plain - // `&mut EmitState` instead of `guard` directly. - let state = &mut *guard; - let result = emit_candidate( - entry, - candidate_id, - read_result, - state.counters, - state.failure_log, - job_id, - state.frame_sequence, - state.emit_frame, - ); - state.emitted_count += 1; - log_progress_if_due( - state.emitted_count, - total_candidates, - state.counters, - run_started_at, - &mut state.last_progress_log_at, - &mut state.last_progress_log_bytes, - ); - result - }, - ) - }) - }) - .collect(); - - handles - .into_iter() - .map(|handle| { - handle.join().unwrap_or_else(|panic_payload| { - Err(io::Error::other(format!( - "lease-run thread panicked: {panic_payload:?}" - ))) - }) - }) - .collect() - }); - - for result in results { - result?; - } - Ok(()) -} - -/// Bundles everything one candidate's emission touches โ€” `counters`, -/// `failure_log`, `frame_sequence`, the `emit_frame` callback itself, -/// and the progress-heartbeat state โ€” behind one lock, so concurrently- -/// running lease runs (drives) never interleave two candidates' frame -/// groups on the wire; see `super`'s "Concurrent reads, concurrent -/// drives, atomic per-candidate emission" doc section for why that's -/// exactly the atomicity the protocol requires, no more and no less. -struct EmitState<'a, F> { - /// Run-wide success/failure/byte counters, updated once per emitted - /// candidate. - counters: &'a mut RunCounters, - /// Append-only failure-record log, written to on a failed candidate. - failure_log: &'a mut FailureLogWriter, - /// Next frame's sequence number, advanced by every frame this - /// candidate emits. - frame_sequence: &'a mut u64, - /// The caller-supplied frame sink. - emit_frame: &'a mut F, - /// Candidates emitted so far, across every lease run combined. - emitted_count: usize, - /// Wall-clock time of the last progress heartbeat. - last_progress_log_at: std::time::Instant, - /// `counters.logical_bytes_succeeded` as of the last heartbeat. - last_progress_log_bytes: u64, -} - -/// How often [`read_and_emit_all_candidates`] logs a progress heartbeat, -/// at minimum โ€” never less often than this many wall-clock seconds -/// apart, regardless of candidate count or throughput. Chosen so a job -/// that's silently grinding for a long time (whether genuinely slow or -/// stuck on one candidate โ€” see `pipeline::read_one_candidate`'s own -/// per-candidate stall warning) is never silent for more than about this -/// long between updates. -const PROGRESS_LOG_MIN_INTERVAL: core::time::Duration = core::time::Duration::from_secs(10); - -/// Also log a heartbeat every this many candidates, even if -/// [`PROGRESS_LOG_MIN_INTERVAL`] hasn't elapsed โ€” keeps a very fast run -/// (thousands of tiny files) from having its own progress signal -/// throttled down to nothing. -const PROGRESS_LOG_CANDIDATE_STRIDE: usize = 1000; - -/// Log an `INFO`-level progress line if either [`PROGRESS_LOG_MIN_INTERVAL`] -/// has elapsed since the last one or `emitted_count` just crossed a -/// [`PROGRESS_LOG_CANDIDATE_STRIDE`] boundary โ€” see `super`'s "Concurrent -/// reads, concurrent drives, atomic per-candidate emission" doc section -/// for why total silence during content reading was a real problem this -/// closes. -/// -/// Also reports `mib_per_sec_since_last_heartbeat` and -/// `mib_per_sec_since_job_start`: `logical_bytes_succeeded` only -/// advances as candidates are actually *emitted* โ€” i.e. bytes handed to -/// `emit_frame`, the real wire write in `--serve` mode โ€” so both figures -/// are a direct measurement of consumer-facing pipe throughput, not an -/// internal per-connection or per-drive read rate. Scan -/// `mib_per_sec_since_last_heartbeat` across a run's log for min/max; -/// the last line's `mib_per_sec_since_job_start` is the run's overall -/// average. -#[expect( - clippy::cast_precision_loss, - reason = "diagnostic-only throughput figures for a log line, not computed against further \ - โ€” same posture as uffs-content's own benchmark report (self_test.rs)" -)] -#[expect( - clippy::float_arithmetic, - reason = "diagnostic-only throughput ratios for a log line, matching self_test.rs's \ - existing benchmark-report precedent" -)] -fn log_progress_if_due( - emitted_count: usize, - total_candidates: usize, - counters: &RunCounters, - run_started_at: std::time::Instant, - last_progress_log_at: &mut std::time::Instant, - last_progress_log_bytes: &mut u64, -) { - let due_by_time = last_progress_log_at.elapsed() >= PROGRESS_LOG_MIN_INTERVAL; - let due_by_count = emitted_count.is_multiple_of(PROGRESS_LOG_CANDIDATE_STRIDE) - || emitted_count == total_candidates; - if !due_by_time && !due_by_count { - return; - } - let interval_secs = last_progress_log_at.elapsed().as_secs_f64(); - let interval_bytes = counters - .logical_bytes_succeeded - .saturating_sub(*last_progress_log_bytes); - let mib_per_sec_since_last_heartbeat = if interval_secs > 0.0_f64 { - (interval_bytes as f64 / (1_024.0_f64 * 1_024.0_f64)) / interval_secs - } else { - 0.0_f64 - }; - let overall_secs = run_started_at.elapsed().as_secs_f64(); - let mib_per_sec_since_job_start = if overall_secs > 0.0_f64 { - (counters.logical_bytes_succeeded as f64 / (1_024.0_f64 * 1_024.0_f64)) / overall_secs - } else { - 0.0_f64 - }; - - *last_progress_log_at = std::time::Instant::now(); - *last_progress_log_bytes = counters.logical_bytes_succeeded; - tracing::info!( - emitted_count, - total_candidates, - succeeded = counters.succeeded_count, - failed_retryable = counters.failed_retryable_count, - failed_terminal = counters.failed_terminal_count, - logical_bytes_succeeded = counters.logical_bytes_succeeded, - mib_per_sec_since_last_heartbeat, - mib_per_sec_since_job_start, - "job: content read progress" - ); -} - -/// Emit one already-read candidate's `FILE_BEGIN`, its `CONTENT_CHUNK`s, -/// and its terminal frame (`FILE_END`/`FILE_FAILED`), in that order, on -/// the caller's own thread โ€” see `super`'s "Concurrent reads, concurrent -/// drives, atomic per-candidate emission" doc section for the atomicity -/// this step is part of. Updates `counters` and appends to `failure_log` -/// for a non-success outcome. -#[expect( - clippy::too_many_arguments, - reason = "the alternative is a bespoke context struct bundling job_id/frame_sequence/ \ - emit_frame purely to satisfy this lint, for a private helper with exactly one \ - call site; not worth the indirection" -)] -fn emit_candidate( - entry: &CandidateEntry, - candidate_id: u64, - content: CandidateContent, - counters: &mut RunCounters, - failure_log: &mut FailureLogWriter, - job_id: [u8; 16], - frame_sequence: &mut u64, - emit_frame: &mut dyn FnMut(Vec) -> io::Result<()>, -) -> io::Result<()> { - let path = WindowsPath::from_str_lossless(&entry.relative_path.to_string_lossy()); - - let file_begin = FileBegin { - candidate_id, - file_reference: entry.file_reference, - path, - logical_size: entry.logical_size, - mtime: entry.mtime_unix_ms, - read_mode: content.read_mode, - attempt_number: 1, - content_object_id: None, - }; - emit_frame(encode_frame( - job_id, - frame_sequence, - FrameType::FileBegin, - &file_begin.encode(), - ))?; - - let chunk_count = super::len_as_u64(content.chunks.len()); - for chunk in &content.chunks { - emit_frame(encode_frame( - job_id, - frame_sequence, - FrameType::ContentChunk, - &chunk.encode(), - ))?; - } - - let read_mode = content.read_mode; - match content.read_error { - None => { - // A metadata-only candidate never had real bytes read (see - // `pipeline::read_one_candidate`'s doc comment), so its - // digest is meaningless โ€” report `None`, matching the wire - // contract `ReadMode::MetadataOnly`'s own doc comment - // documents (design-doc's two-tier delivery-ceiling model). - let content_digest = if read_mode == ReadMode::MetadataOnly { - None - } else { - Some(content.digest) - }; - let file_end = FileEnd { - candidate_id, - total_logical_bytes: content.total_read, - content_digest, - read_mode, - chunk_count, - elapsed_ms: 0, - warning_flags: 0, - }; - emit_frame(encode_frame( - job_id, - frame_sequence, - FrameType::FileEnd, - &file_end.encode(), - ))?; - counters.record_succeeded(content.total_read); - } - Some(err) => { - let os_error_code = err.raw_os_error().map(i64::from); - let message = err.to_string(); - let file_failed = FileFailed { - candidate_id, - outcome: FailedOutcome::Retryable, - failure_stage: FailureStage::Read, - error_code: ErrorCode::ReadIoTransient, - os_error_code, - retry_class: RetryClass::RetryNewSnapshot, - bytes_emitted_before_failure: content.total_read, - message: message.clone(), - }; - emit_frame(encode_frame( - job_id, - frame_sequence, - FrameType::FileFailed, - &file_failed.encode(), - ))?; - counters.record_failed_retryable(); - failure_log.append(&FailureRecord::failed( - candidate_id, - FailedOutcome::Retryable, - FailureStage::Read, - ErrorCode::ReadIoTransient, - os_error_code, - RetryClass::RetryNewSnapshot, - content.total_read, - message, - ))?; - } - } - - Ok(()) -} diff --git a/crates/uffs-content/src/job/workflow/pipeline.rs b/crates/uffs-content/src/job/workflow/pipeline.rs deleted file mode 100644 index 114cfb2ff..000000000 --- a/crates/uffs-content/src/job/workflow/pipeline.rs +++ /dev/null @@ -1,520 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Bounded sliding-window content-read pipeline โ€” the concurrent-read -//! machinery behind [`super::run_job`], split into its own file so -//! `workflow.rs` itself stays under the workspace's file-size policy. -//! See that module's doc comment ("Concurrent reads, sequential -//! emission") for the full design rationale; everything here is -//! mechanism, not policy. - -use std::collections::HashMap; -use std::io; - -use crossbeam_channel::{Receiver, Sender}; -use uffs_content_protocol::codec::{Digest, IncrementalDigest}; -use uffs_content_protocol::frame::{ContentChunk, ReadMode}; - -use crate::job::candidate_source::CandidateEntry; -use crate::job::content_source::ContentSource; - -/// A declared `logical_size` above this is almost certainly corrupted -/// MFT metadata, not a genuine file โ€” used only to log a warning early -/// (see [`read_one_candidate`]), never to reject or cap the read itself, -/// since a real use case (e.g. a VM image or disk image export) can -/// legitimately exceed this. -const IMPLAUSIBLE_LOGICAL_SIZE_BYTES: u64 = 1024 * 1024 * 1024 * 1024; // 1 TiB - -/// A declared `logical_size` at or above this is worth announcing -/// *before* reading it (see [`read_one_candidate`]) โ€” real hardware has -/// shown large/fragmented files reading dramatically slower toward -/// their end than a small-file baseline would predict, without any -/// single candidate ever crossing [`STALL_WARNING_INTERVAL`] on its own. -/// Logging every such candidate up front means a run's throughput dip -/// can always be attributed to a named file, not just inferred after -/// the fact from progress-heartbeat arithmetic. -const NOTABLY_LARGE_LOGICAL_SIZE_BYTES: u64 = 50 * 1024 * 1024; // 50 MiB - -/// How often [`read_one_candidate`] re-warns about a single candidate -/// still being read, once it's been in progress this long. Shorter than -/// might seem necessary on purpose: real hardware has shown a single -/// large/fragmented file's read throughput can collapse well before 30s -/// of elapsed time on that one candidate, so a shorter interval catches -/// a slow candidate sooner without waiting for it to look fully stuck. -const STALL_WARNING_INTERVAL: core::time::Duration = core::time::Duration::from_secs(10); - -/// Split `candidates` into contiguous same-`snapshot_lease_id` runs for -/// [`read_lease_run_pipelined`] โ€” unlike a fixed batch size, a run is -/// never capped: its own concurrency (looked up once, from its first -/// candidate) instead bounds how many of its worker threads run at -/// once, not how many candidates it may contain. See the parent -/// module's "Concurrent reads, sequential emission" doc section for why -/// staying within one lease per run matters. -/// -/// Candidates already arrive grouped contiguously by lease (`run_job`'s -/// enumeration loop appends one root/drive at a time), so a single -/// linear scan suffices โ€” no need to look ahead past the current run. -pub(super) fn lease_runs<'entries>( - candidates: &'entries [(&'entries CandidateEntry, u64)], -) -> Vec<&'entries [(&'entries CandidateEntry, u64)]> { - let mut runs = Vec::new(); - let mut start = 0_usize; - while start < candidates.len() { - let Some((first_entry, _)) = candidates.get(start) else { - break; - }; - let lease_id = first_entry.snapshot_lease_id; - let Some(tail) = candidates.get(start..) else { - break; - }; - let end = tail - .iter() - .position(|(entry, _)| entry.snapshot_lease_id != lease_id) - .map_or(candidates.len(), |offset| start + offset); - let Some(run) = candidates.get(start..end) else { - break; - }; - runs.push(run); - start = end; - } - runs -} - -/// One candidate's content, fully read into memory by -/// [`read_lease_run_pipelined`]/[`read_one_candidate`] and consumed by -/// the parent module's `emit_candidate`. Bounded to exactly one -/// candidate's content per instance โ€” never a whole run or job โ€” since -/// the pipeline's own bounded channels cap how many of these exist in -/// memory at once (see the parent module doc's "Concurrent reads, -/// sequential emission" section). -pub(super) struct CandidateContent { - /// Every `CONTENT_CHUNK` this candidate's content produced, in order. - /// Empty when `read_mode == MetadataOnly`. - pub(super) chunks: Vec, - /// Sum of every chunk's payload length. `0` when `read_mode == - /// MetadataOnly`. - pub(super) total_read: u64, - /// BLAKE3 digest over every chunk's payload, in order. Only - /// meaningful when `read_mode != MetadataOnly` โ€” the parent module's - /// `emit_candidate` reports `content_digest: None` on `FILE_END` for - /// a metadata-only candidate regardless of this value. - pub(super) digest: Digest, - /// Set only if a read failed partway through; `None` means every - /// byte up to `entry.logical_size` was read successfully (or the - /// read was skipped entirely because `read_mode == MetadataOnly`). - pub(super) read_error: Option, - /// Whether this candidate's body was actually streamed - /// (`LogicalSnapshot`) or skipped because `entry.logical_size` - /// exceeded the job's `max_content_delivery_bytes` ceiling - /// (`MetadataOnly`) โ€” see [`read_one_candidate`]'s doc comment. - pub(super) read_mode: ReadMode, -} - -/// Read every candidate in `run` through a bounded sliding-window -/// pipeline of `concurrency` worker threads, invoking `on_ready(index, -/// content)` โ€” `index` into `run` โ€” strictly in order as each -/// candidate's turn comes up, regardless of the order its read actually -/// completed in. See the parent module doc's "Concurrent reads, -/// sequential emission" section for the full rationale and the -/// memory-boundedness argument; in short: this is a genuine sliding -/// window (a worker immediately claims the next unclaimed candidate the -/// moment it finishes its current one), not a fixed-size batch that -/// waits for its slowest member before admitting more work. -/// -/// Three threads-of-control cooperate, all joined via `std::thread::scope` -/// before this function returns: -/// - one **feeder** thread sends candidate indices `0..run.len()`, in order, -/// into a bounded input channel (capacity `concurrency`) โ€” its `send` blocks -/// once that many are unclaimed, which is what keeps a fast run of tiny -/// candidates from letting workers race arbitrarily far ahead of a slow one; -/// - `concurrency` **worker** threads each loop: claim the next index from the -/// input channel, read that candidate, then push `(index, content)` to a -/// bounded output channel (same capacity) โ€” blocking there, not just on the -/// next input claim, if results are piling up faster than they can be -/// consumed; -/// - this function's own body (the **coordinator**, running on the caller's -/// thread inside the `scope` โ€” not a spawned thread) drains the output -/// channel into a small reorder map and calls `on_ready` for `0, 1, 2, ...` -/// in turn as each becomes available. -/// -/// A fourth, bounded **credit** channel keeps the whole pipeline's memory -/// bounded regardless of run length: the feeder must claim one credit -/// before sending each index, and the coordinator returns exactly one -/// credit every time an index resolves (whether `on_ready` was actually -/// called for it or not โ€” see below). Without this, a single slow -/// candidate holding up `next_expected` would not stop the *other* -/// workers from reading every remaining candidate in the run to -/// completion and piling the results into the reorder map unbounded โ€” -/// real hardware has shown this: one multi-GB candidate stalling -/// emission while workers kept finishing (and fully buffering) tens of -/// thousands of others behind it, well past what "a small constant -/// multiple of concurrency" should ever allow. The credit window is -/// generous relative to `concurrency` (workers should rarely feel it on -/// an ordinary run) but always finite. -/// -/// If `on_ready` itself returns an error (e.g. a downstream transport -/// failure), the coordinator stops calling it but keeps draining the -/// output channel to completion anyway โ€” never stopping early and -/// leaving a worker or the feeder blocked on a channel nobody is -/// servicing anymore โ€” and returns that first error once every -/// candidate has actually been read (wasting the now-moot remaining -/// reads in that rare case, in exchange for a pipeline that can never -/// deadlock on early termination). -/// -/// # Errors -/// Returns the first error `on_ready` produced, if any. -pub(super) fn read_lease_run_pipelined( - run: &[(&CandidateEntry, u64)], - concurrency: usize, - content_source: &dyn ContentSource, - max_chunk_bytes: u32, - max_content_delivery_bytes: Option, - mut on_ready: impl FnMut(usize, CandidateContent) -> io::Result<()>, -) -> io::Result<()> { - if run.is_empty() { - return Ok(()); - } - let worker_count = concurrency.max(1).min(run.len()); - // How many candidates may be claimed-but-not-yet-emitted at once โ€” - // see this function's own doc comment on the credit channel. A - // small multiple of worker_count gives workers slack to keep moving - // even while a handful of candidates ahead of the emission cursor - // are still being read, without letting the whole remainder of a - // huge run pile into memory behind one straggler. - let credit_window = worker_count.saturating_mul(4); - - let (input_tx, input_rx): (Sender, Receiver) = - crossbeam_channel::bounded(worker_count); - let (output_tx, output_rx): (Sender, Receiver) = - crossbeam_channel::bounded(worker_count); - let (credit_tx, credit_rx): (Sender<()>, Receiver<()>) = - crossbeam_channel::bounded(credit_window); - for _ in 0..credit_window { - // Never blocks: capacity is exactly credit_window and this sends - // exactly that many, once, before any thread below starts. - let _prefilled = credit_tx.try_send(()).ok(); - } - - std::thread::scope(|scope| { - scope.spawn(move || { - for index in 0..run.len() { - if credit_rx.recv().is_err() { - break; - } - if input_tx.send(index).is_err() { - break; - } - } - // Dropping input_tx here (end of scope) closes the channel - // once every index has been sent, so workers' `recv` loops - // end cleanly instead of blocking forever. - }); - - for _ in 0..worker_count { - let worker_input_rx = input_rx.clone(); - let worker_output_tx = output_tx.clone(); - scope.spawn(move || { - while let Ok(index) = worker_input_rx.recv() { - let Some(&(entry, candidate_id)) = run.get(index) else { - continue; - }; - let content = read_one_candidate_catch_panic( - entry, - candidate_id, - content_source, - max_chunk_bytes, - max_content_delivery_bytes, - ); - if worker_output_tx.send((index, content)).is_err() { - break; - } - } - }); - } - // This scope's own sender handle; every worker holds its own - // clone, so the channel only truly closes once all of them - // finish. - drop(output_tx); - - drain_pipelined_output(run.len(), &output_rx, &credit_tx, &mut on_ready) - }) -} - -/// One worker's completed read, tagged with its index into the -/// enclosing [`read_lease_run_pipelined`] call's `run` slice. -type IndexedContent = (usize, CandidateContent); - -/// The coordinator half of [`read_lease_run_pipelined`]: drain -/// `output_rx` into a reorder map and call `on_ready` for -/// `0..total_candidates` in turn as each becomes available. Extracted -/// so `read_lease_run_pipelined` itself stays under the workspace's -/// `too_many_lines` budget. -/// -/// Returns one credit to `credit_tx` every time an index resolves โ€” -/// whether `on_ready` was actually called for it or not (see this -/// function's own error-handling branch below) โ€” so the feeder in -/// [`read_lease_run_pipelined`] never blocks waiting for a credit that a -/// resolved-but-unemitted index should have released. This is the other -/// half of that function's credit-window backpressure; see its doc -/// comment for why the window exists at all. -/// -/// # Errors -/// Returns the first error `on_ready` produced, after draining every -/// remaining result (see [`read_lease_run_pipelined`]'s doc comment for -/// why finishing the drain, rather than stopping early, is what keeps -/// this deadlock-free). -fn drain_pipelined_output( - total_candidates: usize, - output_rx: &Receiver, - credit_tx: &Sender<()>, - on_ready: &mut dyn FnMut(usize, CandidateContent) -> io::Result<()>, -) -> io::Result<()> { - let mut next_expected = 0_usize; - let mut pending: HashMap = HashMap::new(); - let mut first_error: Option = None; - while next_expected < total_candidates { - if let Some(content) = pending.remove(&next_expected) { - if first_error.is_none() - && let Err(err) = on_ready(next_expected, content) - { - first_error = Some(err); - } - next_expected += 1; - // Best-effort: a disconnected credit channel just means the - // feeder already exited (e.g. it hit a send error on - // input_tx and gave up), not something this coordinator - // needs to react to. - let _credit_returned = credit_tx.send(()).ok(); - continue; - } - match output_rx.recv() { - Ok((index, content)) => { - pending.insert(index, content); - } - // Every worker finished without ever producing - // `next_expected` โ€” unreachable in practice (the feeder - // sends every index in `0..run.len()` and every worker - // processes whatever it claims), but fail safe rather than - // spin. - Err(_) => break, - } - } - first_error.map_or(Ok(()), Err) -} - -/// [`read_one_candidate`], guarded against a panic partway through: -/// `content_source` is a `&dyn ContentSource` trait object this crate -/// doesn't control every implementation of (see that trait's own doc -/// comment), so a third-party impl panicking must not take down this -/// candidate's entire worker thread โ€” and, transitively, every other -/// candidate's read still in flight on this same `thread::scope` (a -/// spawned thread's panic propagates when the scope joins it). A caught -/// panic is reported the same way an `io::Error` from a normal read -/// failure is: as a retryable `FILE_FAILED`, via `read_error`. -/// -/// Mirrors the panic-to-`read_error` conversion the earlier fixed-batch -/// design got for free from `JoinHandle::join()` returning `Err` on a -/// panicked thread โ€” this pipeline's workers are fire-and-forget -/// (`scope.spawn` without a retained handle), so that safety net has to -/// be reintroduced explicitly here instead. -fn read_one_candidate_catch_panic( - entry: &CandidateEntry, - candidate_id: u64, - content_source: &dyn ContentSource, - max_chunk_bytes: u32, - max_content_delivery_bytes: Option, -) -> CandidateContent { - let outcome = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| { - read_one_candidate( - entry, - candidate_id, - content_source, - max_chunk_bytes, - max_content_delivery_bytes, - ) - })); - outcome.unwrap_or_else(|panic_payload| CandidateContent { - chunks: Vec::new(), - total_read: 0, - digest: IncrementalDigest::new().finalize(), - read_error: Some(io::Error::other(format!( - "content-read thread panicked: {panic_payload:?}" - ))), - read_mode: ReadMode::LogicalSnapshot, - }) -} - -/// Read one candidate's content into memory, up to `entry.logical_size` -/// or the first read error โ€” or, if `entry.logical_size` exceeds -/// `max_content_delivery_bytes`, skip the read entirely and report -/// `read_mode: MetadataOnly` with no bytes read at all. This is the only -/// place the delivery ceiling is enforced; the query filters -/// (`ext`/`min_size`/etc.) that decide which files become candidates at -/// all are evaluated earlier, by the daemon โ€” this is a second, -/// independent gate on an already-matched candidate's body. -/// -/// Never touches `emit_frame`/`frame_sequence`/`counters`/`failure_log` -/// โ€” those stay single-threaded, touched only by the parent module's -/// `emit_candidate` afterward. -fn read_one_candidate( - entry: &CandidateEntry, - candidate_id: u64, - content_source: &dyn ContentSource, - max_chunk_bytes: u32, - max_content_delivery_bytes: Option, -) -> CandidateContent { - if max_content_delivery_bytes.is_some_and(|ceiling| entry.logical_size > ceiling) { - return CandidateContent { - chunks: Vec::new(), - total_read: 0, - digest: IncrementalDigest::new().finalize(), - read_error: None, - read_mode: ReadMode::MetadataOnly, - }; - } - log_candidate_size_if_notable(entry, candidate_id); - - let mut session = match content_source.begin_read(entry, candidate_id) { - Ok(session) => session, - Err(err) => { - tracing::warn!( - candidate_id, - path = %entry.relative_path.display(), - error = %err, - "content read: failed to begin read session" - ); - return CandidateContent { - chunks: Vec::new(), - total_read: 0, - digest: IncrementalDigest::new().finalize(), - read_error: Some(err), - read_mode: ReadMode::LogicalSnapshot, - }; - } - }; - - let mut hasher = IncrementalDigest::new(); - let mut offset = 0_u64; - let mut chunk_sequence = 0_u64; - let mut total_read = 0_u64; - let mut chunks = Vec::new(); - let mut read_error = None; - let read_started_at = std::time::Instant::now(); - let mut last_stall_warning_at = read_started_at; - - while offset < entry.logical_size { - warn_if_candidate_read_is_stalling( - entry, - candidate_id, - total_read, - read_started_at, - &mut last_stall_warning_at, - ); - match session.read_at(offset, max_chunk_bytes) { - Ok(bytes) if bytes.is_empty() => break, - Ok(bytes) => { - let read_len = super::len_as_u64(bytes.len()); - hasher.update(&bytes); - total_read += read_len; - chunks.push(ContentChunk { - candidate_id, - chunk_sequence, - logical_offset: offset, - logical_length: read_len, - payload: bytes, - }); - offset += read_len; - chunk_sequence += 1; - } - Err(err) => { - tracing::warn!( - candidate_id, - path = %entry.relative_path.display(), - offset, - error = %err, - "content read failed" - ); - read_error = Some(err); - break; - } - } - } - // `session` drops here, returning its pinned connection (if still - // framing-aligned) to the pool. - - CandidateContent { - chunks, - total_read, - digest: hasher.finalize(), - read_error, - read_mode: ReadMode::LogicalSnapshot, - } -} - -/// Logs a warning if `entry.logical_size` is implausibly large (likely -/// corrupted MFT metadata), or an informational note if it's merely -/// notably large (a real, sizeable file worth naming up front) โ€” -/// extracted from [`read_one_candidate`] purely to keep that function's -/// cognitive complexity down; see [`IMPLAUSIBLE_LOGICAL_SIZE_BYTES`] and -/// [`NOTABLY_LARGE_LOGICAL_SIZE_BYTES`]'s own doc comments for why both -/// exist. -fn log_candidate_size_if_notable(entry: &CandidateEntry, candidate_id: u64) { - if entry.logical_size > IMPLAUSIBLE_LOGICAL_SIZE_BYTES { - tracing::warn!( - candidate_id, - path = %entry.relative_path.display(), - declared_logical_size = entry.logical_size, - "content read: candidate's declared logical_size is implausibly large -- this \ - usually means corrupted/stale MFT metadata for this file (e.g. a reused FRS or a \ - race with the file being resized around snapshot time), not a genuinely huge file; \ - the read below is bounded by this declared size regardless, so a corrupted value \ - here can make one candidate consume a very long time and a lot of memory" - ); - } else if entry.logical_size >= NOTABLY_LARGE_LOGICAL_SIZE_BYTES { - // Announced up front, before any bytes are read, so a later - // throughput dip can always be traced back to a named candidate - // instead of only inferred from progress-heartbeat arithmetic - // after the fact -- real hardware has shown a large/fragmented - // file's read slow down well before it individually crosses - // STALL_WARNING_INTERVAL. - tracing::info!( - candidate_id, - path = %entry.relative_path.display(), - declared_logical_size = entry.logical_size, - "content read: about to read a notably large candidate" - ); - } -} - -/// Logs a warning if this candidate's read has been running for at -/// least [`STALL_WARNING_INTERVAL`] and hasn't already warned within the -/// last [`STALL_WARNING_INTERVAL`] โ€” extracted from -/// [`read_one_candidate`]'s read loop purely to keep that function's -/// cognitive complexity down; see its own doc comment for why this -/// exists (a corrupted `logical_size` or a stuck reader-side round trip -/// must be visible in the log, not silent for an hour). -fn warn_if_candidate_read_is_stalling( - entry: &CandidateEntry, - candidate_id: u64, - total_read: u64, - read_started_at: std::time::Instant, - last_stall_warning_at: &mut std::time::Instant, -) { - if read_started_at.elapsed() < STALL_WARNING_INTERVAL - || last_stall_warning_at.elapsed() < STALL_WARNING_INTERVAL - { - return; - } - *last_stall_warning_at = std::time::Instant::now(); - tracing::warn!( - candidate_id, - path = %entry.relative_path.display(), - declared_logical_size = entry.logical_size, - total_read, - elapsed_secs = read_started_at.elapsed().as_secs(), - "content read: this candidate is taking unusually long -- still in progress, not \ - necessarily hung, but if this repeats every ~30s indefinitely for the same \ - candidate_id, suspect corrupted logical_size (see the warning above, if any) or a \ - stuck reader-side round trip" - ); -} diff --git a/crates/uffs-content/src/lib.rs b/crates/uffs-content/src/lib.rs deleted file mode 100644 index ffb27c4cc..000000000 --- a/crates/uffs-content/src/lib.rs +++ /dev/null @@ -1,107 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -// `MetadataExt::file_index` (the Windows analogue of a Unix inode, -// used by `job::candidate_source::file_identity` for hard-link -// detection) is still gated behind this unstable std feature -// (rust-lang/rust#63010) with no stable alternative. Sound to rely on -// here: `rust-toolchain.toml` pins the exact same nightly across every -// environment (host, Windows, Linux) workspace-wide, not just for this -// crate, and `just toolchain-sync` re-validates every bump attempt -// against it before the pin moves. -#![cfg_attr(windows, feature(windows_by_handle))] - -//! UFFS Content Service โ€” library crate. -//! -//! `uffs-content` is the unprivileged content **coordinator**: read-mode -//! planning, candidate-manifest handling, and framed content streaming. -//! Any privileged VSS-snapshot/raw-extent capability is a narrow internal -//! helper this crate calls into (extending `uffs-broker`'s existing -//! pattern), never a whole-volume handle owned directly by this process. -//! See `docs/dev/architecture/uffs-content-stream-enterprise-design-review.md` -//! (local-only) for the rationale, and Docenta's -//! `uffs-ingest-protocol-v2-vss.md` for the settled manifest/frame -//! contract. The `[[bin]]` in this crate (`src/main.rs`) is a thin entry -//! point over this library, matching the `uffs-daemon` / `uffs_daemon` -//! split. -//! -//! # Status -//! -//! [`run`] (the ephemeral per-run manifest/failure-log/summary model), -//! [`job`] (job intake, candidate enumeration, manifest construction, -//! and protocol framing โ€” both the cross-platform `std::fs`-based -//! stand-ins from `uffs-ingest-implementation-plan.md` ยง9.5 and the -//! real VSS-snapshot/privileged-Reader-backed production path, -//! `job::vss_job::run_vss_job`), and the two-pipe transport server -//! (`serve`, Windows-only) that lets an external consumer actually reach -//! that pipeline, are all real. [`is_implemented`] tracks the -//! VSS-backed pipeline's platform availability, not this crate's own -//! workflow logic. - -extern crate alloc; - -pub mod job; -pub mod run; -// Two-pipe (data + command) transport server: the real entry point an -// external consumer (e.g. Docenta) connects to. Windows-only โ€” named -// pipes, and every job this serves is VSS-backed (`job::vss_job`, -// itself Windows-only). -#[cfg(windows)] -pub(crate) mod serve; - -// `uffs_version::handle_version!` is invoked from `main.rs` only. -// Dev-dependency used by `tests/support/plain_walk.rs` (the independent -// oracle for the E2E dir-walk parity harness), not by this crate's own -// unit tests. -#[cfg(test)] -use blake3 as _; -// Installed by `main.rs::init_tracing()` (the bin target), not used -// directly by this library crate. -#[cfg(windows)] -use tracing_subscriber as _; -use uffs_version as _; - -/// Whether the production, VSS-snapshot-backed pipeline is wired up. -/// -/// `true` on Windows: [`job::candidate_source::VssCandidateSource`] and -/// [`job::content_source::VssContentSource`] are real, and -/// [`job::vss_job::run_vss_job`] has been validated end to end against -/// real hardware (real VSS snapshot, real ephemeral target-selection -/// daemon, real privileged Reader). `false` on every other platform โ€” -/// VSS doesn't exist there, so this pipeline fundamentally can't run. -#[must_use] -#[cfg(windows)] -pub const fn is_implemented() -> bool { - true -} - -/// Non-Windows: see the Windows doc comment above for why this is -/// always `false` here, not a scaffold-vs-real distinction. -#[must_use] -#[cfg(not(windows))] -pub const fn is_implemented() -> bool { - false -} - -/// Run the two-pipe transport server for the process's whole lifetime. -/// -/// The real entry point an external consumer (e.g. Docenta) connects -/// to. See the crate-private `serve` module's doc comment for the -/// wire-level design. -/// -/// # Errors -/// Returns an error only if a pipe itself cannot be created at all. -#[cfg(windows)] -pub fn serve() -> anyhow::Result<()> { - serve::run() -} - -#[cfg(test)] -mod tests { - use super::is_implemented; - - #[test] - fn is_implemented_matches_platform_capability() { - assert_eq!(is_implemented(), cfg!(windows)); - } -} diff --git a/crates/uffs-content/src/main.rs b/crates/uffs-content/src/main.rs deleted file mode 100644 index b5a39296a..000000000 --- a/crates/uffs-content/src/main.rs +++ /dev/null @@ -1,423 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! UFFS Content Service โ€” unprivileged content-coordinator binary for -//! downstream consumers (e.g. Docenta). -//! -//! # Status -//! -//! Job intake, manifest construction, and protocol framing are -//! implemented (`uffs_content::job`), but only against the cross-platform -//! `std::fs`-based candidate/content sources, not real VSS snapshots yet. -//! This bin is still a thin `--version`-only entry point โ€” it does not -//! yet parse a job spec off the command line and dispatch it. See -//! Docenta's `uffs-ingest-protocol-v2-vss.md` for the target contract -//! (the authoritative spec this tool is built against) and -//! `docs/dev/architecture/` (local-only) for the surrounding design -//! review. -//! -//! # Usage -//! -//! ```bash -//! uffs-content --version # Print version (also -V) -//! uffs-content --serve # Run the two-pipe transport server -//! # (the real entry point for a -//! # downstream consumer, e.g. Docenta) -//! uffs-content --self-test-vss-playback # Elevated smoke test: real VSS -//! # snapshot + real Reader playback -//! uffs-content --self-test-vss-query # Elevated smoke test: real -//! # extension-filtered query against -//! # an existing directory, verified -//! # against a ground-truth disk walk -//! uffs-content --self-test-reader-benchmark [query] [--drive C,D,E] # Elevated: -//! # measure real content-read throughput. -//! # [query] defaults to "*"; --drive takes -//! # a comma-separated list (C or C: form, -//! # matching uffs.exe's own --drive flag) -//! # and defaults to every local NTFS drive -//! # when omitted -//! ``` - -// Reserved for the wire types the bin will emit once job intake is wired -// up as a real CLI entry point; not yet used from this thin bin. -// Dev-dependencies used by `uffs_content`'s tests, not by this bin. -// Used by `uffs_content::job::snapshot_client` (the real Snapshot -// Manager pipe client), not by this thin entry point directly. -#[cfg(windows)] -use anyhow as _; -#[cfg(test)] -use blake3 as _; -// Used by `uffs_content::job::workflow`'s pipelined content reader and -// `uffs_content::job::reader_client`'s per-drive connection pool, not -// by this thin entry point directly. -use crossbeam_channel as _; -// Used by `uffs_content::run` (failure log + summary serialization), not -// by this thin entry point directly. -use serde as _; -use serde_json as _; -#[cfg(test)] -use tempfile as _; -// Used by `uffs_content::serve`'s two-pipe transport server, not by this -// thin entry point directly. -#[cfg(windows)] -use tokio as _; -// Used directly by `init_tracing()` on Windows; on every other platform -// that function doesn't exist, so `tracing` (an unconditional -// dependency โ€” see `Cargo.toml`) goes unused by this bin directly. -#[cfg(not(windows))] -use tracing as _; -#[cfg(windows)] -use uffs_broker_protocol as _; -// Used to spawn/query the ephemeral `uffsd` instance (not by this thin -// entry point directly). -#[cfg(windows)] -use uffs_client as _; -use uffs_content_protocol as _; -// Used by `uffs_content::job::reader_client`/`content_source::VssContentSource`, -// not by this thin entry point directly. -#[cfg(windows)] -use uffs_content_reader_protocol as _; -// Used by `uffs_content::job::vss_job` (default-to-all-drives root -// resolution), not by this thin entry point directly. -#[cfg(windows)] -use uffs_mft as _; -// Used by `uffs_content::serve`'s named-pipe owner-only DACL helpers, -// not by this thin entry point directly. -#[cfg(windows)] -use uffs_security as _; -// Used by `uffs_content::job::workflow`, not by this thin entry point -// directly. -use uuid as _; - -/// Install the `tracing` subscriber every `--self-test-*`/`--serve` entry -/// point relies on for diagnostic output (job/lease/daemon/reader -/// lifecycle events across `uffs_content::job`) โ€” mirrors -/// `uffs-broker`/`uffs-content-reader`'s own `fmt()` init exactly -/// (`with_target(false)`, `INFO` by default) so a foreground `--serve` -/// run's log looks the same shape as the Broker's. -/// -/// Uses `try_init` so a test harness embedding this crate that already -/// installed a subscriber doesn't panic. -#[cfg(windows)] -fn init_tracing() { - let init_result = tracing_subscriber::fmt() - .with_target(false) - .with_max_level(tracing::Level::INFO) - .with_writer(std::io::stderr) - .try_init(); - drop(init_result); -} - -#[expect( - clippy::print_stderr, - reason = "the final ready/scaffold status lines below run whether or not a job intake \ - flag matched, and are plain one-line user-facing status text rather than \ - diagnostic logging โ€” every diagnostic-logging path (self-test/benchmark/serve) \ - now has a real tracing subscriber via init_tracing()" -)] -fn main() { - // `--version` / `-V` is handled here, before any job dispatch, so it - // works on every platform and exits 0 โ€” matches `uffs-broker` and - // `uffsd` so the self-update version probe can parse it uniformly. - uffs_version::handle_version!("uffs-content"); - - #[cfg(windows)] - init_tracing(); - #[cfg(windows)] - tracing::info!( - pid = std::process::id(), - version = %uffs_version::version_short!("uffs-content"), - "uffs-content starting" - ); - - let args: Vec = std::env::args().collect(); - if let Some(test_dir) = self_test_vss_playback_dir(&args) { - std::process::exit(run_self_test_vss_playback(&test_dir)); - } - if let Some((root, extension)) = self_test_vss_query_args(&args) { - std::process::exit(run_self_test_vss_query(&root, &extension)); - } - match self_test_reader_benchmark_args(&args) { - Some(Ok((roots, query))) => { - std::process::exit(run_self_test_reader_benchmark(&roots, &query)); - } - Some(Err(positionals)) => { - tracing::error!( - ?positionals, - "--self-test-reader-benchmark takes exactly one bare [query] argument; \ - got more than one (this flag no longer takes an \"all\"/roots positional โ€” \ - use --drive instead, or omit --drive entirely for every local NTFS drive)" - ); - std::process::exit(1); - } - None => {} - } - if args.iter().any(|arg| arg == "--serve") { - std::process::exit(run_serve()); - } - - if uffs_content::is_implemented() { - eprintln!("uffs-content: ready."); - } else { - eprintln!("uffs-content: scaffold only, job intake is not yet implemented."); - } -} - -/// Run [`uffs_content::serve`] and report a fatal startup error, if any. -/// Does not return under normal operation โ€” the server runs for the -/// process's whole lifetime. Returns the process exit code (`1`) only -/// if the server failed to start at all. -#[cfg(windows)] -#[expect( - clippy::print_stderr, - reason = "one-shot CLI diagnostic invoked before any tracing subscriber exists" -)] -fn run_serve() -> i32 { - match uffs_content::serve() { - Ok(()) => 0, - Err(err) => { - eprintln!("FAIL: {err:#}"); - 1 - } - } -} - -/// Non-Windows stub: the two-pipe transport server only ever serves -/// VSS-backed jobs, which don't exist on this platform. -#[cfg(not(windows))] -#[expect( - clippy::print_stderr, - reason = "one-shot CLI diagnostic invoked before any tracing subscriber exists" -)] -fn run_serve() -> i32 { - eprintln!( - "uffs-content: --serve is Windows-only (VSS-backed jobs don't exist on this platform)" - ); - 1 -} - -/// Return the directory argument following `--self-test-vss-playback`, -/// if present. -#[cfg(windows)] -fn self_test_vss_playback_dir(args: &[String]) -> Option { - let flag_index = args - .iter() - .position(|arg| arg == "--self-test-vss-playback")?; - args.get(flag_index + 1).map(std::path::PathBuf::from) -} - -/// Non-Windows stub: `--self-test-vss-playback` needs a real VSS -/// snapshot, which doesn't exist on this platform. -#[cfg(not(windows))] -const fn self_test_vss_playback_dir(_args: &[String]) -> Option { - None -} - -/// Run [`uffs_content::job::self_test::self_test_vss_playback`] and -/// print a PASS/FAIL result โ€” a manual, elevated smoke test proving the -/// real VSS-snapshot + privileged-Reader content pipeline works at -/// runtime on this machine. Returns the process exit code (`0` pass, -/// `1` fail). -#[cfg(windows)] -#[expect( - clippy::print_stderr, - reason = "one-shot CLI diagnostic invoked before any tracing subscriber exists" -)] -fn run_self_test_vss_playback(test_dir: &std::path::Path) -> i32 { - match uffs_content::job::self_test::self_test_vss_playback(test_dir) { - Ok(()) => { - eprintln!( - "PASS: VSS snapshot + Reader playback round trip succeeded ({})", - test_dir.display() - ); - 0 - } - Err(err) => { - eprintln!("FAIL: {err:#}"); - 1 - } - } -} - -/// Non-Windows stub, matching [`self_test_vss_playback_dir`] always -/// returning `None` there (so this is unreachable in practice, but kept -/// for a symmetrical `#[cfg]` shape). -#[cfg(not(windows))] -const fn run_self_test_vss_playback(_test_dir: &std::path::Path) -> i32 { - 1 -} - -/// Return the `(root, extension)` arguments following -/// `--self-test-vss-query`, if present. -#[cfg(windows)] -fn self_test_vss_query_args(args: &[String]) -> Option<(std::path::PathBuf, String)> { - let flag_index = args.iter().position(|arg| arg == "--self-test-vss-query")?; - let root = args.get(flag_index + 1).map(std::path::PathBuf::from)?; - let extension = args.get(flag_index + 2).cloned()?; - Some((root, extension)) -} - -/// Non-Windows stub: `--self-test-vss-query` needs a real VSS snapshot, -/// which doesn't exist on this platform. -#[cfg(not(windows))] -const fn self_test_vss_query_args(_args: &[String]) -> Option<(std::path::PathBuf, String)> { - None -} - -/// Run [`uffs_content::job::self_test::self_test_vss_query_metadata`] and -/// print a PASS/FAIL result. Returns the process exit code (`0` pass, `1` -/// fail). -#[cfg(windows)] -#[expect( - clippy::print_stderr, - reason = "one-shot CLI diagnostic invoked before any tracing subscriber exists" -)] -fn run_self_test_vss_query(root: &std::path::Path, extension: &str) -> i32 { - match uffs_content::job::self_test::self_test_vss_query_metadata(root, extension) { - Ok(()) => { - eprintln!( - "PASS: query metadata/content totals matched ground truth ({}, *.{extension})", - root.display() - ); - 0 - } - Err(err) => { - eprintln!("FAIL: {err:#}"); - 1 - } - } -} - -/// Non-Windows stub, matching [`self_test_vss_query_args`] always -/// returning `None` there (so this is unreachable in practice, but kept -/// for a symmetrical `#[cfg]` shape). -#[cfg(not(windows))] -const fn run_self_test_vss_query(_root: &std::path::Path, _extension: &str) -> i32 { - 1 -} - -/// Success case: `(roots, query)`. Error case: every bare positional -/// argument found, when there was more than the one `[query]` this flag -/// accepts โ€” see [`self_test_reader_benchmark_args`]'s doc comment. -type ReaderBenchmarkArgs = Result<(Vec, String), Vec>; - -/// Return the `(roots, query)` arguments following -/// `--self-test-reader-benchmark`, if present. `query` is the one bare -/// (non-`--drive`) positional argument, defaulting to `"*"` if omitted. -/// `--drive` takes a comma-separated drive-letter list โ€” each entry in -/// `C` or `C:` form, exactly matching `uffs.exe`'s own `--drive` flag -/// (see [`parse_drive_list`]) โ€” resolved to `:\\` roots. -/// Omitting `--drive` resolves to an empty `Vec` โ€” every local NTFS -/// drive, see [`uffs_content::job::vss_job::run_vss_job`]. -/// -/// A *second* bare positional argument (e.g. a leftover `all` from the -/// pre-`--drive` syntax this flag used to have) is a usage error, not -/// silently dropped: `Some(Err(_))` tells [`main`] to `tracing::error!` -/// and exit `1` itself (this function must not call `std::process::exit` -/// directly โ€” `clippy::exit` reserves that to `main`), rather than -/// quietly running the wrong query, which is exactly what used to -/// happen here. -#[cfg(windows)] -fn self_test_reader_benchmark_args(args: &[String]) -> Option { - let flag_index = args - .iter() - .position(|arg| arg == "--self-test-reader-benchmark")?; - let rest = args.get(flag_index + 1..)?; - - let mut roots: Vec = Vec::new(); - let mut positionals: Vec = Vec::new(); - let mut rest_iter = rest.iter(); - while let Some(arg) = rest_iter.next() { - if arg == "--drive" { - if let Some(value) = rest_iter.next() { - roots.extend(parse_drive_list(value)); - } - } else { - positionals.push(arg.clone()); - } - } - - if positionals.len() > 1 { - return Some(Err(positionals)); - } - - let query = positionals - .into_iter() - .next() - .unwrap_or_else(|| "*".to_owned()); - Some(Ok((roots, query))) -} - -/// Parse a comma-separated drive-letter list (each entry `C` or `C:`, -/// case-insensitive, matching `uffs.exe`'s own `--drive` flag) into -/// `:\\` roots. Entries that aren't exactly one ASCII letter -/// (once a trailing `:` is stripped) are silently skipped, matching -/// `uffs.exe`'s own tolerant `--drive` parsing. -#[cfg(windows)] -fn parse_drive_list(value: &str) -> Vec { - value - .split(',') - .filter_map(|part| { - let trimmed = part.trim(); - let letter = trimmed.strip_suffix(':').unwrap_or(trimmed); - let mut chars = letter.chars(); - let ch = chars.next()?; - (chars.next().is_none() && ch.is_ascii_alphabetic()) - .then(|| std::path::PathBuf::from(format!("{}:\\", ch.to_ascii_uppercase()))) - }) - .collect() -} - -/// Non-Windows stub: `--self-test-reader-benchmark` needs a real VSS -/// snapshot, which doesn't exist on this platform. -#[cfg(not(windows))] -const fn self_test_reader_benchmark_args(_args: &[String]) -> Option { - None -} - -/// Run [`uffs_content::job::self_test::self_test_reader_benchmark`] and -/// print the measured content-read throughput. Returns the process exit -/// code (`0` pass, `1` fail). -#[cfg(windows)] -#[expect( - clippy::print_stderr, - reason = "one-shot CLI diagnostic invoked before any tracing subscriber exists" -)] -fn run_self_test_reader_benchmark(roots: &[std::path::PathBuf], query: &str) -> i32 { - match uffs_content::job::self_test::self_test_reader_benchmark(roots, query) { - Ok(report) => { - #[expect( - clippy::cast_precision_loss, - reason = "diagnostic-only display value, not computed against further" - )] - #[expect( - clippy::float_arithmetic, - reason = "diagnostic-only unit conversion for a printed benchmark report" - )] - let content_mib = report.content_bytes as f64 / (1_024.0_f64 * 1_024.0_f64); - eprintln!( - "PASS: {} candidates ({} succeeded) โ€” {:.2} MiB content-read in {} ms \ - ({:.2} MiB/s); enumeration+manifest: {} ms", - report.candidate_count, - report.succeeded_count, - content_mib, - report.content_read_ms, - report.throughput_mib_per_sec, - report.enumeration_ms, - ); - 0 - } - Err(err) => { - eprintln!("FAIL: {err:#}"); - 1 - } - } -} - -/// Non-Windows stub, matching [`self_test_reader_benchmark_args`] always -/// returning `None` there (so this is unreachable in practice, but kept -/// for a symmetrical `#[cfg]` shape). -#[cfg(not(windows))] -const fn run_self_test_reader_benchmark(_roots: &[std::path::PathBuf], _query: &str) -> i32 { - 1 -} diff --git a/crates/uffs-content/src/run/failure_log.rs b/crates/uffs-content/src/run/failure_log.rs deleted file mode 100644 index 04440cf72..000000000 --- a/crates/uffs-content/src/run/failure_log.rs +++ /dev/null @@ -1,199 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Append-only JSONL failure log: one record per non-success candidate. - -use std::fs::OpenOptions; -use std::io::{self, Write as _}; -use std::path::Path; - -use serde::{Deserialize, Serialize}; -use uffs_content_protocol::error::ErrorCode; -use uffs_content_protocol::frame::{FailedOutcome, FailureStage, RetryClass}; - -/// Discriminant for a [`FailureRecord`]'s outcome. -/// -/// Mirrors the non-success half of -/// [`uffs_content_protocol::state::CandidateOutcome`] (excludes -/// `Succeeded` โ€” a successful candidate is never written to this log, -/// only to the manifest and the content stream itself). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum FailureOutcomeKind { - /// May be retried in a later job attempt against a new snapshot. - FailedRetryable, - /// Will not succeed on retry. - FailedTerminal, - /// Deferred to manual or later handling. - DeferredManual, -} - -impl From for FailureOutcomeKind { - fn from(outcome: FailedOutcome) -> Self { - match outcome { - FailedOutcome::Retryable => Self::FailedRetryable, - FailedOutcome::Terminal => Self::FailedTerminal, - } - } -} - -/// One non-success candidate outcome, as persisted to the run's failure -/// log. -/// -/// Serialized one JSON object per line (JSONL), appended as candidates -/// resolve โ€” see [`FailureLogWriter`]. A candidate present in the -/// manifest but absent from both this log and the successful-content -/// stream simply hasn't resolved yet; a reader must not infer success -/// from mere absence. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct FailureRecord { - /// Candidate this record terminates. - pub candidate_id: u64, - /// Which of the three non-success outcomes this is. - pub outcome: FailureOutcomeKind, - /// Which pipeline stage the failure occurred at. Absent for - /// `DeferredManual` (a deferral isn't a failure at a stage). - #[serde(skip_serializing_if = "Option::is_none", default)] - pub failure_stage: Option, - /// Stable machine-readable error/reason code - /// ([`ErrorCode::as_str`]). - pub error_code: String, - /// Underlying OS error code, if applicable. - #[serde(skip_serializing_if = "Option::is_none", default)] - pub os_error_code: Option, - /// How this failure may be retried. Absent for `DeferredManual`. - #[serde(skip_serializing_if = "Option::is_none", default)] - pub retry_class: Option, - /// Bytes emitted before the failure (0 for a deferral, which never - /// starts streaming a body). - pub bytes_emitted_before_failure: u64, - /// Human-readable diagnostic message. - pub message: String, -} - -impl FailureRecord { - /// Build a record for a `FAILED_RETRYABLE`/`FAILED_TERMINAL` outcome. - #[must_use] - #[expect( - clippy::too_many_arguments, - reason = "single call site, flat args mirroring FILE_FAILED's own field list" - )] - pub fn failed>( - candidate_id: u64, - outcome: FailedOutcome, - failure_stage: FailureStage, - error_code: ErrorCode, - os_error_code: Option, - retry_class: RetryClass, - bytes_emitted_before_failure: u64, - message: S, - ) -> Self { - Self { - candidate_id, - outcome: outcome.into(), - failure_stage: Some(failure_stage_label(failure_stage).to_owned()), - error_code: error_code.as_str().to_owned(), - os_error_code, - retry_class: Some(retry_class_label(retry_class).to_owned()), - bytes_emitted_before_failure, - message: message.into(), - } - } - - /// Build a record for a `DEFERRED_MANUAL` outcome. - #[must_use] - pub fn deferred>( - candidate_id: u64, - reason_code: ErrorCode, - message: S, - ) -> Self { - Self { - candidate_id, - outcome: FailureOutcomeKind::DeferredManual, - failure_stage: None, - error_code: reason_code.as_str().to_owned(), - os_error_code: None, - retry_class: None, - bytes_emitted_before_failure: 0, - message: message.into(), - } - } - - /// Serialize this record as one JSON line (no trailing newline). - /// - /// # Errors - /// Returns an error if JSON serialization fails. - pub fn to_json_line(&self) -> serde_json::Result { - serde_json::to_string(self) - } -} - -/// Stable `snake_case` label for a [`FailureStage`], for JSON โ€” kept local -/// to this log rather than added to the wire-protocol type, since the -/// binary wire codec (design-doc ยง5.4) and this auxiliary JSON log are -/// deliberately separate concerns. -const fn failure_stage_label(stage: FailureStage) -> &'static str { - match stage { - FailureStage::SnapshotCreate => "snapshot_create", - FailureStage::SnapshotOpen => "snapshot_open", - FailureStage::Enumeration => "enumeration", - FailureStage::Identity => "identity", - FailureStage::StreamResolution => "stream_resolution", - FailureStage::RunlistValidation => "runlist_validation", - FailureStage::Read => "read", - FailureStage::Reconstruction => "reconstruction", - FailureStage::Hash => "hash", - FailureStage::Transport => "transport", - FailureStage::ConsumerAck => "consumer_ack", - FailureStage::Internal => "internal", - } -} - -/// Stable `snake_case` label for a [`RetryClass`], for JSON โ€” see -/// [`failure_stage_label`] for why this lives here instead of on the -/// wire-protocol type. -const fn retry_class_label(class: RetryClass) -> &'static str { - match class { - RetryClass::RetrySameJob => "retry_same_job", - RetryClass::RetryNewSnapshot => "retry_new_snapshot", - RetryClass::RetryAfterResourceChange => "retry_after_resource_change", - RetryClass::RetryWithManualHandler => "retry_with_manual_handler", - RetryClass::RetryWithCredentialOrKey => "retry_with_credential_or_key", - RetryClass::DoNotRetry => "do_not_retry", - } -} - -/// Append-only writer for a run's failure log. -/// -/// Opens (or creates) the file in append mode and flushes after every -/// record, so a reader tailing the file mid-run always sees complete -/// lines โ€” there is no cross-call buffering to lose on a crash. -#[derive(Debug)] -pub struct FailureLogWriter { - /// The open file handle, in append mode. - file: std::fs::File, -} - -impl FailureLogWriter { - /// Open (creating if absent) the failure log at `path` for - /// appending. - /// - /// # Errors - /// Propagates the underlying [`io::Error`] from opening the file. - pub fn open(path: &Path) -> io::Result { - let file = OpenOptions::new().create(true).append(true).open(path)?; - Ok(Self { file }) - } - - /// Append one record as a JSON line. - /// - /// # Errors - /// Returns an error if serialization or the write/flush fails. - pub fn append(&mut self, record: &FailureRecord) -> io::Result<()> { - let line = record - .to_json_line() - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; - writeln!(self.file, "{line}")?; - self.file.flush() - } -} diff --git a/crates/uffs-content/src/run/mod.rs b/crates/uffs-content/src/run/mod.rs deleted file mode 100644 index db833adf4..000000000 --- a/crates/uffs-content/src/run/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Ephemeral per-run bookkeeping. -//! -//! A run's state is intentionally **not** a transactional per-candidate -//! job database. Three artifacts on disk fully describe a run: -//! -//! 1. The immutable candidate manifest ([`uffs_content_protocol::manifest`]) โ€” -//! written once, before streaming starts, and never modified. -//! 2. An append-only JSONL failure log (`run-.failures.jsonl`) โ€” one -//! [`FailureRecord`] line per non-success candidate, appended as candidates -//! resolve. See [`FailureLogWriter`]. -//! 3. A [`RunSummary`], atomically finalized only once every candidate has -//! reached a terminal outcome (`run-.summary.json`). See -//! [`RunCounters::finalize`] and [`RunSummary::finalize_to_disk`]. -//! -//! There is no durable per-candidate ledger, no lease/attempt history, -//! and no crash-recovery reconciliation: if the process or machine dies -//! before the summary is finalized, the run is simply incomplete โ€” the -//! existence of a valid final summary file *is* the job-completion -//! marker. Re-running from a fresh VSS snapshot is the recovery path, -//! not resuming mid-job: Docenta's own content-hash deduplication makes -//! re-streaming already-ingested content on a rerun a no-op on the -//! consumer side, so restarting from zero wastes no meaningful work. - -mod failure_log; -mod summary; - -pub use failure_log::{FailureLogWriter, FailureOutcomeKind, FailureRecord}; -pub use summary::{RunCounters, RunSummary, SummaryFinalizeError}; - -#[cfg(test)] -mod tests; diff --git a/crates/uffs-content/src/run/summary.rs b/crates/uffs-content/src/run/summary.rs deleted file mode 100644 index 39bb71552..000000000 --- a/crates/uffs-content/src/run/summary.rs +++ /dev/null @@ -1,236 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! In-memory run counters + the atomically finalized run summary. - -use std::fs::{self, File}; -use std::io::{self, Write as _}; -use std::path::{Path, PathBuf}; - -use serde::{Deserialize, Serialize}; - -/// In-memory tally of candidate outcomes as a run streams. -/// -/// Never persisted mid-run โ€” see the [`crate::run`] module docs for why -/// there is no durable per-candidate ledger. Only -/// [`RunCounters::finalize`] turns this into a [`RunSummary`], and only -/// once every candidate is accounted for. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct RunCounters { - /// Total candidates in the finalized manifest this run is streaming. - pub candidate_count: u64, - /// Candidates that succeeded. - pub succeeded_count: u64, - /// Candidates that failed retryably. - pub failed_retryable_count: u64, - /// Candidates that failed terminally. - pub failed_terminal_count: u64, - /// Candidates deferred to manual handling. - pub deferred_manual_count: u64, - /// Total logical bytes across all successful candidates. - pub logical_bytes_succeeded: u64, -} - -impl RunCounters { - /// Start a fresh counter set for a run whose manifest has - /// `candidate_count` candidates. - #[must_use] - pub const fn new(candidate_count: u64) -> Self { - Self { - candidate_count, - succeeded_count: 0, - failed_retryable_count: 0, - failed_terminal_count: 0, - deferred_manual_count: 0, - logical_bytes_succeeded: 0, - } - } - - /// Record one succeeded candidate. - pub const fn record_succeeded(&mut self, logical_size: u64) { - self.succeeded_count += 1; - self.logical_bytes_succeeded += logical_size; - } - - /// Record one failed-retryable candidate. - pub const fn record_failed_retryable(&mut self) { - self.failed_retryable_count += 1; - } - - /// Record one failed-terminal candidate. - pub const fn record_failed_terminal(&mut self) { - self.failed_terminal_count += 1; - } - - /// Record one deferred-manual candidate. - pub const fn record_deferred_manual(&mut self) { - self.deferred_manual_count += 1; - } - - /// Total candidates that have reached *any* terminal outcome so far. - #[must_use] - pub const fn resolved_count(&self) -> u64 { - self.succeeded_count - + self.failed_retryable_count - + self.failed_terminal_count - + self.deferred_manual_count - } - - /// Whether every candidate in the manifest has a terminal outcome. - #[must_use] - pub const fn is_complete(&self) -> bool { - self.resolved_count() == self.candidate_count - } - - /// Turn these counters into a [`RunSummary`], failing if any - /// candidate has not yet reached a terminal outcome. - /// - /// # Errors - /// Returns [`SummaryFinalizeError::Incomplete`] if - /// [`RunCounters::is_complete`] is false. - pub fn finalize( - self, - run_id: String, - started_at_unix_ms: i64, - finished_at_unix_ms: i64, - ) -> Result { - if !self.is_complete() { - return Err(SummaryFinalizeError::Incomplete { - candidate_count: self.candidate_count, - resolved_count: self.resolved_count(), - }); - } - Ok(RunSummary { - run_id, - started_at_unix_ms, - finished_at_unix_ms, - candidate_count: self.candidate_count, - succeeded_count: self.succeeded_count, - failed_retryable_count: self.failed_retryable_count, - failed_terminal_count: self.failed_terminal_count, - deferred_manual_count: self.deferred_manual_count, - logical_bytes_succeeded: self.logical_bytes_succeeded, - }) - } -} - -/// Why [`RunCounters::finalize`] refused to produce a [`RunSummary`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SummaryFinalizeError { - /// Not every candidate has reached a terminal outcome yet. - Incomplete { - /// Candidates in the manifest. - candidate_count: u64, - /// Candidates that have resolved so far. - resolved_count: u64, - }, -} - -impl core::fmt::Display for SummaryFinalizeError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::Incomplete { - candidate_count, - resolved_count, - } => write!( - f, - "cannot finalize run summary: {resolved_count} of {candidate_count} \ - candidates have a terminal outcome" - ), - } - } -} - -impl core::error::Error for SummaryFinalizeError {} - -/// The finalized, immutable record of one completed run. -/// -/// The existence of a valid file at this summary's final path (no -/// `.partial` suffix) is the job-completion marker: if a run or the -/// machine crashes before [`RunSummary::finalize_to_disk`] renames the -/// `.partial` file into place, the run is incomplete, full stop. There is -/// no partial-completion state to reconcile โ€” rerun the job from a new -/// VSS snapshot (see the [`crate::run`] module docs). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct RunSummary { - /// Identifier for this run (matches the manifest's `job_id`, rendered - /// as a string for JSON). - pub run_id: String, - /// When the run started, Unix milliseconds. - pub started_at_unix_ms: i64, - /// When the run finished (all candidates resolved), Unix - /// milliseconds. - pub finished_at_unix_ms: i64, - /// Total candidates in the finalized manifest. - pub candidate_count: u64, - /// Candidates that succeeded. - pub succeeded_count: u64, - /// Candidates that failed retryably. - pub failed_retryable_count: u64, - /// Candidates that failed terminally. - pub failed_terminal_count: u64, - /// Candidates deferred to manual handling. - pub deferred_manual_count: u64, - /// Total logical bytes across all successful candidates. - pub logical_bytes_succeeded: u64, -} - -impl RunSummary { - /// Atomically finalize this summary to `final_path`. - /// - /// Writes to a sibling `.partial` file first, `fsync`s it, then - /// renames it into place โ€” the rename is the atomic step a reader - /// can rely on to never observe a half-written summary. `final_path` - /// must not already exist (a run's summary is written exactly once). - /// - /// # Errors - /// Returns an [`io::ErrorKind::AlreadyExists`] error if `final_path` - /// already exists, and otherwise propagates the underlying - /// [`io::Error`] from any filesystem step. - pub fn finalize_to_disk(&self, final_path: &Path) -> io::Result<()> { - if final_path.exists() { - return Err(io::Error::new( - io::ErrorKind::AlreadyExists, - format!("run summary already finalized at {}", final_path.display()), - )); - } - let partial_path = partial_path_for(final_path); - let json = serde_json::to_vec_pretty(self) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; - let mut partial_file = File::create(&partial_path)?; - partial_file.write_all(&json)?; - partial_file.sync_all()?; - drop(partial_file); - fs::rename(&partial_path, final_path)?; - Ok(()) - } - - /// Read a previously finalized summary from `final_path`. - /// - /// Returns `Ok(None)` if no file exists there yet (i.e. the run has - /// not completed) rather than an error โ€” "not finalized" is an - /// expected, common state, not a failure. - /// - /// # Errors - /// Propagates I/O errors other than "not found", and wraps a JSON - /// parse failure (for a file that exists but is not a valid summary) - /// as an [`io::Error`]. - pub fn load_if_finalized(final_path: &Path) -> io::Result> { - match fs::read(final_path) { - Ok(bytes) => { - let summary = serde_json::from_slice(&bytes) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; - Ok(Some(summary)) - } - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), - Err(err) => Err(err), - } - } -} - -/// The `.partial` sibling path used during atomic finalization. -fn partial_path_for(final_path: &Path) -> PathBuf { - let mut partial = final_path.as_os_str().to_owned(); - partial.push(".partial"); - PathBuf::from(partial) -} diff --git a/crates/uffs-content/src/run/tests.rs b/crates/uffs-content/src/run/tests.rs deleted file mode 100644 index 521ca87b1..000000000 --- a/crates/uffs-content/src/run/tests.rs +++ /dev/null @@ -1,179 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Tests for the ephemeral run-state model (manifest + failure log + -//! atomically finalized summary). - -use std::fs; - -use uffs_content_protocol::error::ErrorCode; -use uffs_content_protocol::frame::{FailedOutcome, FailureStage, RetryClass}; - -use super::{ - FailureLogWriter, FailureOutcomeKind, FailureRecord, RunCounters, SummaryFinalizeError, -}; - -#[test] -fn finalize_rejects_incomplete_run() { - let mut counters = RunCounters::new(3); - counters.record_succeeded(100); - counters.record_failed_terminal(); - // Only 2 of 3 candidates resolved. - assert!(!counters.is_complete()); - - let err = counters - .finalize("run-1".to_owned(), 1_000, 2_000) - .expect_err("must not finalize with unresolved candidates"); - assert_eq!(err, SummaryFinalizeError::Incomplete { - candidate_count: 3, - resolved_count: 2, - }); -} - -#[test] -fn finalize_succeeds_once_every_candidate_is_resolved() { - let mut counters = RunCounters::new(4); - counters.record_succeeded(100); - counters.record_succeeded(200); - counters.record_failed_retryable(); - counters.record_deferred_manual(); - assert!(counters.is_complete()); - - let summary = counters - .finalize("run-2".to_owned(), 1_000, 5_000) - .expect("all candidates resolved, finalize must succeed"); - assert_eq!(summary.candidate_count, 4); - assert_eq!(summary.succeeded_count, 2); - assert_eq!(summary.failed_retryable_count, 1); - assert_eq!(summary.failed_terminal_count, 0); - assert_eq!(summary.deferred_manual_count, 1); - assert_eq!(summary.logical_bytes_succeeded, 300); -} - -#[test] -fn atomic_finalize_writes_final_file_and_removes_partial() { - let dir = tempfile::tempdir().expect("create temp dir"); - let final_path = dir.path().join("run-3.summary.json"); - let partial_path = dir.path().join("run-3.summary.json.partial"); - - let mut counters = RunCounters::new(1); - counters.record_succeeded(42); - let summary = counters - .finalize("run-3".to_owned(), 10, 20) - .expect("complete run finalizes"); - - summary - .finalize_to_disk(&final_path) - .expect("finalize_to_disk must succeed"); - - assert!(final_path.exists(), "final summary file must exist"); - assert!( - !partial_path.exists(), - "partial file must be gone after rename" - ); - - let loaded = super::RunSummary::load_if_finalized(&final_path) - .expect("load must succeed") - .expect("summary must be present"); - assert_eq!(loaded, summary); -} - -#[test] -fn finalize_to_disk_refuses_to_overwrite_existing_summary() { - let dir = tempfile::tempdir().expect("create temp dir"); - let final_path = dir.path().join("run-4.summary.json"); - - let mut counters = RunCounters::new(1); - counters.record_succeeded(1); - let summary = counters - .finalize("run-4".to_owned(), 0, 1) - .expect("complete run finalizes"); - summary - .finalize_to_disk(&final_path) - .expect("first finalize must succeed"); - - let err = summary - .finalize_to_disk(&final_path) - .expect_err("second finalize to the same path must fail"); - assert_eq!(err.kind(), std::io::ErrorKind::AlreadyExists); -} - -#[test] -fn unfinalized_run_reports_no_summary_rather_than_fabricating_one() { - let dir = tempfile::tempdir().expect("create temp dir"); - let final_path = dir.path().join("run-5.summary.json"); - - let loaded = - super::RunSummary::load_if_finalized(&final_path).expect("missing file is not an error"); - assert!( - loaded.is_none(), - "no summary file present means the run is incomplete, not a default/empty summary" - ); -} - -#[test] -fn failure_log_appends_and_round_trips_jsonl() { - let dir = tempfile::tempdir().expect("create temp dir"); - let log_path = dir.path().join("run-6.failures.jsonl"); - - let failed = FailureRecord::failed( - 7, - FailedOutcome::Retryable, - FailureStage::Read, - ErrorCode::ReadIoTransient, - Some(5), - RetryClass::RetryNewSnapshot, - 1_024, - "transient read error", - ); - let deferred = FailureRecord::deferred(9, ErrorCode::CompressedManual, "NTFS-compressed"); - - let mut writer = FailureLogWriter::open(&log_path).expect("open failure log"); - writer.append(&failed).expect("append failed record"); - writer.append(&deferred).expect("append deferred record"); - drop(writer); - - let contents = fs::read_to_string(&log_path).expect("read failure log"); - let lines: Vec<&str> = contents.lines().collect(); - assert_eq!(lines.len(), 2, "one JSON object per appended record"); - - let line_failed = lines.first().expect("first line present"); - let line_deferred = lines.get(1).expect("second line present"); - - let decoded_failed: FailureRecord = - serde_json::from_str(line_failed).expect("decode first line"); - assert_eq!(decoded_failed, failed); - assert_eq!(decoded_failed.outcome, FailureOutcomeKind::FailedRetryable); - - let decoded_deferred: FailureRecord = - serde_json::from_str(line_deferred).expect("decode second line"); - assert_eq!(decoded_deferred, deferred); - assert_eq!(decoded_deferred.outcome, FailureOutcomeKind::DeferredManual); - assert!(decoded_deferred.failure_stage.is_none()); - assert!(decoded_deferred.retry_class.is_none()); -} - -#[test] -fn failure_log_writer_appends_across_reopens() { - let dir = tempfile::tempdir().expect("create temp dir"); - let log_path = dir.path().join("run-7.failures.jsonl"); - - let mut first_writer = FailureLogWriter::open(&log_path).expect("open failure log"); - first_writer - .append(&FailureRecord::deferred(1, ErrorCode::SparseManual, "a")) - .expect("append first"); - drop(first_writer); - - let mut second_writer = FailureLogWriter::open(&log_path).expect("reopen failure log"); - second_writer - .append(&FailureRecord::deferred(2, ErrorCode::SparseManual, "b")) - .expect("append second after reopen"); - drop(second_writer); - - let contents = fs::read_to_string(&log_path).expect("read failure log"); - assert_eq!( - contents.lines().count(), - 2, - "reopening must append, not truncate" - ); -} diff --git a/crates/uffs-content/src/serve/command_pipe.rs b/crates/uffs-content/src/serve/command_pipe.rs deleted file mode 100644 index 241fbea40..000000000 --- a/crates/uffs-content/src/serve/command_pipe.rs +++ /dev/null @@ -1,221 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Command pipe: job submission/resume and every control frame -//! (`WINDOW_UPDATE`/`FILE_ACK`/`JOB_CANCEL` in, `PROGRESS`/`HEARTBEAT` -//! out โ€” the latter two not wired up yet, see the module doc's v1 gaps). -//! -//! Unlike [`super::stream`]'s data pipe (job-owned, exits once the job -//! completes), this pipe is server-lifetime: it loops accepting -//! connections for as long as the process runs, since it must remain -//! reachable across job boundaries โ€” a `JOB_SUBMIT` for the *next* job -//! has to land somewhere even after the previous job's data pipe has -//! long since torn down. - -use alloc::sync::Arc; - -use tokio::net::windows::named_pipe::NamedPipeServer; -use uffs_content_protocol::COMMAND_PIPE_NAME; -use uffs_content_protocol::codec::Reader as WireReader; -use uffs_content_protocol::frame::{ - FileAck, FrameEnvelope, FrameType, JobCancel, JobSubmit, WindowUpdate, -}; - -use super::{ControlSignal, ServerState, stream}; -use crate::job::intake::JobRequest; - -/// Run the command pipe server for the process's whole lifetime. -/// -/// # Errors -/// Returns an error only if the pipe itself cannot be created at all. -#[expect( - clippy::infinite_loop, - reason = "server-lifetime accept loop: exits only via process shutdown, matching \ - uffs-broker's own sweep-expired-leases loop" -)] -pub(super) async fn serve(state: Arc) -> anyhow::Result<()> { - let mut first_instance = true; - loop { - let mut pipe = - super::pipe_io::accept_connection(COMMAND_PIPE_NAME, &mut first_instance).await; - tracing::info!("consumer connected on command pipe"); - serve_connection(&state, &mut pipe).await; - } -} - -/// Read and dispatch messages from one connected command-pipe client -/// until it disconnects or sends something malformed. -async fn serve_connection(state: &Arc, pipe: &mut NamedPipeServer) { - loop { - match super::pipe_io::read_one_message(pipe).await { - Ok(Some(message)) => dispatch(state, &message).await, - Ok(None) => { - tracing::info!("consumer disconnected from command pipe"); - return; - } - Err(err) => { - tracing::warn!(error = %err, "malformed command-pipe message; closing connection"); - return; - } - } - } -} - -/// Decode one framed message and dispatch it to the right handler. -async fn dispatch(state: &Arc, message: &[u8]) { - let mut reader = WireReader::new(message); - let (envelope, payload) = match FrameEnvelope::decode(&mut reader, u64::MAX) { - Ok(decoded) => decoded, - Err(err) => { - tracing::warn!(error = %err, "failed to decode command-pipe frame envelope"); - return; - } - }; - - match envelope.frame_type { - FrameType::JobSubmit => handle_job_submit(state, &JobSubmit::decode(&payload)), - FrameType::JobResume => handle_job_resume(state, envelope.job_id), - FrameType::WindowUpdate => handle_window_update(state, envelope.job_id, &payload).await, - FrameType::FileAck => handle_file_ack(state, envelope.job_id, &payload).await, - FrameType::JobCancel => handle_job_cancel(state, envelope.job_id, &payload).await, - producer_to_consumer @ (FrameType::JobBegin - | FrameType::FileBegin - | FrameType::ContentChunk - | FrameType::FileEnd - | FrameType::FileFailed - | FrameType::FileDeferred - | FrameType::Progress - | FrameType::Heartbeat - | FrameType::JobEnd) => { - tracing::warn!( - ?producer_to_consumer, - "producer-to-consumer frame type on command pipe; ignoring" - ); - } - } -} - -/// `JOB_SUBMIT`: start a new job, unless one is already active (v1's -/// documented one-job-at-a-time scope โ€” see the crate module doc). -fn handle_job_submit(state: &Arc, submit: &JobSubmit) { - { - let active = state - .active - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if active.is_some() { - tracing::warn!( - "JOB_SUBMIT rejected: a job is already active (v1 serves one at a time)" - ); - return; - } - } - let request: JobRequest = match serde_json::from_slice(&submit.job_spec_json) { - Ok(request) => request, - Err(err) => { - tracing::warn!(error = %err, "JOB_SUBMIT payload did not decode as a JobRequest"); - return; - } - }; - let run_dir = std::env::temp_dir().join(format!( - "uffs-content-serve-{}", - uuid::Uuid::new_v4().simple() - )); - if let Err(err) = std::fs::create_dir_all(&run_dir) { - tracing::warn!(error = %err, path = %run_dir.display(), "failed to create job run dir"); - return; - } - stream::spawn(Arc::clone(state), request, run_dir); -} - -/// `JOB_RESUME`: a reconnecting consumer naming a job it wants to keep -/// receiving. Nothing to *do* here beyond logging โ€” the job's own data- -/// pipe accept loop ([`super::stream`]) is already waiting for exactly -/// this reconnection, and picks it up on its own the moment the -/// consumer opens a new data-pipe connection. If `job_id` doesn't match -/// (or there is no) active job, this producer process doesn't have that -/// job anymore (it crashed/restarted since) โ€” the already-decided -/// fallback in `crate::run`'s own doc comment applies: the consumer -/// starts a fresh `JOB_SUBMIT` with a new VSS snapshot. -fn handle_job_resume(state: &Arc, job_id: [u8; 16]) { - let active = state - .active - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - match &*active { - Some(job) if job.job_id == job_id => { - tracing::info!(job_id = %super::pipe_io::hex_job_id(job_id), "JOB_RESUME acknowledged (data pipe reconnect expected)"); - } - _ => { - tracing::warn!(job_id = %super::pipe_io::hex_job_id(job_id), "JOB_RESUME for an unknown/no-longer-active job"); - } - } -} - -/// `WINDOW_UPDATE`: forward to the job's streaming task. -async fn handle_window_update(state: &Arc, job_id: [u8; 16], payload: &[u8]) { - let mut reader = WireReader::new(payload); - let Ok(update) = WindowUpdate::decode(&mut reader) else { - tracing::warn!("failed to decode WINDOW_UPDATE payload"); - return; - }; - send_signal( - state, - job_id, - ControlSignal::WindowGrant(update.additional_window_bytes), - ) - .await; -} - -/// `FILE_ACK`: forward accepted acks to the job's streaming task. A -/// rejected ack (digest mismatch on the consumer's side) is logged, not -/// forwarded โ€” the candidate stays pending and will be retransmitted, -/// matching design-doc ยง12.9's "a digest mismatch is REJECTED" and -/// ยง9.4's retransmit-on-non-ack contract. -async fn handle_file_ack(state: &Arc, job_id: [u8; 16], payload: &[u8]) { - let mut reader = WireReader::new(payload); - let Ok(ack) = FileAck::decode(&mut reader) else { - tracing::warn!("failed to decode FILE_ACK payload"); - return; - }; - if ack.consumer_status == uffs_content_protocol::frame::ConsumerAckStatus::Rejected { - tracing::warn!( - candidate_id = ack.candidate_id, - error_code = ?ack.consumer_error_code, - "consumer rejected candidate; leaving it pending for retransmission" - ); - return; - } - send_signal(state, job_id, ControlSignal::FileAcked(ack.candidate_id)).await; -} - -/// `JOB_CANCEL`: forward to the job's streaming task. -async fn handle_job_cancel(state: &Arc, job_id: [u8; 16], payload: &[u8]) { - let mut reader = WireReader::new(payload); - let reason = - JobCancel::decode(&mut reader).map_or_else(|_| String::new(), |cancel| cancel.reason); - send_signal(state, job_id, ControlSignal::Cancel(reason)).await; -} - -/// Send `signal` to `job_id`'s streaming task, if it is the currently -/// active job. Logs and drops the signal otherwise (a control frame for -/// a job this producer process doesn't know about โ€” already resolved, -/// or from a stale/mistaken consumer). -async fn send_signal(state: &Arc, job_id: [u8; 16], signal: ControlSignal) { - let control_tx = { - let active = state - .active - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - match &*active { - Some(job) if job.job_id == job_id => job.control_tx.clone(), - _ => { - tracing::warn!(job_id = %super::pipe_io::hex_job_id(job_id), "control signal for an unknown/no-longer-active job"); - return; - } - } - }; - if control_tx.send(signal).await.is_err() { - tracing::warn!(job_id = %super::pipe_io::hex_job_id(job_id), "job's control channel closed; signal dropped"); - } -} diff --git a/crates/uffs-content/src/serve/mod.rs b/crates/uffs-content/src/serve/mod.rs deleted file mode 100644 index 942258225..000000000 --- a/crates/uffs-content/src/serve/mod.rs +++ /dev/null @@ -1,98 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -#![cfg(windows)] - -//! Two-pipe transport server: the real entry point an external consumer -//! (e.g. Docenta) connects to. -//! -//! Two named pipes, per the design conversation this implements: -//! -//! - [`uffs_content_protocol::DATA_PIPE_NAME`] โ€” the content stream itself -//! (`JOB_BEGIN`/`FILE_BEGIN`/`CONTENT_CHUNK`/.../`JOB_END`), producer to -//! consumer, paced by a [`crate::job::window::WindowTracker`]. Owned by the -//! job's own streaming task ([`stream`]) โ€” see that module's doc comment for -//! why "accept loop lives with the job, not as a separate always-on server" -//! is what makes reconnect-and-resume correct. -//! - [`uffs_content_protocol::COMMAND_PIPE_NAME`] โ€” job submission/resume, -//! `WINDOW_UPDATE`/`FILE_ACK`/`JOB_CANCEL` (consumer to producer), and -//! `PROGRESS`/`HEARTBEAT` (producer to consumer). Always low-volume, so it -//! stays responsive no matter how backed up the data pipe is. -//! -//! # v1 scope: one job at a time -//! -//! This server serves exactly one active job at a time โ€” -//! [`ServerState::active`] is a single slot, not a map. A second -//! `JOB_SUBMIT` while a job is already running is rejected. This is a -//! deliberate, documented scope cut (not a design ceiling): concurrent -//! multi-job serving would need the data pipe to demultiplex frames by -//! `job_id` (today one connection carries exactly one job's stream) and -//! the command pipe to route control signals to the right job's task -//! instead of "the" active job. Revisit if a real multi-job requirement -//! shows up. - -mod command_pipe; -mod pipe_io; -mod stream; - -use alloc::sync::Arc; -use std::sync::Mutex; - -use tokio::sync::mpsc; - -use crate::job::registry::JobRegistry; - -/// A signal the command pipe delivers to the active job's streaming task -/// ([`stream::spawn`]). -pub(crate) enum ControlSignal { - /// `WINDOW_UPDATE`: raise the send budget by this many bytes. - WindowGrant(u64), - /// `FILE_ACK`: candidate id the consumer has durably accepted. - FileAcked(u64), - /// `JOB_CANCEL`: stop streaming. The `String` is a diagnostic reason - /// only. - Cancel(String), -} - -/// Handle to the currently-active job, from the command pipe's point of -/// view. -struct ActiveJob { - /// The producer-assigned id for this job (see [`stream::spawn`]'s doc - /// comment for why the producer, not the consumer, assigns it). - job_id: [u8; 16], - /// Delivers [`ControlSignal`]s to the streaming task. - control_tx: mpsc::Sender, -} - -/// Server-wide shared state, held behind an `Arc` by both the command -/// pipe and every job's streaming task. -struct ServerState { - /// Resume state for whichever job is (or was) active. - registry: Arc, - /// The single active job's control handle, if a job is running. See - /// the module doc for why this is one slot, not a map, in v1. - active: Mutex>, -} - -/// Run the command pipe server for the process's whole lifetime. Each -/// `JOB_SUBMIT` spawns a job-owned data-pipe streaming task -/// ([`stream::spawn`]) alongside it. -/// -/// # Errors -/// Returns an error only if the command pipe itself cannot be created at -/// all. -pub(crate) fn run() -> anyhow::Result<()> { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - rt.block_on(serve()) -} - -/// Async body of [`run`]. -async fn serve() -> anyhow::Result<()> { - let state = Arc::new(ServerState { - registry: Arc::new(JobRegistry::new()), - active: Mutex::new(None), - }); - command_pipe::serve(state).await -} diff --git a/crates/uffs-content/src/serve/pipe_io.rs b/crates/uffs-content/src/serve/pipe_io.rs deleted file mode 100644 index 5a7f1e072..000000000 --- a/crates/uffs-content/src/serve/pipe_io.rs +++ /dev/null @@ -1,250 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Shared named-pipe helpers: pipe-instance creation (owner-only DACL) -//! and `[u32 LE length][payload]`-framed read/write. -//! -//! Reused by both the command pipe and each job's data-pipe connection -//! โ€” mirrors `uffs-content-reader`'s own `pipe_server.rs` helpers (and, -//! further back, `uffs-daemon`'s named-pipe server), just parameterized -//! over the pipe name instead of hardcoding one. - -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; -use tokio::net::windows::named_pipe::{NamedPipeServer, PipeMode, ServerOptions}; - -/// Fixed overhead every encoded frame carries before its payload: the -/// 48-byte envelope header plus its two `u32` checksums (see -/// `uffs_content_protocol::frame::FrameEnvelope`'s own doc comment). -const FRAME_ENVELOPE_OVERHEAD_BYTES: u32 = 56; - -/// `ContentChunk`'s own fixed fields plus its payload's `u32` length -/// prefix, preceding the chunk payload itself: `candidate_id`(8) + -/// `chunk_sequence`(8) + `logical_offset`(8) + `logical_length`(8) + -/// length-prefix(4) = 36. -const CONTENT_CHUNK_FIXED_OVERHEAD_BYTES: u32 = 36; - -/// `FileBegin`'s own fixed fields plus a `WindowsPath`'s own encoding -/// byte + `u32` length prefix, preceding the path bytes themselves: -/// `candidate_id`(8) + `file_reference`(8) + path-encoding(1) + -/// path-length-prefix(4) + `logical_size`(8) + `mtime`(8) + -/// `read_mode`(1) + `attempt_number`(4) + `content_object_id`(1 + 8 -/// worst case) = 51. -const FILE_BEGIN_FIXED_OVERHEAD_BYTES: u32 = 51; - -/// Extra headroom absorbing a small future addition to any frame's fixed -/// fields (e.g. one more optional field) without silently reintroducing -/// the exact ceiling-vs-payload mismatch this constant's derivation -/// exists to prevent. -const FRAME_SIZE_SAFETY_MARGIN_BYTES: u32 = 4096; - -/// `a` if greater, else `b` โ€” `u32::max` as a `const fn`, spelled out -/// directly rather than relying on `Ord::max`'s const-stability version -/// (this workspace's pinned toolchain predates it becoming reliably -/// const across all the targets this crate builds for). -const fn max_u32(lhs: u32, rhs: u32) -> u32 { - if lhs > rhs { lhs } else { rhs } -} - -/// Maximum single framed message size accepted on either pipe. -/// -/// Must comfortably exceed the largest frame this crate can actually -/// produce, so a spec-compliant consumer reader never rejects a -/// legitimate frame. The two candidates for "largest frame" are a full -/// `CONTENT_CHUNK` (`crate::job::workflow::DEFAULT_MAX_CHUNK_BYTES` -/// payload bytes) and a `FILE_BEGIN` carrying a maximum-length path -/// (`uffs_content_protocol::manifest::MAX_PATH_BYTES`) โ€” this ceiling is -/// derived from both plus a safety margin specifically so the three -/// constants can never silently drift out of sync again. A prior version -/// of this constant was a bare `64 * 1024`, smaller than either worst -/// case by construction โ€” any file whose content reached the (also -/// `64 * 1024`) default max chunk size, or any path near the protocol's -/// own 32,767-UTF-16-code-unit maximum, would have produced a frame a -/// consumer built to this same ceiling would reject outright. -pub(super) const MAX_MESSAGE_BYTES: u32 = max_u32( - FRAME_ENVELOPE_OVERHEAD_BYTES - + CONTENT_CHUNK_FIXED_OVERHEAD_BYTES - + crate::job::workflow::DEFAULT_MAX_CHUNK_BYTES, - FRAME_ENVELOPE_OVERHEAD_BYTES - + FILE_BEGIN_FIXED_OVERHEAD_BYTES - + uffs_content_protocol::manifest::MAX_PATH_BYTES, -) + FRAME_SIZE_SAFETY_MARGIN_BYTES; - -/// How long to back off before retrying pipe-instance creation after a -/// transient failure. -const PIPE_RETRY_BACKOFF: core::time::Duration = core::time::Duration::from_millis(100); - -/// Build a single named-pipe server instance bound to `pipe_name` with -/// an owner-only DACL. Set `first = true` ONLY for the initial instance -/// (enables `FIRST_PIPE_INSTANCE` squat protection). -pub(super) fn create_server(pipe_name: &str, first: bool) -> anyhow::Result { - let parsed = uffs_security::pipe::PipeName::parse(pipe_name) - .map_err(|err| anyhow::anyhow!("invalid pipe name {pipe_name:?}: {err}"))?; - let sd = uffs_security::pipe::OwnerOnlySd::for_current_user() - .map_err(|err| anyhow::anyhow!("owner-only DACL build failed: {err}"))?; - let mut sa = sd.as_security_attributes(); - - let mut opts = ServerOptions::new(); - opts.access_inbound(true) - .access_outbound(true) - .pipe_mode(PipeMode::Byte) - .in_buffer_size(65_536) - .out_buffer_size(65_536) - .reject_remote_clients(true); - if first { - opts.first_pipe_instance(true); - } - - // SAFETY: `sa` is a valid `SECURITY_ATTRIBUTES` borrowing a - // `SECURITY_DESCRIPTOR` owned by `sd`, which outlives this call. - #[expect(unsafe_code, reason = "Win32 FFI โ€” create named-pipe server")] - let server = unsafe { - opts.create_with_security_attributes_raw( - parsed.as_str(), - core::ptr::from_mut(&mut sa).cast(), - ) - }?; - Ok(server) -} - -/// Read one `[u32 LE length][payload]`-framed message, or `Ok(None)` on -/// a clean EOF (the peer disconnected between messages). -pub(super) async fn read_one_message( - pipe: &mut NamedPipeServer, -) -> anyhow::Result>> { - let mut length_bytes = [0_u8; 4]; - match pipe.read_exact(&mut length_bytes).await { - Ok(_) => {} - Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), - Err(err) => return Err(err.into()), - } - let length = u32::from_le_bytes(length_bytes); - anyhow::ensure!( - length <= MAX_MESSAGE_BYTES, - "message length {length} exceeds maximum {MAX_MESSAGE_BYTES}" - ); - let mut payload = vec![0_u8; length as usize]; - pipe.read_exact(&mut payload).await?; - Ok(Some(payload)) -} - -/// Write one `[u32 LE length][payload]`-framed message. -pub(super) async fn write_one_message( - pipe: &mut NamedPipeServer, - payload: &[u8], -) -> anyhow::Result<()> { - let length = u32::try_from(payload.len()) - .map_err(|err| anyhow::anyhow!("payload too large to frame: {err}"))?; - pipe.write_all(&length.to_le_bytes()).await?; - pipe.write_all(payload).await?; - pipe.flush().await?; - Ok(()) -} - -/// Repeatedly create a pipe instance and wait for a client to connect, -/// backing off on transient creation failures. Shared by the -/// server-lifetime command pipe and each job's data pipe โ€” both need the -/// exact same "create, back off on failure, wait for connect" sequence, -/// only the pipe name differs. -pub(super) async fn accept_connection( - pipe_name: &str, - first_instance: &mut bool, -) -> NamedPipeServer { - loop { - let pipe = match create_server(pipe_name, *first_instance) { - Ok(pipe) => pipe, - Err(err) => { - tracing::warn!(error = %err, pipe_name, "pipe instance unavailable; retrying shortly"); - tokio::time::sleep(PIPE_RETRY_BACKOFF).await; - continue; - } - }; - *first_instance = false; - if pipe.connect().await.is_ok() { - return pipe; - } - } -} - -/// Render a `job_id` as a short hex string for logging. -pub(super) fn hex_job_id(job_id: [u8; 16]) -> String { - use core::fmt::Write as _; - job_id - .iter() - .fold(String::with_capacity(32), |mut out, byte| { - #[expect( - clippy::let_underscore_must_use, - reason = "String::write_fmt never fails" - )] - let _ = write!(out, "{byte:02x}"); - out - }) -} - -#[cfg(test)] -mod tests { - use uffs_content_protocol::frame::{ - ContentChunk, FileBegin, FrameEnvelope, FrameType, PROTOCOL_VERSION, ReadMode, - }; - use uffs_content_protocol::manifest::MAX_PATH_BYTES; - use uffs_content_protocol::path_encoding::WindowsPath; - - use super::MAX_MESSAGE_BYTES; - use crate::job::workflow::DEFAULT_MAX_CHUNK_BYTES; - - fn encoded_len(frame_type: FrameType, payload: &[u8]) -> u32 { - let bytes = FrameEnvelope { - protocol_version: PROTOCOL_VERSION, - frame_type, - flags: 0, - job_id: [0; 16], - frame_sequence: 0, - } - .encode(payload); - u32::try_from(bytes.len()).unwrap_or(u32::MAX) - } - - /// Locks the exact bug this constant's derivation replaced: a full - /// `CONTENT_CHUNK` at the current max chunk size must always fit - /// under the pipe's own message ceiling. - #[test] - fn max_message_bytes_fits_a_full_content_chunk() { - let payload = vec![0_u8; DEFAULT_MAX_CHUNK_BYTES as usize]; - let chunk = ContentChunk { - candidate_id: u64::MAX, - chunk_sequence: u64::MAX, - logical_offset: u64::MAX, - logical_length: u64::from(DEFAULT_MAX_CHUNK_BYTES), - payload, - }; - let encoded = encoded_len(FrameType::ContentChunk, &chunk.encode()); - assert!( - encoded <= MAX_MESSAGE_BYTES, - "a full CONTENT_CHUNK frame ({encoded} bytes) must fit under \ - MAX_MESSAGE_BYTES ({MAX_MESSAGE_BYTES} bytes)" - ); - } - - /// Same, for a `FILE_BEGIN` carrying the protocol's own maximum path - /// length โ€” the other worst-case frame this ceiling must cover. - #[test] - fn max_message_bytes_fits_a_file_begin_with_a_maximum_length_path() { - let max_code_units = (MAX_PATH_BYTES / 2) as usize; - let path = WindowsPath::from_code_units(vec![u16::from(b'x'); max_code_units]); - let file_begin = FileBegin { - candidate_id: u64::MAX, - file_reference: u64::MAX, - path, - logical_size: u64::MAX, - mtime: i64::MAX, - read_mode: ReadMode::LogicalSnapshot, - attempt_number: u32::MAX, - content_object_id: Some(u64::MAX), - }; - let encoded = encoded_len(FrameType::FileBegin, &file_begin.encode()); - assert!( - encoded <= MAX_MESSAGE_BYTES, - "a maximum-length-path FILE_BEGIN frame ({encoded} bytes) must fit under \ - MAX_MESSAGE_BYTES ({MAX_MESSAGE_BYTES} bytes)" - ); - } -} diff --git a/crates/uffs-content/src/serve/stream.rs b/crates/uffs-content/src/serve/stream.rs deleted file mode 100644 index dd70a2f34..000000000 --- a/crates/uffs-content/src/serve/stream.rs +++ /dev/null @@ -1,714 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Per-job streaming task: owns the data-pipe connection for exactly -//! one job, groups its frames by candidate (file-boundary resume โ€” see -//! design-doc ยง6.5/ยง9.4), and paces emission through a -//! [`crate::job::window::WindowTracker`]. -//! -//! # Why the accept loop lives *with* the job, not as a separate -//! always-on server -//! -//! A data-pipe disconnect mid-candidate must not corrupt the stream: the -//! only correct move is to stop, wait for a fresh connection, and -//! restart that candidate from its `FILE_BEGIN` (never send a partial -//! candidate split across two connections). Owning the accept loop here -//! means "waiting for a (re)connection" and "waiting for send-window -//! budget" are the same kind of pause, handled by the same loop, instead -//! of needing a separate always-on data-pipe server to coordinate with -//! whichever job happens to be active. -//! -//! # Incremental production, not whole-job materialization -//! -//! [`crate::job::vss_job::run_vss_job`] runs on a dedicated blocking -//! thread and emits each encoded frame through a bounded channel -//! ([`FRAME_CHANNEL_CAPACITY`]) as soon as it exists, instead of -//! returning the whole job's frames in one `Vec` โ€” see that function's -//! (and `workflow::run_job`'s) own doc comments for why. This task is -//! the consumer side of that channel: it classifies each arriving frame -//! into [`Grouped`], sends a candidate's group once the send-window -//! admits it, and evicts a candidate's buffered frames the moment a -//! `FILE_ACK` confirms it's no longer needed for a resend. Combined with -//! the channel's own small bounded capacity, peak memory stays close to -//! the send-window size (a small, fixed budget) rather than growing with -//! the job's total content โ€” never the whole job at once. -//! -//! # v1 simplifications, documented rather than silent -//! -//! - Window size is a fixed default, not negotiated per job โ€” see -//! [`DEFAULT_WINDOW_BYTES`]. - -use alloc::sync::Arc; -use std::collections::HashMap; -use std::path::PathBuf; - -use tokio::net::windows::named_pipe::NamedPipeServer; -use tokio::sync::mpsc; -use uffs_content_protocol::DATA_PIPE_NAME; -use uffs_content_protocol::frame::{FrameEnvelope, FrameType, JobBegin}; - -use super::{ActiveJob, ControlSignal, ServerState, pipe_io}; -use crate::job::intake::JobRequest; -use crate::job::vss_job::run_vss_job; -use crate::job::window::WindowTracker; - -/// Default per-job send-window budget (design-doc ยง13.1 -/// `max_unacknowledged_bytes`). Not yet negotiated per job with the -/// consumer (`JOB_SUBMIT`'s payload is just the job spec JSON today) โ€” -/// a fixed, generous default until per-job negotiation is worth adding. -const DEFAULT_WINDOW_BYTES: u64 = 16 * 1024 * 1024; - -/// How many produced-but-not-yet-classified frames the bounded channel -/// between the blocking producer thread and this streaming task may -/// hold before the producer blocks on send. This is what keeps the -/// producer from running arbitrarily far ahead of a stalled consumer โ€” -/// worst case, a full channel holds this many maximum-size frames -/// (~64 KiB each at the default chunk size), a few MiB, nowhere near an -/// entire job's content. -const FRAME_CHANNEL_CAPACITY: usize = 32; - -/// Spawn the streaming task for a freshly submitted job. -/// -/// The producer, not the consumer, assigns the real `job_id`: -/// [`run_vss_job`] already generates a fresh one internally, matching -/// every other call site in this crate, and there is no reason to plumb -/// an externally-chosen id through that already-real, -/// already-validated-on-hardware function just to satisfy a wire -/// nicety. The consumer learns the real `job_id` from `JOB_BEGIN`, the -/// first frame on the data pipe. -pub(super) fn spawn(state: Arc, request: JobRequest, run_dir: PathBuf) { - tokio::spawn(async move { - if let Err(err) = run(&state, request, &run_dir).await { - tracing::error!(error = %err, "job streaming task failed"); - } - }); -} - -/// Async body of [`spawn`]. -async fn run( - state: &Arc, - request: JobRequest, - run_dir: &std::path::Path, -) -> anyhow::Result<()> { - let (frame_tx, mut frame_rx) = mpsc::channel::>(FRAME_CHANNEL_CAPACITY); - let run_dir_owned = run_dir.to_path_buf(); - let job_task = tokio::task::spawn_blocking(move || { - run_vss_job(&request, &run_dir_owned, move |frame| { - frame_tx.blocking_send(frame).map_err(|_err| { - std::io::Error::other("streaming task's frame receiver was dropped") - }) - }) - }); - - // JOB_BEGIN is always the first frame `run_job` emits โ€” receive - // (and classify) frames until it arrives, so `job_id` and - // `candidate_count` are known before anything else happens. - let mut grouped = Grouped::default(); - while grouped.job_begin.is_none() { - match frame_rx.recv().await { - Some(bytes) => grouped.classify_one_frame(bytes), - None => break, - } - } - let Some(job_begin_bytes) = grouped.job_begin.clone() else { - return Err(surface_job_task_error( - job_task.await, - "producer finished without emitting JOB_BEGIN", - )); - }; - let (job_id, candidate_count) = decode_job_begin(&job_begin_bytes)?; - - let candidate_ids: Vec = (1..=candidate_count).collect(); - state.registry.register(job_id, candidate_ids); - - let (control_tx, mut control_rx) = mpsc::channel(32); - set_active(state, Some(ActiveJob { job_id, control_tx })); - - let mut window = WindowTracker::new(DEFAULT_WINDOW_BYTES); - let mut frame_rx_closed = false; - - serve_data_pipe( - state, - job_id, - &job_begin_bytes, - &mut grouped, - &mut frame_rx, - &mut frame_rx_closed, - &mut control_rx, - &mut window, - ) - .await; - set_active(state, None); - state.registry.remove(job_id); - - match job_task.await { - Ok(Ok(_outcome)) => Ok(()), - Ok(Err(err)) => Err(err), - Err(err) => Err(anyhow::anyhow!("streaming task panicked: {err}")), - } -} - -/// Turn the blocking job task's already-resolved result into the real -/// error that made it finish without ever emitting `JOB_BEGIN`, falling -/// back to `fallback` only if the task itself reports success (a -/// producer bug: finishing cleanly without emitting the one frame every -/// job must start with). -fn surface_job_task_error( - job_task_result: Result< - anyhow::Result, - tokio::task::JoinError, - >, - fallback: &str, -) -> anyhow::Error { - match job_task_result { - Ok(Ok(_outcome)) => anyhow::anyhow!("{fallback}"), - Ok(Err(err)) => err, - Err(err) => anyhow::anyhow!("streaming task panicked: {err}"), - } -} - -/// Decode a `JOB_BEGIN` frame's envelope + payload, returning -/// `(job_id, candidate_count)`. -fn decode_job_begin(frame_bytes: &[u8]) -> anyhow::Result<([u8; 16], u64)> { - let mut reader = uffs_content_protocol::codec::Reader::new(frame_bytes); - let (envelope, payload) = FrameEnvelope::decode(&mut reader, u64::MAX) - .map_err(|err| anyhow::anyhow!("failed to decode JOB_BEGIN envelope: {err}"))?; - let mut payload_reader = uffs_content_protocol::codec::Reader::new(&payload); - let job_begin = JobBegin::decode(&mut payload_reader) - .map_err(|err| anyhow::anyhow!("failed to decode JOB_BEGIN payload: {err}"))?; - Ok((envelope.job_id, job_begin.candidate_count)) -} - -/// Outer accept loop: (re)connect the data pipe and stream `job_id`'s -/// frames over it until the job completes or is terminated. Reconnects -/// transparently on a write failure โ€” the only way a partial candidate -/// mid-connection gets resolved is a fresh connection restarting that -/// candidate from its `FILE_BEGIN` (see the module doc's "why the accept -/// loop lives with the job" section). -#[expect( - clippy::too_many_arguments, - reason = "every parameter is per-job state that must persist across every reconnect on \ - this job's data pipe; bundling into a context struct would only move the \ - sprawl, since `stream_over_connection`'s own `wait_for_progress` needs several \ - of these as genuinely disjoint borrows inside a `tokio::select!` (a shared \ - context struct would force one exclusive borrow there instead)" -)] -async fn serve_data_pipe( - state: &Arc, - job_id: [u8; 16], - job_begin: &[u8], - grouped: &mut Grouped, - frame_rx: &mut mpsc::Receiver>, - frame_rx_closed: &mut bool, - control_rx: &mut mpsc::Receiver, - window: &mut WindowTracker, -) { - let mut first_instance = true; - loop { - let mut pipe = pipe_io::accept_connection(DATA_PIPE_NAME, &mut first_instance).await; - tracing::info!(job_id = %pipe_io::hex_job_id(job_id), "consumer connected on data pipe"); - - if pipe_io::write_one_message(&mut pipe, job_begin) - .await - .is_err() - { - continue; - } - - match stream_over_connection( - &mut pipe, - state, - job_id, - grouped, - frame_rx, - frame_rx_closed, - control_rx, - window, - ) - .await - { - ConnectionOutcome::Reconnect => {} - ConnectionOutcome::JobComplete | ConnectionOutcome::Terminated => return, - } - } -} - -/// What ended one data-pipe connection's streaming loop. -enum ConnectionOutcome { - /// Every candidate was acked and `JOB_END` was sent (best-effort). - JobComplete, - /// The consumer cancelled, the control channel died, or the producer - /// ended without ever completing the manifest it committed to โ€” - /// nothing further to send or wait for. - Terminated, - /// The connection dropped mid-stream; the caller should accept a - /// fresh one and resume from wherever the registry says is pending. - Reconnect, -} - -/// Stream `job_id`'s not-yet-acked candidates over `pipe` until it -/// completes, is terminated, or the connection itself fails. -#[expect( - clippy::too_many_arguments, - reason = "see `serve_data_pipe`'s own reason" -)] -async fn stream_over_connection( - pipe: &mut NamedPipeServer, - state: &Arc, - job_id: [u8; 16], - grouped: &mut Grouped, - frame_rx: &mut mpsc::Receiver>, - frame_rx_closed: &mut bool, - control_rx: &mut mpsc::Receiver, - window: &mut WindowTracker, -) -> ConnectionOutcome { - loop { - if state.registry.is_complete(job_id) == Some(true) { - return send_job_end_once_complete( - pipe, - state, - job_id, - grouped, - frame_rx, - frame_rx_closed, - control_rx, - window, - ) - .await; - } - - let candidate_id = match next_pending_candidate_ready_to_send( - state, - job_id, - grouped, - frame_rx, - frame_rx_closed, - control_rx, - window, - ) - .await - { - Ok(id) => id, - Err(outcome) => return outcome, - }; - - let Some(group) = grouped.by_candidate.get(&candidate_id) else { - // Evicted between being selected and being sent โ€” only - // reachable if it was acked without ever being sent, which - // `next_pending_candidate_ready_to_send` never allows; fail - // safe rather than loop forever on it. - tracing::warn!( - candidate_id, - "candidate group vanished before it could be sent" - ); - state.registry.ack(job_id, candidate_id); - continue; - }; - let group_bytes: u64 = group.iter().map(|frame| frame.len() as u64).sum(); - if send_group(pipe, group).await.is_err() { - tracing::info!( - job_id = %pipe_io::hex_job_id(job_id), - "data pipe write failed; waiting for reconnect" - ); - return ConnectionOutcome::Reconnect; - } - window.record_sent(group_bytes); - } -} - -/// Every candidate is acked (the caller already checked -/// `registry.is_complete`) โ€” wait for `JOB_END` itself to have arrived -/// from the producer (it is always the last frame emitted, but the -/// channel may not have delivered it yet) and send it. -#[expect( - clippy::too_many_arguments, - reason = "see `serve_data_pipe`'s own reason" -)] -async fn send_job_end_once_complete( - pipe: &mut NamedPipeServer, - state: &Arc, - job_id: [u8; 16], - grouped: &mut Grouped, - frame_rx: &mut mpsc::Receiver>, - frame_rx_closed: &mut bool, - control_rx: &mut mpsc::Receiver, - window: &mut WindowTracker, -) -> ConnectionOutcome { - while grouped.job_end.is_none() { - if let Err(outcome) = wait_for_progress( - frame_rx, - frame_rx_closed, - control_rx, - state, - job_id, - window, - grouped, - ) - .await - { - return outcome; - } - if *frame_rx_closed && grouped.job_end.is_none() { - tracing::error!( - job_id = %pipe_io::hex_job_id(job_id), - "producer finished without ever emitting JOB_END" - ); - return ConnectionOutcome::Terminated; - } - } - let job_end = grouped.job_end.clone().unwrap_or_default(); - if let Err(err) = pipe_io::write_one_message(pipe, &job_end).await { - tracing::warn!(error = %err, job_id = %pipe_io::hex_job_id(job_id), "failed to send JOB_END"); - } - ConnectionOutcome::JobComplete -} - -/// Find the next not-yet-acked candidate whose complete frame group has -/// both been produced and fits the current send window, waiting on -/// production and/or control signals for as long as neither condition -/// holds yet. -async fn next_pending_candidate_ready_to_send( - state: &Arc, - job_id: [u8; 16], - grouped: &mut Grouped, - frame_rx: &mut mpsc::Receiver>, - frame_rx_closed: &mut bool, - control_rx: &mut mpsc::Receiver, - window: &mut WindowTracker, -) -> Result { - loop { - let Some(candidate_id) = state - .registry - .pending(job_id) - .and_then(|pending| pending.into_iter().next()) - else { - // Nothing left to send this connection (a resume race, or - // every candidate already sent) โ€” wait for an ack that - // completes the job, or a cancel, before re-checking. - wait_for_progress( - frame_rx, - frame_rx_closed, - control_rx, - state, - job_id, - window, - grouped, - ) - .await?; - continue; - }; - if !grouped.by_candidate.contains_key(&candidate_id) { - // Not produced yet โ€” wait for more frames to arrive. - wait_for_progress( - frame_rx, - frame_rx_closed, - control_rx, - state, - job_id, - window, - grouped, - ) - .await?; - if *frame_rx_closed && !grouped.by_candidate.contains_key(&candidate_id) { - // The producer is done and never produced this - // candidate's frames at all โ€” a producer-side bug - // (every registered candidate id came from the same - // manifest `run_job` itself built), not something to - // spin on forever. - tracing::error!( - candidate_id, - job_id = %pipe_io::hex_job_id(job_id), - "producer finished without ever producing this candidate's frames" - ); - return Err(ConnectionOutcome::Terminated); - } - continue; - } - let group_bytes: u64 = grouped.by_candidate.get(&candidate_id).map_or(0, |group| { - group.iter().map(|frame| frame.len() as u64).sum() - }); - while !window.can_admit(group_bytes) { - wait_for_progress( - frame_rx, - frame_rx_closed, - control_rx, - state, - job_id, - window, - grouped, - ) - .await?; - } - return Ok(candidate_id); - } -} - -/// Wait for either the next frame from the producer (classifying it into -/// `grouped`) or the next control signal (applying it), whichever -/// arrives first. Once the producer channel has closed, stops selecting -/// on it โ€” a closed [`mpsc::Receiver`] resolves immediately on every -/// poll, which would otherwise starve the control-signal branch in a -/// tight loop โ€” and only waits on `control_rx` from then on. -async fn wait_for_progress( - frame_rx: &mut mpsc::Receiver>, - frame_rx_closed: &mut bool, - control_rx: &mut mpsc::Receiver, - state: &Arc, - job_id: [u8; 16], - window: &mut WindowTracker, - grouped: &mut Grouped, -) -> Result<(), ConnectionOutcome> { - if *frame_rx_closed { - return match apply_next_signal(control_rx, state, job_id, window, grouped).await { - SignalOutcome::Applied => Ok(()), - SignalOutcome::Cancelled | SignalOutcome::ControlChannelClosed => { - Err(ConnectionOutcome::Terminated) - } - }; - } - tokio::select! { - frame = frame_rx.recv() => { - match frame { - Some(bytes) => grouped.classify_one_frame(bytes), - None => *frame_rx_closed = true, - } - Ok(()) - } - signal = apply_next_signal(control_rx, state, job_id, window, grouped) => { - match signal { - SignalOutcome::Applied => Ok(()), - SignalOutcome::Cancelled | SignalOutcome::ControlChannelClosed => { - Err(ConnectionOutcome::Terminated) - } - } - } - } -} - -/// What happened when [`apply_next_signal`] waited for and applied one -/// [`ControlSignal`]. -enum SignalOutcome { - /// A `WindowGrant` or `FileAcked` signal was applied; the caller - /// should re-check its own loop condition (window budget, pending - /// candidates) since state just changed. - Applied, - /// The consumer sent `JOB_CANCEL`. - Cancelled, - /// The control channel closed โ€” the command pipe's dispatcher (and - /// with it, this job's only path to further acks/cancellation) is - /// gone. - ControlChannelClosed, -} - -/// Block until one [`ControlSignal`] arrives and apply it: a -/// `WindowGrant` raises `window`'s ceiling, a `FileAcked` updates the -/// registry and evicts that candidate's buffered frames from `grouped` -/// (they're never needed again โ€” an ack is a promise the consumer never -/// needs a resend), a `Cancel` is reported (not applied here โ€” the -/// caller owns job teardown). -async fn apply_next_signal( - control_rx: &mut mpsc::Receiver, - state: &Arc, - job_id: [u8; 16], - window: &mut WindowTracker, - grouped: &mut Grouped, -) -> SignalOutcome { - let Some(signal) = control_rx.recv().await else { - return SignalOutcome::ControlChannelClosed; - }; - match signal { - ControlSignal::WindowGrant(additional_bytes) => { - window.grant(additional_bytes); - SignalOutcome::Applied - } - ControlSignal::FileAcked(candidate_id) => { - state.registry.ack(job_id, candidate_id); - grouped.by_candidate.remove(&candidate_id); - SignalOutcome::Applied - } - ControlSignal::Cancel(reason) => { - tracing::info!(job_id = %pipe_io::hex_job_id(job_id), reason, "job cancelled by consumer"); - SignalOutcome::Cancelled - } - } -} - -/// Write every frame in one candidate's group, in order. -async fn send_group(pipe: &mut NamedPipeServer, group: &[Vec]) -> anyhow::Result<()> { - for frame in group { - pipe_io::write_one_message(pipe, frame).await?; - } - Ok(()) -} - -/// Incrementally accumulated frame buckets: the leading `JOB_BEGIN`, each -/// not-yet-acked candidate's frames received so far (or complete, in -/// manifest order), and the trailing `JOB_END` โ€” built one frame at a -/// time via [`Grouped::classify_one_frame`] as frames arrive from the -/// producer, rather than all at once from a prebuilt slice (see the -/// module doc comment). -#[derive(Default)] -struct Grouped { - /// The job's `JOB_BEGIN` frame, once received. - job_begin: Option>, - /// Every other not-yet-evicted frame, bucketed by the `candidate_id` - /// it belongs to, in arrival (= manifest emission) order. A - /// candidate's entry is removed once `FILE_ACK` confirms it's no - /// longer needed for a resend (see [`apply_next_signal`]). - by_candidate: HashMap>>, - /// The job's `JOB_END` frame, once received (always the last frame - /// the producer emits). - job_end: Option>, - /// Which candidate a non-`FILE_BEGIN` frame belongs to โ€” carried - /// across calls to [`Self::classify_one_frame`], mirroring the local - /// `current` variable a whole-slice classifier would keep instead. - current_candidate: Option, -} - -impl Grouped { - /// Classify one already-decoded-length frame into this job's - /// buckets, exactly matching `run_job`'s own emission order - /// (`JOB_BEGIN` first, `JOB_END` last, every per-candidate frame - /// group in between starting with `FILE_BEGIN`). - fn classify_one_frame(&mut self, frame_bytes: Vec) { - let mut reader = uffs_content_protocol::codec::Reader::new(&frame_bytes); - let Ok((envelope, payload)) = FrameEnvelope::decode(&mut reader, u64::MAX) else { - return; - }; - match envelope.frame_type { - FrameType::JobBegin => self.job_begin = Some(frame_bytes), - FrameType::JobEnd => self.job_end = Some(frame_bytes), - FrameType::FileBegin => { - let mut payload_reader = uffs_content_protocol::codec::Reader::new(&payload); - if let Ok(file_begin) = - uffs_content_protocol::frame::FileBegin::decode(&mut payload_reader) - { - self.current_candidate = Some(file_begin.candidate_id); - self.by_candidate - .entry(file_begin.candidate_id) - .or_default() - .push(frame_bytes); - } - } - FrameType::ContentChunk - | FrameType::FileEnd - | FrameType::FileFailed - | FrameType::FileDeferred - | FrameType::FileAck - | FrameType::Progress - | FrameType::Heartbeat - | FrameType::JobCancel - | FrameType::WindowUpdate - | FrameType::JobResume - | FrameType::JobSubmit => { - if let Some(candidate_id) = self.current_candidate { - self.by_candidate - .entry(candidate_id) - .or_default() - .push(frame_bytes); - } - } - } - } -} - -/// Set (or clear) the server's single active-job slot. -fn set_active(state: &Arc, active: Option) { - let mut slot = state - .active - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - *slot = active; -} - -#[cfg(test)] -mod tests { - use uffs_content_protocol::frame::{ - FileBegin, FrameEnvelope, FrameType, JobBegin, PROTOCOL_VERSION, ReadMode, - }; - use uffs_content_protocol::manifest::AuthorizationMode; - use uffs_content_protocol::path_encoding::WindowsPath; - - use super::Grouped; - - const JOB_ID: [u8; 16] = [7; 16]; - - fn encode(frame_sequence: u64, frame_type: FrameType, payload: &[u8]) -> Vec { - FrameEnvelope { - protocol_version: PROTOCOL_VERSION, - frame_type, - flags: 0, - job_id: JOB_ID, - frame_sequence, - } - .encode(payload) - } - - fn file_begin_frame(sequence: u64, candidate_id: u64) -> Vec { - let file_begin = FileBegin { - candidate_id, - file_reference: candidate_id, - path: WindowsPath::from_str_lossless("file.bin"), - logical_size: 0, - mtime: 0, - read_mode: ReadMode::LogicalSnapshot, - attempt_number: 1, - content_object_id: None, - }; - encode(sequence, FrameType::FileBegin, &file_begin.encode()) - } - - #[test] - fn classify_one_frame_buckets_job_begin_and_job_end_separately() { - let mut grouped = Grouped::default(); - let job_begin = JobBegin { - job_id: JOB_ID, - source_id: [0; 16], - snapshot_id: Vec::new(), - snapshot_created_at: 0, - manifest_digest: [0; 32], - candidate_count: 0, - authorization_mode: AuthorizationMode::AdminExport, - ordering: uffs_content_protocol::frame::FrameOrdering::None, - content_semantics: uffs_content_protocol::frame::ContentSemantics::UnnamedLogicalStream, - digest_algorithm: uffs_content_protocol::frame::DigestAlgorithm::Blake3, - max_chunk_bytes: 65536, - max_content_delivery_bytes: None, - }; - let job_begin_bytes = encode(0, FrameType::JobBegin, &job_begin.encode()); - grouped.classify_one_frame(job_begin_bytes.clone()); - assert_eq!(grouped.job_begin, Some(job_begin_bytes)); - assert!(grouped.by_candidate.is_empty()); - assert_eq!(grouped.job_end, None); - } - - #[test] - fn classify_one_frame_groups_frames_under_the_most_recent_file_begin() { - let mut grouped = Grouped::default(); - let begin_1 = file_begin_frame(0, 1); - let chunk_1 = encode(1, FrameType::ContentChunk, b"chunk-for-candidate-1"); - let begin_2 = file_begin_frame(2, 2); - let chunk_2 = encode(3, FrameType::ContentChunk, b"chunk-for-candidate-2"); - - for frame in [ - begin_1.clone(), - chunk_1.clone(), - begin_2.clone(), - chunk_2.clone(), - ] { - grouped.classify_one_frame(frame); - } - - assert_eq!(grouped.by_candidate.get(&1), Some(&vec![begin_1, chunk_1])); - assert_eq!(grouped.by_candidate.get(&2), Some(&vec![begin_2, chunk_2])); - } - - #[test] - fn classify_one_frame_ignores_undecodable_bytes() { - let mut grouped = Grouped::default(); - grouped.classify_one_frame(b"not a valid frame".to_vec()); - assert_eq!(grouped.job_begin, None); - assert!(grouped.by_candidate.is_empty()); - assert_eq!(grouped.job_end, None); - } -} diff --git a/crates/uffs-content/tests/e2e_dir_walk_parity_fake_reader.rs b/crates/uffs-content/tests/e2e_dir_walk_parity_fake_reader.rs deleted file mode 100644 index 86d7e7549..000000000 --- a/crates/uffs-content/tests/e2e_dir_walk_parity_fake_reader.rs +++ /dev/null @@ -1,243 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Fast, cross-platform end-to-end validation: does `uffs-content`'s -//! fake (`std::fs`-backed) pipeline produce exactly the same file set and -//! content as a plain, independent directory walk? -//! -//! This is `uffs-ingest-implementation-plan.md` ยง9.5's "fast" harness โ€” -//! it substitutes `std::fs`-backed candidate/content sources -//! (`uffs_content::job::candidate_source::DirWalkCandidateSource`, -//! `uffs_content::job::content_source::FsContentSource`) for the real -//! VSS-snapshot/privileged-Reader machinery (not yet built โ€” UFI.1/UFI.2), -//! so it runs everywhere, on every PR, with no elevation and no Windows -//! dependency. It still exercises the real Coordinator: candidate -//! enumeration, manifest construction and wire encoding, protocol -//! framing, and the ephemeral run-state bookkeeping โ€” only the "how do -//! we get the candidate list and bytes" step is faked. -//! -//! The real-VSS variant (ยง9.4, Windows-only, `#[ignore]`) is not wired -//! up as an end-to-end test yet โ€” `uffs-content-reader` and the Broker -//! Snapshot Manager both exist now (see `crates/uffs-content-reader/` -//! and `crates/uffs-broker/src/broker/snapshot_manager/`), but nothing -//! yet calls `job::vss_orchestrator`/`job::reader_client` end to end -//! from `workflow::run_job`. - -// `#[cfg(test)]` so clippy's test-code relaxations (`allow-expect-in-tests` -// et al. in `clippy.toml`) apply inside `support/`'s helper files too โ€” -// those relaxations key off the enclosing item chain carrying -// `#[cfg(test)]`, which a plain `mod support;` here would not provide. -#[cfg(test)] -mod support; - -// This crate's own dependencies (shared across the lib, bin, and every -// integration test binary), not used directly from this particular test. -// Windows-only deps, not used directly from this cross-platform fake- -// pipeline test โ€” see `src/main.rs`'s matching markers for the same -// per-target rationale (each test binary is its own compilation unit). -#[cfg(windows)] -use anyhow as _; -// Used by `uffs_content::job::workflow`'s pipelined content reader, -// exercised by this cross-platform test via `run_job` itself. -use crossbeam_channel as _; -use serde as _; -use serde_json as _; -#[cfg(windows)] -use tokio as _; -// Unconditional dependency of `uffs-content` (see that crate's -// `Cargo.toml`) โ€” not named directly by this cross-platform test. -use tracing as _; -#[cfg(windows)] -use tracing_subscriber as _; -#[cfg(windows)] -use uffs_broker_protocol as _; -#[cfg(windows)] -use uffs_client as _; -#[cfg(windows)] -use uffs_content_reader_protocol as _; -#[cfg(windows)] -use uffs_mft as _; -#[cfg(windows)] -use uffs_security as _; -use uffs_version as _; -use uuid as _; - -#[cfg(test)] -mod tests { - use uffs_content::job::candidate_source::DirWalkCandidateSource; - use uffs_content::job::content_source::FsContentSource; - use uffs_content::job::intake::JobRequest; - use uffs_content::job::workflow::{JobOutcome, ReadConcurrency, run_job}; - - use crate::support; - use crate::support::fixture_tree::FixtureFile; - use crate::support::plain_walk::PlainWalkEntry; - use crate::support::test_consumer::ConsumedJob; - - #[test] - fn ingest_output_matches_plain_directory_walk() { - let source_dir = tempfile::tempdir().expect("create source temp dir"); - let fixture_files = support::fixture_tree::build(source_dir.path()); - assert!( - fixture_files - .iter() - .any(|file| file.relative_path.to_string_lossy().contains("hardlink")), - "fixture must include a hard-linked file" - ); - - // 1. Independent oracle โ€” shares no code with the pipeline under test. - let mut expected = support::plain_walk::plain_walk(source_dir.path()); - expected.sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); - assert_fixture_matches_oracle(&fixture_files, &expected); - - // 2. Run the real pipeline against the fake (std::fs) sources. - let run_dir = tempfile::tempdir().expect("create run temp dir"); - let request = JobRequest { - source_id: "fixture-source".to_owned(), - roots: vec![source_dir.path().to_path_buf()], - query: "*".to_owned(), - ..Default::default() - }; - let mut frames = Vec::new(); - let outcome = run_job( - &request, - &DirWalkCandidateSource, - &FsContentSource, - run_dir.path(), - // >1, and smaller than the fixture's own file count, so this - // parity check also exercises the sliding-window concurrent- - // read path (`read_lease_run_pipelined`) with more candidates - // than the window is wide, not just a single pass. - &ReadConcurrency::flat(3), - &[], - 0, - |frame| { - frames.push(frame); - Ok(()) - }, - ) - .expect("run_job must succeed"); - - // 3. Structural assertions (design-doc ยง21.7) before content is even compared. - assert_structural_invariants(&outcome, expected.len()); - - // 4. Decode the actual wire bytes as a real consumer would โ€” this is what - // catches a framing bug a structure-passthrough shortcut would miss - // entirely. - let consumed = support::test_consumer::consume(&outcome.manifest_bytes, &frames); - assert_eq!( - consumed.candidate_count, outcome.run_summary.candidate_count, - "manifest header's candidate_count must match the run summary's" - ); - assert!(consumed.failed_retryable.is_empty()); - assert!(consumed.failed_terminal.is_empty()); - assert!(consumed.deferred_manual.is_empty()); - - assert_content_matches_oracle(&consumed, &expected); - - // 5. Every FILE_END digest must equal recomputing BLAKE3 over the actual - // emitted bytes the consumer buffered โ€” not just the producer's - // self-reported digest โ€” catching a "digest computed over the wrong bytes" - // bug that a self-reported-digest-only check would miss entirely. - assert_digests_recompute(&consumed); - } - - /// Sanity-checks the fixture generator itself before trusting it as - /// the oracle's input: every file it says it wrote must appear in - /// the oracle's own independent walk with matching size/content. - fn assert_fixture_matches_oracle(fixture_files: &[FixtureFile], expected: &[PlainWalkEntry]) { - for file in fixture_files { - let found = expected - .iter() - .find(|entry| entry.relative_path == file.relative_path) - .unwrap_or_else(|| { - panic!( - "fixture file {:?} must appear in the oracle walk", - file.relative_path - ) - }); - assert_eq!( - found.size, - u64::try_from(file.content.len()).unwrap_or(u64::MAX) - ); - assert_eq!( - *found.digest.as_bytes(), - *blake3::hash(&file.content).as_bytes() - ); - } - } - - /// Checks the completeness invariant (design-doc ยง21.7) and that - /// plain, ordinary fixture files never fail or defer. - fn assert_structural_invariants(outcome: &JobOutcome, expected_count: usize) { - assert_eq!( - outcome.run_summary.candidate_count, - outcome.run_summary.succeeded_count - + outcome.run_summary.failed_retryable_count - + outcome.run_summary.failed_terminal_count - + outcome.run_summary.deferred_manual_count - ); - assert_eq!( - outcome.run_summary.failed_retryable_count, 0, - "plain ordinary fixture files must not fail" - ); - assert_eq!( - outcome.run_summary.failed_terminal_count, 0, - "plain ordinary fixture files must not fail" - ); - assert_eq!( - outcome.run_summary.deferred_manual_count, 0, - "plain ordinary fixture files must not defer" - ); - assert_eq!( - outcome.run_summary.candidate_count, - u64::try_from(expected_count).unwrap_or(u64::MAX) - ); - } - - /// The actual "matches a plain dir walk" check: every succeeded - /// candidate's `(path, size, digest)` must exactly match the oracle. - fn assert_content_matches_oracle(consumed: &ConsumedJob, expected: &[PlainWalkEntry]) { - let mut actual: Vec<_> = consumed - .succeeded - .iter() - .map(|file| { - ( - file.relative_path.clone(), - file.total_logical_bytes, - file.reported_digest, - ) - }) - .collect(); - actual.sort(); - let expected_tuples: Vec<_> = expected - .iter() - .map(|entry| { - ( - entry.relative_path.clone(), - entry.size, - *entry.digest.as_bytes(), - ) - }) - .collect(); - assert_eq!( - actual, expected_tuples, - "ingest output must exactly match a plain directory walk" - ); - } - - /// Recomputes BLAKE3 over the consumer-buffered bytes for every - /// succeeded file, independently of the producer's self-reported - /// digest. - fn assert_digests_recompute(consumed: &ConsumedJob) { - for file in &consumed.succeeded { - let recomputed = blake3::hash(&file.buffered_content); - assert_eq!( - *recomputed.as_bytes(), - file.reported_digest, - "producer's self-reported digest must match independent recomputation for {:?}", - file.relative_path - ); - } - } -} diff --git a/crates/uffs-content/tests/e2e_real_vss_content_reader.rs b/crates/uffs-content/tests/e2e_real_vss_content_reader.rs deleted file mode 100644 index 02de4ec92..000000000 --- a/crates/uffs-content/tests/e2e_real_vss_content_reader.rs +++ /dev/null @@ -1,126 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Real-VSS, real-Reader end-to-end playback test โ€” -//! `uffs-ingest-implementation-plan.md` ยง9.4's real-VSS variant, and -//! ยง5.4's "real snapshot, real file, assert bytes match" live test. -//! -//! Thin wrapper: calls -//! [`uffs_content::job::self_test::self_test_vss_playback`] โ€” the exact -//! same production function `uffs-content --self-test-vss-playback` and -//! `scripts/windows/content-reader-validation.rs` both exercise, so none -//! of the three ever drift apart (mirrors `uffs-broker`'s -//! `--self-test-vss` / `cargo test -p uffs-broker -- --ignored` / -//! `scripts/windows/vss-snapshot-validation.rs` trio). -//! -//! # Requirements to actually run this test -//! -//! - Windows, and this test process itself running elevated (VSS snapshot -//! creation and the Reader's `OpenFileById` device-path open both require it -//! โ€” see `job::vss_orchestrator`'s and -//! `uffs-content-reader/src/reader/logical.rs`'s doc comments for why -//! elevation is a deliberate v1 choice, not yet Broker-mediated). -//! - `uffs-broker --install` already run once on the host (or the Broker's -//! Snapshot Manager reachable some other way). -//! - `uffsd` and `uffs-content-reader` built and discoverable next to this test -//! binary (both are spawned as child processes). -//! -//! None of that is available in this workspace's ordinary CI lanes, so -//! this is `#[ignore]` โ€” run explicitly on a prepared Windows box with -//! `cargo test -p uffs-content --test e2e_real_vss_content_reader -- -//! --ignored`. - -#![cfg(test)] - -// This crate's own dependencies (shared across every integration test -// binary in this crate), not used directly from this particular test -// module on every platform. -// These are Windows-only deps of `uffs-content` itself -// (`job::vss_orchestrator`/`ephemeral_daemon`/`reader_client`/ -// `self_test`), reached transitively through `uffs_content::job:: -// self_test::self_test_vss_playback` but not named directly by this -// thin test โ€” same rationale as `src/main.rs`'s matching markers. -#[cfg(windows)] -use anyhow as _; -use blake3 as _; -// Used by `uffs_content::job::workflow`'s pipelined content reader, not -// by this thin test directly (its own real test body is windows-only, -// see below โ€” but the dependency itself is unconditional). -use crossbeam_channel as _; -use serde as _; -use serde_json as _; -// Used only inside the `#[cfg(windows)] mod windows_tests` below โ€” the -// real test body needs Windows, so these are otherwise -// visible-but-unused on every other platform. -#[cfg(not(windows))] -use tempfile as _; -#[cfg(windows)] -use tokio as _; -// Unconditional dependency of `uffs-content` (see that crate's -// `Cargo.toml`) โ€” reached transitively on every platform, not named -// directly by this thin test on either. -use tracing as _; -#[cfg(windows)] -use tracing_subscriber as _; -#[cfg(windows)] -use uffs_broker_protocol as _; -#[cfg(windows)] -use uffs_client as _; -#[cfg(not(windows))] -use uffs_content as _; -use uffs_content_protocol as _; -#[cfg(windows)] -use uffs_content_reader_protocol as _; -#[cfg(windows)] -use uffs_mft as _; -#[cfg(windows)] -use uffs_security as _; -use uffs_version as _; -use uuid as _; - -/// All real test code is Windows-only โ€” see the module doc for why. -#[cfg(windows)] -mod windows_tests { - /// Playback through the real VSS + Reader pipeline must reproduce a - /// freshly created, uniquely-named sample file's content exactly. - /// - /// # Requirements - /// See this file's module doc comment. - #[test] - #[ignore = "requires Windows, elevation, an installed uffs-broker, and \ - uffsd/uffs-content-reader built alongside the test binary"] - fn real_vss_playback_matches_original_file_content() { - let test_dir = tempfile::tempdir().expect("create test dir"); - uffs_content::job::self_test::self_test_vss_playback(test_dir.path()) - .expect("real VSS snapshot + Reader playback round trip must succeed"); - } - - /// An extension-filtered query against a real, pre-existing directory - /// (an arbitrary number of real files, not a synthetic sample) must - /// report metadata and streamed-content totals that exactly match an - /// independent ground-truth filesystem walk. - /// - /// The root directory is necessarily machine-specific (a real drive - /// with real files already on it), so it can't be hardcoded here โ€” - /// set `UFFS_CONTENT_QUERY_TEST_ROOT` (e.g. `G:\`) and, optionally, - /// `UFFS_CONTENT_QUERY_TEST_EXT` (default `txt`). - /// - /// # Requirements - /// See this file's module doc comment, plus `UFFS_CONTENT_QUERY_TEST_ROOT` - /// above. - #[test] - #[ignore = "requires Windows, elevation, an installed uffs-broker, \ - uffsd/uffs-content-reader built alongside the test binary, and \ - UFFS_CONTENT_QUERY_TEST_ROOT set to a real directory"] - fn real_vss_query_metadata_matches_ground_truth_disk_walk() { - let root = std::env::var("UFFS_CONTENT_QUERY_TEST_ROOT") - .expect("set UFFS_CONTENT_QUERY_TEST_ROOT to a real directory, e.g. G:\\"); - let extension = - std::env::var("UFFS_CONTENT_QUERY_TEST_EXT").unwrap_or_else(|_| "txt".to_owned()); - uffs_content::job::self_test::self_test_vss_query_metadata( - std::path::Path::new(&root), - &extension, - ) - .expect("real VSS query metadata/content totals must match ground truth"); - } -} diff --git a/crates/uffs-content/tests/support/fixture_tree.rs b/crates/uffs-content/tests/support/fixture_tree.rs deleted file mode 100644 index 9bd913b4f..000000000 --- a/crates/uffs-content/tests/support/fixture_tree.rs +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Builds a deterministic test directory tree exercising the size -//! classes, encoding edge cases, and hard-link semantics called for by -//! `uffs-ingest-implementation-plan.md` ยง9.2. -//! -//! Deliberately smaller than the plan's full real-VSS test sizes (a real -//! 64 MiB+ file): this harness runs on every PR, so its "large" bucket is -//! scaled down to something that still forces multi-chunk streaming -//! without slowing every CI run. The real-VSS test (ยง9.4, Windows-only, -//! `#[ignore]`) is where the full production size classes belong. - -use std::path::{Path, PathBuf}; - -/// One file this harness deliberately created. -#[derive(Debug, Clone)] -pub(crate) struct FixtureFile { - /// Path relative to the fixture tree's root. - pub relative_path: PathBuf, - /// Exact bytes written. - pub content: Vec, -} - -/// Builds the fixture tree under `root` (must already exist), returning -/// every file it created. -pub(crate) fn build(root: &Path) -> Vec { - let mut files = vec![ - write_file(root, Path::new("zero_byte.dat"), &[]), - write_file( - root, - Path::new("resident_small.txt"), - &deterministic_content("resident_small.txt", 200), - ), - write_file( - root, - Path::new("small_16k.bin"), - &deterministic_content("small_16k.bin", 16 * 1024), - ), - write_file( - root, - Path::new("medium_256k.bin"), - &deterministic_content("medium_256k.bin", 256 * 1024), - ), - write_file( - root, - Path::new("large_2m.bin"), - &deterministic_content("large_2m.bin", 2 * 1024 * 1024), - ), - ]; - - let nested_relative: PathBuf = ["a", "b", "c", "d", "nested.txt"].iter().collect(); - files.push(write_file( - root, - &nested_relative, - &deterministic_content("nested.txt", 512), - )); - - // Non-ASCII / non-BMP name (the emoji requires a UTF-16 surrogate - // pair) โ€” exercises the lossless Windows path encoding. - let unicode_relative = PathBuf::from("ๆ—ฅๆœฌ่ชž_๐Ÿ˜€.txt"); - files.push(write_file( - root, - &unicode_relative, - &deterministic_content("unicode-name-seed", 128), - )); - - // Hard link: two directory entries, one underlying file. - let original_relative = PathBuf::from("hardlink_original.dat"); - let original_content = deterministic_content("hardlink_original.dat", 4096); - files.push(write_file(root, &original_relative, &original_content)); - let linked_relative = PathBuf::from("hardlink_copy.dat"); - std::fs::hard_link(root.join(&original_relative), root.join(&linked_relative)) - .expect("hard_link must succeed"); - files.push(FixtureFile { - relative_path: linked_relative, - content: original_content, - }); - - files -} - -/// Writes `content` at `root.join(relative)`, creating parent directories -/// as needed, and returns the corresponding [`FixtureFile`]. -fn write_file(root: &Path, relative: &Path, content: &[u8]) -> FixtureFile { - let absolute = root.join(relative); - if let Some(parent) = absolute.parent() { - std::fs::create_dir_all(parent).expect("create_dir_all must succeed"); - } - std::fs::write(&absolute, content).expect("write must succeed"); - FixtureFile { - relative_path: relative.to_path_buf(), - content: content.to_vec(), - } -} - -/// Deterministic, non-uniform content: expands a BLAKE3 hash of `seed` -/// via its extendable output, so content is reproducible across runs but -/// not trivially compressible/all-zeros โ€” catching a digest bug that -/// only manifests on repetitive content. -fn deterministic_content(seed: &str, length: usize) -> Vec { - let mut output = vec![0_u8; length]; - let mut hasher = blake3::Hasher::new(); - hasher.update(seed.as_bytes()); - let mut xof_reader = hasher.finalize_xof(); - xof_reader.fill(&mut output); - output -} diff --git a/crates/uffs-content/tests/support/mod.rs b/crates/uffs-content/tests/support/mod.rs deleted file mode 100644 index f0eb04ecc..000000000 --- a/crates/uffs-content/tests/support/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Shared support for the end-to-end dir-walk parity harness -//! (`uffs-ingest-implementation-plan.md` ยง9). - -pub(crate) mod fixture_tree; -pub(crate) mod plain_walk; -pub(crate) mod test_consumer; diff --git a/crates/uffs-content/tests/support/plain_walk.rs b/crates/uffs-content/tests/support/plain_walk.rs deleted file mode 100644 index d257a76be..000000000 --- a/crates/uffs-content/tests/support/plain_walk.rs +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Independent reference directory walk โ€” the end-to-end parity test's -//! oracle. -//! -//! Deliberately does not call into `uffs-content`, `uffs-content-protocol`, -//! `uffs-core`, or `uffs-mft`: the entire point is an implementation that -//! shares no code with the pipeline under test, so a shared bug can't -//! hide from the comparison. - -use std::fs; -use std::path::{Path, PathBuf}; - -/// One file as seen by a plain recursive directory walk. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct PlainWalkEntry { - /// Path relative to the walked root. - pub relative_path: PathBuf, - /// File size in bytes. - pub size: u64, - /// BLAKE3 digest of the file's content, computed directly (not - /// through `uffs-content-protocol::codec::digest`). - pub digest: blake3::Hash, -} - -/// Recursively walks `root`, hashing every regular file's content with -/// `blake3` directly. -#[must_use] -pub(crate) fn plain_walk(root: &Path) -> Vec { - let mut out = Vec::new(); - walk(root, root, &mut out); - out -} - -fn walk(root: &Path, dir: &Path, out: &mut Vec) { - let mut entries: Vec = fs::read_dir(dir) - .expect("read_dir must succeed") - .collect::>() - .expect("collecting read_dir entries must succeed"); - entries.sort_by_key(fs::DirEntry::path); - - for entry in entries { - let path = entry.path(); - let metadata = entry.metadata().expect("metadata must succeed"); - if metadata.is_dir() { - walk(root, &path, out); - } else if metadata.is_file() { - let content = fs::read(&path).expect("read must succeed"); - let relative_path = path.strip_prefix(root).unwrap_or(&path).to_path_buf(); - out.push(PlainWalkEntry { - relative_path, - size: metadata.len(), - digest: blake3::hash(&content), - }); - } - } -} diff --git a/crates/uffs-content/tests/support/test_consumer.rs b/crates/uffs-content/tests/support/test_consumer.rs deleted file mode 100644 index d180d70c4..000000000 --- a/crates/uffs-content/tests/support/test_consumer.rs +++ /dev/null @@ -1,152 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Minimal stand-in for a downstream consumer (e.g. Docenta): decodes the -//! public wire protocol โ€” the manifest and every frame โ€” using only -//! `uffs-content-protocol`'s decoders, exactly as a real consumer would. -//! This is what lets the parity test catch a framing bug that a -//! structure-passthrough shortcut would miss (design-doc ยง21's intent). - -use std::collections::HashMap; -use std::path::PathBuf; - -use uffs_content_protocol::codec::{Digest, Reader}; -use uffs_content_protocol::frame::{ - ContentChunk, FailedOutcome, FileBegin, FileDeferred, FileEnd, FileFailed, FrameEnvelope, - FrameType, -}; -use uffs_content_protocol::manifest::{CandidateRecord, ManifestHeader, ManifestTrailer}; - -/// One candidate's fully consumed success outcome. -#[derive(Debug, Clone)] -pub(crate) struct ConsumedSuccess { - /// Path from the manifest's candidate record. - pub relative_path: PathBuf, - /// `FILE_END.total_logical_bytes`. - pub total_logical_bytes: u64, - /// The producer's self-reported `FILE_END.content_digest`. - pub reported_digest: Digest, - /// Every byte this consumer actually received via `CONTENT_CHUNK` - /// frames for this candidate, in the order received. - pub buffered_content: Vec, -} - -/// Everything decoded from one job's manifest + frame stream. -#[derive(Debug, Clone, Default)] -pub(crate) struct ConsumedJob { - /// `ManifestHeader::candidate_count`. - pub candidate_count: u64, - /// Candidates that reached `FILE_END`. - pub succeeded: Vec, - /// Candidate IDs that reached `FILE_FAILED` with a retryable outcome. - pub failed_retryable: Vec, - /// Candidate IDs that reached `FILE_FAILED` with a terminal outcome. - pub failed_terminal: Vec, - /// Candidate IDs that reached `FILE_DEFERRED`. - pub deferred_manual: Vec, -} - -/// Decode `manifest_bytes` + `frames` (each already a complete, -/// envelope-wrapped frame, as emitted by -/// `uffs_content::job::workflow::run_job`) into a [`ConsumedJob`]. -#[must_use] -pub(crate) fn consume(manifest_bytes: &[u8], frames: &[Vec]) -> ConsumedJob { - let mut manifest_reader = Reader::new(manifest_bytes); - let header = ManifestHeader::decode(&mut manifest_reader).expect("decode manifest header"); - - let mut paths_by_candidate_id: HashMap = HashMap::new(); - for _ in 0..header.candidate_count { - let record = - CandidateRecord::decode(&mut manifest_reader).expect("decode candidate record"); - paths_by_candidate_id.insert( - record.candidate_id, - PathBuf::from(record.path.display_lossy()), - ); - } - ManifestTrailer::decode(&mut manifest_reader).expect("decode manifest trailer"); - - let mut job = ConsumedJob { - candidate_count: header.candidate_count, - ..ConsumedJob::default() - }; - let mut buffers: HashMap> = HashMap::new(); - - for frame_bytes in frames { - let mut frame_reader = Reader::new(frame_bytes); - let (envelope, payload) = - FrameEnvelope::decode(&mut frame_reader, u64::MAX).expect("decode frame envelope"); - let mut payload_reader = Reader::new(&payload); - apply_frame( - envelope.frame_type, - &mut payload_reader, - &paths_by_candidate_id, - &mut buffers, - &mut job, - ); - } - - job -} - -/// Applies one decoded frame to the in-progress [`ConsumedJob`]/buffer -/// state. Split out of [`consume`] purely to keep that function's body -/// short โ€” not a reusable abstraction on its own. -fn apply_frame( - frame_type: FrameType, - payload_reader: &mut Reader<'_>, - paths_by_candidate_id: &HashMap, - buffers: &mut HashMap>, - job: &mut ConsumedJob, -) { - match frame_type { - FrameType::FileBegin => { - let file_begin = FileBegin::decode(payload_reader).expect("decode FILE_BEGIN"); - buffers.insert(file_begin.candidate_id, Vec::new()); - } - FrameType::ContentChunk => { - let chunk = - ContentChunk::decode(payload_reader, u32::MAX).expect("decode CONTENT_CHUNK"); - buffers - .entry(chunk.candidate_id) - .or_default() - .extend_from_slice(&chunk.payload); - } - FrameType::FileEnd => { - let file_end = FileEnd::decode(payload_reader).expect("decode FILE_END"); - let buffered_content = buffers.remove(&file_end.candidate_id).unwrap_or_default(); - let relative_path = paths_by_candidate_id - .get(&file_end.candidate_id) - .cloned() - .unwrap_or_default(); - let reported_digest = file_end - .content_digest - .expect("a succeeded file must report a digest in this harness (no delivery ceiling is set)"); - job.succeeded.push(ConsumedSuccess { - relative_path, - total_logical_bytes: file_end.total_logical_bytes, - reported_digest, - buffered_content, - }); - } - FrameType::FileFailed => { - let file_failed = FileFailed::decode(payload_reader).expect("decode FILE_FAILED"); - match file_failed.outcome { - FailedOutcome::Retryable => job.failed_retryable.push(file_failed.candidate_id), - FailedOutcome::Terminal => job.failed_terminal.push(file_failed.candidate_id), - } - } - FrameType::FileDeferred => { - let file_deferred = FileDeferred::decode(payload_reader).expect("decode FILE_DEFERRED"); - job.deferred_manual.push(file_deferred.candidate_id); - } - FrameType::JobBegin - | FrameType::FileAck - | FrameType::Progress - | FrameType::Heartbeat - | FrameType::JobEnd - | FrameType::JobCancel - | FrameType::WindowUpdate - | FrameType::JobResume - | FrameType::JobSubmit => {} - } -} diff --git a/crates/uffs-mft/src/lcn_resolve.rs b/crates/uffs-mft/src/lcn_resolve.rs index f8f4df93e..3a39595d5 100644 --- a/crates/uffs-mft/src/lcn_resolve.rs +++ b/crates/uffs-mft/src/lcn_resolve.rs @@ -9,7 +9,7 @@ //! whatever order a search happened to return them in (or even sorted by //! ascending FRS, which only weakly-to-moderately correlates with //! physical layout on a volume that's been reorganized over years โ€” see -//! `docs/architecture/content-stream-tool-design.md`) leaves most of a +//! `docs/dev/architecture/content-stream-tool-design.md`) leaves most of a //! drive's achievable seek-distance reduction on the table. Resolving //! true LCN up front and sorting by it captures the rest. //! diff --git a/scripts/windows/content-query-metadata-validation.rs b/scripts/windows/content-query-metadata-validation.rs deleted file mode 100644 index 4efd21962..000000000 --- a/scripts/windows/content-query-metadata-validation.rs +++ /dev/null @@ -1,370 +0,0 @@ -#!/usr/bin/env rust-script -//! ```cargo -//! [dependencies] -//! anyhow = "1.0" -//! colored = "2.0" -//! ``` -// ============================================================================= -// scripts/windows/content-query-metadata-validation โ€” Query Metadata Smoke Test -// ============================================================================= -// -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. -// -// Real, runnable proof that a complex (extension-filtered) query against -// the real VSS + ephemeral-daemon pipeline reports correct metadata and -// streams correct content for an *arbitrary, pre-existing* directory of -// real files โ€” not just the single-synthetic-file case -// `content-reader-validation.rs` covers. Runs `uffs-content -// --self-test-vss-query `, which leases a real VSS snapshot -// of `root`'s drive, spawns the real ephemeral target-selection daemon, -// evaluates a real `ext:` query against it, streams every matching -// candidate's content through the real privileged `uffs-content-reader`, -// and asserts three independent totals against a ground-truth `std::fs` -// walk of the same directory: candidate count, the manifest's own -// `logical_size` sum, and the bytes actually streamed over CONTENT_CHUNK -// frames. -// -// This is a thin wrapper: the round-trip + verification logic lives once, -// in production code, at -// crates/uffs-content/src/job/self_test.rs (`self_test_vss_query_metadata`) -// โ€” the exact same function this script's target and -// `cargo test -p uffs-content -- --ignored` both exercise, so none of the -// three ever drift apart. Mirrors content-reader-validation.rs's own shape. -// -// Requirements: -// - Windows with NTFS -// - Administrator privileges (VSS snapshot creation, and the Reader's -// OpenFileById device-path open, both need it) -// - uffs-content.exe, uffsd.exe, and uffs-content-reader.exe built and -// sitting in the same directory (production install layout, or -// `cargo build --release` output: all three land in target/release/) -// - uffs-broker --install already run once on this host -// - `root` must already exist and contain at least one file with the -// given extension -// -// Usage: -// rust-script scripts/windows/content-query-metadata-validation.rs G:\ txt -// rust-script scripts/windows/content-query-metadata-validation.rs D:\logs log --bin path\to\uffs-content.exe -// rust-script scripts/windows/content-query-metadata-validation.rs G:\ txt --timeout-secs 120 - -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use colored::Colorize; - -/// How long to wait for `--self-test-vss-query` before killing it and -/// reporting a timeout. Streaming an arbitrary, possibly large, real -/// corpus of files (not one synthetic file) can take much longer than -/// `content-reader-validation.rs`'s 60s budget. -const DEFAULT_TIMEOUT_SECS: u64 = 180; - -/// How often the child-process watchdog re-checks `tasklist`. -const WATCHDOG_POLL_INTERVAL: Duration = Duration::from_secs(1); - -/// Every child-process image name the pipeline may spawn, watched by -/// the watchdog so a hang shows *which* stage it's stuck in. -const WATCHED_IMAGES: &[&str] = &["uffsd.exe", "uffs-content-reader.exe"]; - -/// Parsed script arguments. -struct ScriptArgs { - /// Path to the `uffs-content` binary to exercise. - bin: String, - /// Existing directory to query. - root: String, - /// Extension to filter on (no leading dot, e.g. `"txt"`). - extension: String, - /// How long to wait before killing the child and reporting a timeout. - timeout: Duration, -} - -/// Parse CLI args. -/// -/// Usage: `rust-script content-query-metadata-validation -/// [--bin ] [--timeout-secs ]` -fn parse_script_args() -> ScriptArgs { - let args: Vec = std::env::args().collect(); - let mut positional: Vec = Vec::new(); - let mut bin_override: Option = None; - let mut timeout_secs = DEFAULT_TIMEOUT_SECS; - - let mut i = 1; - while i < args.len() { - match args[i].as_str() { - "--bin" | "--binary" => { - bin_override = args.get(i + 1).cloned(); - i += 2; - } - "--timeout-secs" => { - if let Some(value) = args.get(i + 1).and_then(|value| value.parse().ok()) { - timeout_secs = value; - } - i += 2; - } - other if !other.starts_with('-') => { - positional.push(other.to_string()); - i += 1; - } - _ => { - i += 1; - } - } - } - - if positional.len() < 2 { - eprintln!( - "{} usage: content-query-metadata-validation [--bin ] \ - [--timeout-secs ]", - "โœ—".red() - ); - eprintln!(" example: rust-script scripts/windows/content-query-metadata-validation.rs G:\\ txt"); - std::process::exit(1); - } - - ScriptArgs { - bin: bin_override.unwrap_or_else(default_binary), - root: positional[0].clone(), - extension: positional[1].clone(), - timeout: Duration::from_secs(timeout_secs), - } -} - -/// Locate an existing `uffs-content` binary; do **not** auto-build. -/// -/// Search order: -/// 1. `target\release\uffs-content.exe` โ€” `cargo build --release` output -/// 2. `$USERPROFILE\bin\uffs-content.exe` โ€” `just use` install location -/// 3. Bare `uffs-content.exe` โ€” falls through to PATH lookup -fn default_binary() -> String { - let home = std::env::var("USERPROFILE").unwrap_or_else(|_| ".".to_string()); - let candidates = [ - PathBuf::from("target") - .join("release") - .join("uffs-content.exe"), - PathBuf::from(&home).join("bin").join("uffs-content.exe"), - ]; - for candidate in &candidates { - if candidate.exists() { - return candidate.to_string_lossy().into_owned(); - } - } - "uffs-content.exe".to_string() -} - -/// The expected sibling binary paths (`uffsd.exe`, `uffs-content-reader.exe`), -/// mirroring `uffs-content`'s own spawn-a-sibling-binary lookup -/// (`job::ephemeral_daemon::find_daemon_exe` / -/// `job::reader_client::find_reader_exe`) โ€” both must sit next to `bin`. -fn sibling_binary_paths(bin: &str) -> Vec { - let dir = PathBuf::from(bin) - .parent() - .map_or_else(|| PathBuf::from("."), std::path::Path::to_path_buf); - WATCHED_IMAGES - .iter() - .map(|name| dir.join(name)) - .collect() -} - -/// Print ` --version -v` (the long, build-fingerprinted form -/// every UFFS binary supports) before running anything โ€” makes a stale- -/// binary mismatch across the three cooperating processes obvious up -/// front instead of something to reverse-engineer from a hang. -fn print_binary_version(label: &str, path: &std::path::Path) { - match Command::new(path).args(["--version", "-v"]).output() { - Ok(output) if output.status.success() => { - let text = String::from_utf8_lossy(&output.stdout); - for (i, line) in text.lines().enumerate() { - if i == 0 { - eprintln!(" {label} {}", line.cyan()); - } else { - eprintln!(" {} {line}", " ".repeat(label.len())); - } - } - } - Ok(output) => { - eprintln!( - " {label} {} exited {} โ€” {}", - "?".yellow(), - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); - } - Err(err) => { - eprintln!( - " {label} {} not found at {}: {err}", - "โœ—".red(), - path.display() - ); - } - } -} - -/// Whether a process with the given image name currently exists, -/// checked via `tasklist`. -fn process_running(image_name: &str) -> bool { - Command::new("tasklist") - .args(["/FI", &format!("IMAGENAME eq {image_name}"), "/NH"]) - .output() - .is_ok_and(|output| { - String::from_utf8_lossy(&output.stdout) - .to_lowercase() - .contains(&image_name.to_lowercase()) - }) -} - -/// Spawn a background thread that logs each watched image's RUNNING / -/// NOT RUNNING transitions to the terminal, until `stop` is set. -/// Returns the thread's `JoinHandle` so the caller can `stop` then -/// `join` it once the round trip finishes. -fn spawn_process_watchdog(stop: &Arc) -> std::thread::JoinHandle<()> { - let stop = Arc::clone(stop); - std::thread::spawn(move || { - let mut last_seen: Vec> = vec![None; WATCHED_IMAGES.len()]; - while !stop.load(Ordering::Relaxed) { - for (index, image_name) in WATCHED_IMAGES.iter().enumerate() { - let running = process_running(image_name); - if last_seen.get(index).copied().flatten() != Some(running) { - if running { - eprintln!(" [watchdog] {} {image_name}", "RUNNING".green()); - } else { - eprintln!(" [watchdog] {} {image_name}", "NOT RUNNING".yellow()); - } - if let Some(slot) = last_seen.get_mut(index) { - *slot = Some(running); - } - } - } - std::thread::sleep(WATCHDOG_POLL_INTERVAL); - } - }) -} - -fn main() { - let script_start = Instant::now(); - let args = parse_script_args(); - - eprintln!(); - eprintln!("โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—"); - eprintln!("โ•‘ UFFS Content Query Metadata Smoke Test โ•‘"); - eprintln!("โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•"); - eprintln!(" Binary: {}", args.bin.cyan()); - eprintln!(" Root: {}", args.root.cyan()); - eprintln!(" Extension: {}", args.extension.cyan()); - eprintln!(); - - if !cfg!(windows) { - eprintln!( - " {} uffs-content's real VSS + Reader pipeline is Windows-only โ€” nothing to test on this platform.", - "โš ".yellow() - ); - std::process::exit(1); - } - - print_binary_version("uffs-content: ", std::path::Path::new(&args.bin)); - for sibling in sibling_binary_paths(&args.bin) { - let label = format!( - "{}:", - sibling.file_stem().map_or_else( - || "sibling".to_string(), - |stem| stem.to_string_lossy().into_owned() - ) - ); - print_binary_version(&format!("{label:<20}"), &sibling); - } - eprintln!(); - - eprintln!( - " Running: {} --self-test-vss-query {} {} (timeout: {}s)", - args.bin, - args.root, - args.extension, - args.timeout.as_secs() - ); - eprintln!(" โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€"); - - // `Stdio::inherit()` + `.spawn()` โ€” deliberately NOT `.output()`, and - // `.spawn()` (not `.status()`) so the loop below can poll and kill - // on timeout. Same rationale as `content-reader-validation.rs`. - let mut child = match Command::new(&args.bin) - .arg("--self-test-vss-query") - .arg(&args.root) - .arg(&args.extension) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn() - { - Ok(child) => child, - Err(err) => { - eprintln!(" {} failed to spawn {}: {err}", "โœ—".red(), args.bin); - eprintln!( - " (build it first: cargo build --release -p uffs-content -p uffs-daemon -p uffs-content-reader)" - ); - std::process::exit(1); - } - }; - - let watchdog_stop = Arc::new(AtomicBool::new(false)); - let watchdog = spawn_process_watchdog(&watchdog_stop); - - let deadline = Instant::now() + args.timeout; - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break Some(status), - Ok(None) => {} - Err(err) => { - eprintln!(" {} failed to poll child process: {err}", "โœ—".red()); - std::process::exit(1); - } - } - if Instant::now() >= deadline { - eprintln!( - " {} timed out after {}s โ€” killing uffs-content", - "โœ—".red(), - args.timeout.as_secs() - ); - if let Err(err) = child.kill() { - eprintln!(" {} failed to kill timed-out process: {err}", "โœ—".red()); - } - let _ = child.wait(); - break None; - } - std::thread::sleep(Duration::from_millis(200)); - }; - - watchdog_stop.store(true, Ordering::Relaxed); - let _ = watchdog.join(); - - let elapsed_ms = script_start.elapsed().as_millis(); - - eprintln!(" โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€"); - match status { - Some(status) if status.success() => { - eprintln!( - " {} query metadata/content totals matched ground truth ({elapsed_ms}ms)", - "โœ“".green() - ); - eprintln!(); - std::process::exit(0); - } - Some(status) => { - eprintln!( - " {} query metadata/content verification failed ({elapsed_ms}ms)", - "โœ—".red() - ); - eprintln!(); - std::process::exit(status.code().unwrap_or(1)); - } - None => { - eprintln!( - " {} query metadata/content verification timed out ({elapsed_ms}ms) โ€” the \ - last line printed above (and the watchdog log) show where it was stuck", - "โœ—".red() - ); - eprintln!(); - std::process::exit(124); - } - } -} diff --git a/scripts/windows/content-reader-validation.rs b/scripts/windows/content-reader-validation.rs deleted file mode 100644 index 2b32b6d94..000000000 --- a/scripts/windows/content-reader-validation.rs +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env rust-script -//! ```cargo -//! [dependencies] -//! anyhow = "1.0" -//! colored = "2.0" -//! ``` -// ============================================================================= -// scripts/windows/content-reader-validation โ€” Content Reader Playback Smoke Test -// ============================================================================= -// -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. -// -// Real, runnable proof that the whole content pipeline (Broker VSS -// lease -> ephemeral target-selection uffsd -> candidate enumeration -> -// privileged uffs-content-reader -> streamed content) actually works at -// runtime on this machine, not just compiles and links: creates a -// uniquely-named sample file, runs the real job pipeline against it, -// and asserts the played-back bytes exactly match what was written. -// -// This is a thin wrapper: it spawns `uffs-content --self-test-vss-playback -// ` and reports its exit status. The round-trip logic itself -// (create snapshot -> spawn ephemeral daemon -> enumerate candidate -> -// spawn Reader -> stream content -> verify -> tear down) lives once, in -// production code, at crates/uffs-content/src/job/self_test.rs -// (`self_test_vss_playback`) โ€” the exact same function this script's -// target and `cargo test -p uffs-content -- --ignored` both exercise, -// so none of the three ever drift apart. Mirrors -// scripts/windows/vss-snapshot-validation.rs's own shape. -// -// Requirements: -// - Windows with NTFS -// - Administrator privileges (VSS snapshot creation, and the Reader's -// OpenFileById device-path open, both need it) -// - uffs-content.exe, uffsd.exe, and uffs-content-reader.exe built and -// sitting in the same directory (production install layout, or -// `cargo build --release` output: all three land in target/release/) -// - uffs-broker --install already run once on this host -// -// Usage: -// rust-script scripts/windows/content-reader-validation.rs -// rust-script scripts/windows/content-reader-validation.rs C:\Temp\uffs-content-test -// rust-script scripts/windows/content-reader-validation.rs --bin path\to\uffs-content.exe -// rust-script scripts/windows/content-reader-validation.rs --timeout-secs 60 - -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::{Duration, Instant}; - -use colored::Colorize; - -/// How long to wait for `--self-test-vss-playback` before killing it and -/// reporting a timeout, absent a `--timeout-secs` override. The round -/// trip involves a real VSS snapshot create/delete plus spawning two -/// extra child processes (`uffsd`, `uffs-content-reader`), so this is -/// more generous than `vss-snapshot-validation.rs`'s own 30s budget. -const DEFAULT_TIMEOUT_SECS: u64 = 60; - -/// How often the child-process watchdog re-checks `tasklist`. -const WATCHDOG_POLL_INTERVAL: Duration = Duration::from_secs(1); - -/// Every child-process image name the pipeline may spawn, watched by -/// the watchdog so a hang shows *which* stage it's stuck in. -const WATCHED_IMAGES: &[&str] = &["uffsd.exe", "uffs-content-reader.exe"]; - -/// Parsed script arguments. -struct ScriptArgs { - /// Path to the `uffs-content` binary to exercise. - bin: String, - /// Directory the self-test creates its sample file under. - test_dir: String, - /// How long to wait before killing the child and reporting a timeout. - timeout: Duration, -} - -/// Parse CLI args. -/// -/// Usage: `rust-script content-reader-validation [test-dir] [--bin ] -/// [--timeout-secs ]` -fn parse_script_args() -> ScriptArgs { - let args: Vec = std::env::args().collect(); - let mut test_dir: Option = None; - let mut bin_override: Option = None; - let mut timeout_secs = DEFAULT_TIMEOUT_SECS; - - let mut i = 1; - while i < args.len() { - match args[i].as_str() { - "--bin" | "--binary" => { - bin_override = args.get(i + 1).cloned(); - i += 2; - } - "--timeout-secs" => { - if let Some(value) = args.get(i + 1).and_then(|value| value.parse().ok()) { - timeout_secs = value; - } - i += 2; - } - other if !other.starts_with('-') && test_dir.is_none() => { - test_dir = Some(other.to_string()); - i += 1; - } - _ => { - i += 1; - } - } - } - - ScriptArgs { - bin: bin_override.unwrap_or_else(default_binary), - test_dir: test_dir.unwrap_or_else(default_test_dir), - timeout: Duration::from_secs(timeout_secs), - } -} - -/// Locate an existing `uffs-content` binary; do **not** auto-build. -/// -/// Search order: -/// 1. `target\release\uffs-content.exe` โ€” `cargo build --release` output -/// 2. `$USERPROFILE\bin\uffs-content.exe` โ€” `just use` install location -/// 3. Bare `uffs-content.exe` โ€” falls through to PATH lookup -fn default_binary() -> String { - let home = std::env::var("USERPROFILE").unwrap_or_else(|_| ".".to_string()); - let candidates = [ - PathBuf::from("target") - .join("release") - .join("uffs-content.exe"), - PathBuf::from(&home).join("bin").join("uffs-content.exe"), - ]; - for candidate in &candidates { - if candidate.exists() { - return candidate.to_string_lossy().into_owned(); - } - } - "uffs-content.exe".to_string() -} - -/// Default self-test directory: `%TEMP%\uffs-content-self-test`, or -/// `.\uffs-content-self-test` if `TEMP` isn't set. -fn default_test_dir() -> String { - let temp = std::env::var("TEMP") - .or_else(|_| std::env::var("TMP")) - .unwrap_or_else(|_| ".".to_string()); - PathBuf::from(temp) - .join("uffs-content-self-test") - .to_string_lossy() - .into_owned() -} - -/// The expected sibling binary paths (`uffsd.exe`, `uffs-content-reader.exe`), -/// mirroring `uffs-content`'s own spawn-a-sibling-binary lookup -/// (`job::ephemeral_daemon::find_daemon_exe` / -/// `job::reader_client::find_reader_exe`) โ€” both must sit next to `bin`. -fn sibling_binary_paths(bin: &str) -> Vec { - let dir = PathBuf::from(bin) - .parent() - .map_or_else(|| PathBuf::from("."), std::path::Path::to_path_buf); - WATCHED_IMAGES - .iter() - .map(|name| dir.join(name)) - .collect() -} - -/// Print ` --version -v` (the long, build-fingerprinted form -/// every UFFS binary supports) before running anything โ€” makes a stale- -/// binary mismatch across the three cooperating processes obvious up -/// front instead of something to reverse-engineer from a hang, matching -/// `vss-snapshot-validation.rs`'s own rationale. -fn print_binary_version(label: &str, path: &std::path::Path) { - match Command::new(path).args(["--version", "-v"]).output() { - Ok(output) if output.status.success() => { - let text = String::from_utf8_lossy(&output.stdout); - for (i, line) in text.lines().enumerate() { - if i == 0 { - eprintln!(" {label} {}", line.cyan()); - } else { - eprintln!(" {} {line}", " ".repeat(label.len())); - } - } - } - Ok(output) => { - eprintln!( - " {label} {} exited {} โ€” {}", - "?".yellow(), - output.status, - String::from_utf8_lossy(&output.stderr).trim() - ); - } - Err(err) => { - eprintln!( - " {label} {} not found at {}: {err}", - "โœ—".red(), - path.display() - ); - } - } -} - -/// Whether a process with the given image name currently exists, -/// checked via `tasklist`. -fn process_running(image_name: &str) -> bool { - Command::new("tasklist") - .args(["/FI", &format!("IMAGENAME eq {image_name}"), "/NH"]) - .output() - .is_ok_and(|output| { - String::from_utf8_lossy(&output.stdout) - .to_lowercase() - .contains(&image_name.to_lowercase()) - }) -} - -/// Spawn a background thread that logs each watched image's RUNNING / -/// NOT RUNNING transitions to the terminal, until `stop` is set. -/// Returns the thread's `JoinHandle` so the caller can `stop` then -/// `join` it once the round trip finishes. -fn spawn_process_watchdog(stop: &Arc) -> std::thread::JoinHandle<()> { - let stop = Arc::clone(stop); - std::thread::spawn(move || { - let mut last_seen: Vec> = vec![None; WATCHED_IMAGES.len()]; - while !stop.load(Ordering::Relaxed) { - for (index, image_name) in WATCHED_IMAGES.iter().enumerate() { - let running = process_running(image_name); - if last_seen.get(index).copied().flatten() != Some(running) { - if running { - eprintln!(" [watchdog] {} {image_name}", "RUNNING".green()); - } else { - eprintln!(" [watchdog] {} {image_name}", "NOT RUNNING".yellow()); - } - if let Some(slot) = last_seen.get_mut(index) { - *slot = Some(running); - } - } - } - std::thread::sleep(WATCHDOG_POLL_INTERVAL); - } - }) -} - -fn main() { - let script_start = Instant::now(); - let args = parse_script_args(); - - eprintln!(); - eprintln!("โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—"); - eprintln!("โ•‘ UFFS Content Reader Playback Smoke Test โ•‘"); - eprintln!("โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•"); - eprintln!(" Binary: {}", args.bin.cyan()); - eprintln!(" Test dir: {}", args.test_dir.cyan()); - eprintln!(); - - if !cfg!(windows) { - eprintln!( - " {} uffs-content's real VSS + Reader pipeline is Windows-only โ€” nothing to test on this platform.", - "โš ".yellow() - ); - std::process::exit(1); - } - - print_binary_version("uffs-content: ", std::path::Path::new(&args.bin)); - for sibling in sibling_binary_paths(&args.bin) { - let label = format!( - "{}:", - sibling.file_stem().map_or_else( - || "sibling".to_string(), - |stem| stem.to_string_lossy().into_owned() - ) - ); - print_binary_version(&format!("{label:<20}"), &sibling); - } - eprintln!(); - - eprintln!( - " Running: {} --self-test-vss-playback {} (timeout: {}s)", - args.bin, - args.test_dir, - args.timeout.as_secs() - ); - eprintln!(" โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€"); - - // `Stdio::inherit()` + `.spawn()` โ€” deliberately NOT `.output()`, and - // `.spawn()` (not `.status()`) so the loop below can poll and kill - // on timeout. Same rationale as `vss-snapshot-validation.rs`. - let mut child = match Command::new(&args.bin) - .arg("--self-test-vss-playback") - .arg(&args.test_dir) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn() - { - Ok(child) => child, - Err(err) => { - eprintln!(" {} failed to spawn {}: {err}", "โœ—".red(), args.bin); - eprintln!( - " (build it first: cargo build --release -p uffs-content -p uffs-daemon -p uffs-content-reader)" - ); - std::process::exit(1); - } - }; - - let watchdog_stop = Arc::new(AtomicBool::new(false)); - let watchdog = spawn_process_watchdog(&watchdog_stop); - - let deadline = Instant::now() + args.timeout; - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break Some(status), - Ok(None) => {} - Err(err) => { - eprintln!(" {} failed to poll child process: {err}", "โœ—".red()); - std::process::exit(1); - } - } - if Instant::now() >= deadline { - eprintln!( - " {} timed out after {}s โ€” killing uffs-content", - "โœ—".red(), - args.timeout.as_secs() - ); - if let Err(err) = child.kill() { - eprintln!(" {} failed to kill timed-out process: {err}", "โœ—".red()); - } - let _ = child.wait(); - break None; - } - std::thread::sleep(Duration::from_millis(200)); - }; - - watchdog_stop.store(true, Ordering::Relaxed); - let _ = watchdog.join(); - - let elapsed_ms = script_start.elapsed().as_millis(); - - eprintln!(" โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€"); - match status { - Some(status) if status.success() => { - eprintln!( - " {} VSS snapshot + Reader playback round trip passed ({elapsed_ms}ms)", - "โœ“".green() - ); - eprintln!(); - std::process::exit(0); - } - Some(status) => { - eprintln!( - " {} VSS snapshot + Reader playback round trip failed ({elapsed_ms}ms)", - "โœ—".red() - ); - eprintln!(); - std::process::exit(status.code().unwrap_or(1)); - } - None => { - eprintln!( - " {} VSS snapshot + Reader playback round trip timed out ({elapsed_ms}ms) โ€” the \ - last line printed above (and the watchdog log) show where it was stuck", - "โœ—".red() - ); - eprintln!(); - std::process::exit(124); - } - } -} diff --git a/scripts/windows/measure_raw_throughput.rs b/scripts/windows/measure_raw_throughput.rs deleted file mode 100644 index 81142e172..000000000 --- a/scripts/windows/measure_raw_throughput.rs +++ /dev/null @@ -1,373 +0,0 @@ -#!/usr/bin/env rust-script -//! ```cargo -//! [target.'cfg(windows)'.dependencies] -//! windows = { version = "0.62", features = [ -//! "Win32_Foundation", -//! "Win32_Security", -//! "Win32_Storage_FileSystem", -//! "Win32_System_IO", -//! "Win32_System_Ioctl", -//! ] } -//! ``` -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Measures the raw sequential-read throughput floor of a physical -//! volume, independent of `uffs-content`'s per-file read pipeline -- -//! a `dd`-style streaming read straight off the block device. -//! -//! Real-hardware benchmarking kept showing sustained content-read -//! throughput in the single-digit MiB/s range on some drives even -//! after ascending-FRS ordering and true-LCN-sorted ordering were both -//! in place (see `check_frs_vs_lcn.rs`). Without a floor measurement, -//! "the pipeline is slow" and "this drive simply cannot go faster than -//! ~8 MiB/s at this access pattern" are indistinguishable -- this tool -//! answers that by reading raw bytes sequentially off the volume, -//! bypassing candidate enumeration, `OpenFileById`, and every other -//! layer of the real pipeline entirely. -//! -//! Reads three zones by default (outer edge, middle, inner edge of the -//! volume) since HDDs are markedly faster at the outer edge (larger -//! track circumference) than the inner one -- a single-point -//! measurement can be misleadingly optimistic or pessimistic depending -//! on where it happens to land. Reports per-zone MiB/s plus the -//! per-chunk min/max within each zone, so a zone whose *average* looks -//! fine but which stalls badly on some chunks (a sign of bad sectors, -//! thermal throttling, or a drive silently retrying) is still visible. -//! -//! # Usage -//! ```text -//! rust-script scripts/windows/measure_raw_throughput.rs [zone_mib=512] [chunk_mib=4] [device_path] -//! ``` -//! `device_path` optionally overrides `\\.\:` -- pass a VSS -//! snapshot device path (e.g. from a `uffs-broker --run` log's -//! `device=\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopyNNN` line) to -//! measure the same device `uffs-content` actually reads through, -//! rather than the live volume. - -#[cfg(windows)] -mod imp { - use std::time::Instant; - - use windows::Win32::Foundation::{CloseHandle, HANDLE}; - use windows::Win32::Storage::FileSystem::{ - CreateFileW, FILE_BEGIN, FILE_FLAG_SEQUENTIAL_SCAN, FILE_GENERIC_READ, FILE_SHARE_DELETE, - FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING, ReadFile, SetFilePointerEx, - }; - use windows::Win32::System::IO::DeviceIoControl; - use windows::Win32::System::Ioctl::{FSCTL_GET_NTFS_VOLUME_DATA, NTFS_VOLUME_DATA_BUFFER}; - use windows::core::PCWSTR; - - /// One probe location within the volume. - struct Zone { - label: &'static str, - /// Fraction of the volume's total size to seek to before reading - /// (clamped so the read never runs off the end). - fraction: f64, - } - - const ZONES: [Zone; 3] = [ - Zone { - label: "outer edge (start of volume)", - fraction: 0.02, - }, - Zone { - label: "middle of volume", - fraction: 0.50, - }, - Zone { - label: "inner edge (near end of volume)", - fraction: 0.90, - }, - ]; - - pub fn main() { - let args: Vec = std::env::args().collect(); - let Some(drive) = args.get(1) else { - eprintln!( - "usage: measure_raw_throughput.rs [zone_mib=512] [chunk_mib=4] \ - [device_path]\n\ - \n\ - Reads zone_mib of raw sequential data from each of three zones (outer/\n\ - middle/inner) on DriveLetter (or device_path, if given) and reports \n\ - MiB/s -- the raw physical floor, independent of uffs-content's own \n\ - per-file read pipeline." - ); - std::process::exit(2); - }; - let zone_mib: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(512); - let chunk_mib: u64 = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(4); - let device_override = args.get(4).cloned(); - - let path = device_override.unwrap_or_else(|| format!("\\\\.\\{drive}:")); - println!("Opening {path} ..."); - let handle = match open_read_handle(&path) { - Ok(handle) => handle, - Err(err) => { - eprintln!( - "failed to open {path}: {err}\n(needs Administrator to open a raw volume \ - handle)" - ); - std::process::exit(1); - } - }; - - let volume_bytes = match query_size(handle) { - Ok(size) => size, - Err(err) => { - eprintln!("failed to query volume size: {err}"); - close(handle); - std::process::exit(1); - } - }; - println!( - "Volume size: {:.1} GiB", - volume_bytes as f64 / (1024.0 * 1024.0 * 1024.0) - ); - - let chunk_bytes = chunk_mib * 1024 * 1024; - let zone_bytes = (zone_mib * 1024 * 1024).min(volume_bytes / 4); - let mut zone_results = Vec::with_capacity(ZONES.len()); - - for zone in &ZONES { - let max_offset = volume_bytes.saturating_sub(zone_bytes); - let raw_offset = ((volume_bytes as f64 * zone.fraction) as u64).min(max_offset); - // Volume-handle I/O requires a sector-aligned offset (fails - // with ERROR_INVALID_PARAMETER otherwise) even for buffered - // reads -- unlike a regular file, there's no cache-manager - // layer translating an arbitrary byte offset for you. A - // fraction like 0.02 or 0.9 essentially never lands on a - // sector boundary by chance, so round down to a 1 MiB - // boundary, comfortably covering every real sector/stripe - // size in use today. - let offset = align_down(raw_offset, OFFSET_ALIGNMENT); - println!(); - println!( - "=== {} (offset {:.1} GiB, reading {} MiB) ===", - zone.label, - offset as f64 / (1024.0 * 1024.0 * 1024.0), - zone_bytes / (1024 * 1024) - ); - match read_zone(handle, offset, zone_bytes, chunk_bytes) { - Ok(result) => { - println!( - " {:.1} MiB/s average ({} chunks; fastest chunk {:.1} MiB/s, slowest \ - chunk {:.1} MiB/s)", - result.mib_per_sec, - result.chunk_count, - result.fastest_chunk_mib_per_sec, - result.slowest_chunk_mib_per_sec - ); - zone_results.push((zone.label, result.mib_per_sec)); - } - Err(err) => eprintln!(" read failed: {err}"), - } - } - - close(handle); - - if zone_results.is_empty() { - eprintln!("No zone read succeeded -- can't report a floor."); - std::process::exit(1); - } - - println!(); - println!("=== Summary ==="); - let slowest = zone_results - .iter() - .copied() - .fold(f64::INFINITY, |acc, (_, mib)| acc.min(mib)); - let fastest = zone_results - .iter() - .copied() - .fold(0.0_f64, |acc, (_, mib)| acc.max(mib)); - for (label, mib) in &zone_results { - println!(" {label}: {mib:.1} MiB/s"); - } - println!(); - println!( - "Raw sequential floor for this device: ~{slowest:.1} MiB/s (slowest zone) to \ - ~{fastest:.1} MiB/s (fastest zone)." - ); - println!( - "Compare against uffs-content's own \"mib_per_sec_since_job_start\" progress lines \ - for the same drive: if the pipeline number is close to this floor, the drive itself \ - is the bottleneck (ordering/concurrency can't help further); if the pipeline number \ - is far below even the slowest zone here, something in the read pattern (seeking, \ - per-file open/close overhead, fragmentation -- see check_frs_vs_lcn.rs) is still \ - costing real throughput." - ); - } - - /// One zone's read result. - struct ZoneResult { - mib_per_sec: f64, - chunk_count: usize, - fastest_chunk_mib_per_sec: f64, - slowest_chunk_mib_per_sec: f64, - } - - /// Byte offsets and read lengths against a raw volume handle must be - /// sector-aligned (Windows rejects anything else with - /// `ERROR_INVALID_PARAMETER`, even for buffered/cached access) -- - /// 1 MiB comfortably covers every real physical/logical sector or - /// stripe size in use today. - const OFFSET_ALIGNMENT: u64 = 1024 * 1024; - - /// Rounds `value` down to the nearest multiple of `alignment`. - const fn align_down(value: u64, alignment: u64) -> u64 { - value - (value % alignment) - } - - /// Seeks to `offset` and reads `total_bytes` sequentially in - /// `chunk_bytes`-sized calls, timing the whole zone and each - /// individual chunk. Only ever issues full `chunk_bytes`-sized reads - /// -- a short final read would need its own (smaller) alignment - /// reasoning, so any less-than-a-full-chunk remainder is simply left - /// unread rather than risking a second alignment failure mode. - fn read_zone( - handle: HANDLE, - offset: u64, - total_bytes: u64, - chunk_bytes: u64, - ) -> Result { - seek(handle, offset)?; - - let mut buf = vec![0_u8; usize::try_from(chunk_bytes).unwrap_or(4 * 1024 * 1024)]; - let mut remaining = total_bytes; - let mut chunk_count = 0_usize; - let mut fastest_mib_per_sec = 0.0_f64; - let mut slowest_mib_per_sec = f64::INFINITY; - let zone_started_at = Instant::now(); - - while remaining >= chunk_bytes { - let dest = &mut buf[..]; - let chunk_started_at = Instant::now(); - let mut bytes_read = 0_u32; - // SAFETY: `handle` is a valid, open, synchronous file handle - // for the duration of this call; `dest` is a valid, writable - // buffer sized to the requested read length. - let result = - unsafe { ReadFile(handle, Some(dest), Some(&raw mut bytes_read), None) }; - result.map_err(|err| format!("ReadFile failed: {err}"))?; - if bytes_read == 0 { - break; // hit end of volume before filling this zone - } - let chunk_secs = chunk_started_at.elapsed().as_secs_f64(); - if chunk_secs > 0.0 { - let chunk_mib_per_sec = (f64::from(bytes_read) / (1024.0 * 1024.0)) / chunk_secs; - fastest_mib_per_sec = fastest_mib_per_sec.max(chunk_mib_per_sec); - slowest_mib_per_sec = slowest_mib_per_sec.min(chunk_mib_per_sec); - } - chunk_count += 1; - remaining = remaining.saturating_sub(u64::from(bytes_read)); - } - - let zone_secs = zone_started_at.elapsed().as_secs_f64(); - let bytes_actually_read = total_bytes.saturating_sub(remaining); - let mib_per_sec = if zone_secs > 0.0 { - (bytes_actually_read as f64 / (1024.0 * 1024.0)) / zone_secs - } else { - 0.0 - }; - - Ok(ZoneResult { - mib_per_sec, - chunk_count, - fastest_chunk_mib_per_sec: fastest_mib_per_sec, - slowest_chunk_mib_per_sec: if slowest_mib_per_sec.is_finite() { - slowest_mib_per_sec - } else { - 0.0 - }, - }) - } - - /// Opens `path` (a `\\.\:` volume path or a VSS device path) - /// for sequential read access. - fn open_read_handle(path: &str) -> Result { - let wide: Vec = path - .encode_utf16() - .chain(core::iter::once(0)) - .collect(); - // SAFETY: `wide` is UTF-16 and NUL-terminated for the duration of - // this call; no other pointers are passed. - let handle = unsafe { - CreateFileW( - PCWSTR::from_raw(wide.as_ptr()), - FILE_GENERIC_READ.0, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - None, - OPEN_EXISTING, - FILE_FLAG_SEQUENTIAL_SCAN, - None, - ) - }; - handle.map_err(|err| err.to_string()) - } - - /// Total size in bytes of the volume/device behind `handle`. - /// - /// `GetFileSizeEx` does not work on raw volume/device handles (it - /// fails with `ERROR_INVALID_PARAMETER`) -- volume size instead - /// comes from `FSCTL_GET_NTFS_VOLUME_DATA`'s `TotalClusters * - /// BytesPerCluster`, the same ioctl `uffs-mft`'s own - /// `VolumeHandle::get_ntfs_volume_data` uses for the same reason. - fn query_size(handle: HANDLE) -> Result { - let mut volume_data = NTFS_VOLUME_DATA_BUFFER::default(); - let mut bytes_returned: u32 = 0; - let buffer_size = - u32::try_from(size_of::()).unwrap_or(u32::MAX); - - // SAFETY: `handle` is a valid, open volume handle; `volume_data` - // points to valid writable storage of `buffer_size` bytes; and - // `bytes_returned` is a valid out-pointer for the duration of - // this call. - unsafe { - DeviceIoControl( - handle, - FSCTL_GET_NTFS_VOLUME_DATA, - None, - 0, - Some(core::ptr::from_mut(&mut volume_data).cast()), - buffer_size, - Some(&raw mut bytes_returned), - None, - ) - } - .map_err(|err| format!("FSCTL_GET_NTFS_VOLUME_DATA failed: {err}"))?; - - let total_clusters = volume_data.TotalClusters.cast_unsigned(); - let bytes_per_cluster = u64::from(volume_data.BytesPerCluster); - total_clusters - .checked_mul(bytes_per_cluster) - .ok_or_else(|| "volume size overflowed u64".to_owned()) - } - - /// Moves `handle`'s file pointer to `offset` bytes from the start. - fn seek(handle: HANDLE, offset: u64) -> Result<(), String> { - let distance = i64::try_from(offset).map_err(|err| err.to_string())?; - // SAFETY: `handle` is a valid, open handle; no output pointer is - // requested. - unsafe { SetFilePointerEx(handle, distance, None, FILE_BEGIN) } - .map_err(|err| err.to_string()) - } - - /// Closes `handle`, ignoring the (practically infallible) result. - fn close(handle: HANDLE) { - // SAFETY: `handle` was returned by a successful `CreateFileW` - // call above and is closed exactly once, here. - let _ = unsafe { CloseHandle(handle) }; - } -} - -#[cfg(windows)] -fn main() { - imp::main(); -} - -#[cfg(not(windows))] -fn main() { - eprintln!("measure_raw_throughput.rs opens raw Windows volume handles -- Windows only."); - std::process::exit(1); -}