From 59059305101dd3ff2b3932516546f5a301ec8e5b Mon Sep 17 00:00:00 2001 From: share121 Date: Fri, 19 Jun 2026 08:18:17 +0800 Subject: [PATCH 01/26] =?UTF-8?q?wip:=20=E6=9A=82=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/Cargo.toml | 40 +++++ crates/fast-down-api/README.md | 9 ++ crates/fast-down-api/src/config.rs | 150 ++++++++++++++++++ crates/fast-down-api/src/core/download.rs | 35 ++++ crates/fast-down-api/src/core/mod.rs | 4 + crates/fast-down-api/src/core/prefetch.rs | 26 +++ crates/fast-down-api/src/event.rs | 3 + crates/fast-down-api/src/lib.rs | 42 +++++ .../fast-down-api/src/utils/build_header.rs | 12 ++ crates/fast-down-api/src/utils/gen_path.rs | 5 + crates/fast-down-api/src/utils/mod.rs | 5 + 11 files changed, 331 insertions(+) create mode 100644 crates/fast-down-api/Cargo.toml create mode 100644 crates/fast-down-api/README.md create mode 100644 crates/fast-down-api/src/config.rs create mode 100644 crates/fast-down-api/src/core/download.rs create mode 100644 crates/fast-down-api/src/core/mod.rs create mode 100644 crates/fast-down-api/src/core/prefetch.rs create mode 100644 crates/fast-down-api/src/event.rs create mode 100644 crates/fast-down-api/src/lib.rs create mode 100644 crates/fast-down-api/src/utils/build_header.rs create mode 100644 crates/fast-down-api/src/utils/gen_path.rs create mode 100644 crates/fast-down-api/src/utils/mod.rs diff --git a/crates/fast-down-api/Cargo.toml b/crates/fast-down-api/Cargo.toml new file mode 100644 index 0000000..35ae02c --- /dev/null +++ b/crates/fast-down-api/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "fast-down-api" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +description = "A convenient and easy-to-use FFI wrapper for fast-down" +documentation = "https://docs.rs/fast-down-api" +readme = "README.md" +homepage.workspace = true +repository.workspace = true +license.workspace = true +keywords = ["concurrency", "downloader", "parallel", "performance", "thread"] +categories = ["network-programming"] + +[dependencies] +fast-down = { workspace = true, features = ["cookie-store", "fast-puller", "file", "getifaddrs", "mem", "reqwest-tls", "serde"] } +parking_lot = { version = "0.12", features = ["serde"] } +serde = "1.0.156" +tokio = "1.39" +tokio-util = "0.7" +url = "2.5.8" +reqwest = { version = "0.13.2", default-features = false, features = [ + "brotli", + "default-tls", + "deflate", + "gzip", + "http2", + "socks", + "system-proxy", + "zstd", + "cookies", +] } +thiserror = "2.0" +crossfire = "3.0" +toml = "1.1.2" +anyhow = "1.0.102" +mime_guess = "2.0.5" + +[lints] +workspace = true diff --git a/crates/fast-down-api/README.md b/crates/fast-down-api/README.md new file mode 100644 index 0000000..325b1d1 --- /dev/null +++ b/crates/fast-down-api/README.md @@ -0,0 +1,9 @@ +# fast-down-ffi + +[![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) +[![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![Latest version](https://img.shields.io/crates/v/fast-down-ffi.svg)](https://crates.io/crates/fast-down-ffi) +[![Documentation](https://docs.rs/fast-down-ffi/badge.svg)](https://docs.rs/fast-down-ffi) +[![License](https://img.shields.io/crates/l/fast-down-ffi.svg)](https://github.com/fast-down/core/blob/main/LICENSE) + +This library provides a convenient and easy-to-use wrapper around fast-down diff --git a/crates/fast-down-api/src/config.rs b/crates/fast-down-api/src/config.rs new file mode 100644 index 0000000..6d582b9 --- /dev/null +++ b/crates/fast-down-api/src/config.rs @@ -0,0 +1,150 @@ +use fast_down::{ProgressEntry, Proxy}; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, net::IpAddr, sync::Arc, time::Duration}; + +/// File write method for downloaded data. +/// +/// - `Mmap`: memory-mapped I/O (fastest, default) +/// - `Std`: buffered standard file I/O +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub enum WriteMethod { + #[default] + Mmap, + Std, +} + +/// Configuration for a download task. +/// +/// All fields have sensible defaults; see [`Config::default`] for values. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[allow(clippy::struct_excessive_bools)] +pub struct Config { + /// Number of threads. Recommended: `32` / `16` / `8`. More threads does not always mean faster. + pub threads: usize, + /// Proxy setting. Supports https, http, and socks5 proxies. + pub proxy: Proxy, + /// Custom request headers. + pub headers: HashMap, + /// Minimum chunk size in bytes. Recommended: `8 * 1024 * 1024` + /// + /// - Chunks that are too small may cause heavy contention. + /// - When chunking is no longer possible, speculative mode is used. + pub min_chunk_size: u64, + /// Whether to ensure data is fully flushed to disk. Recommended: `false` + /// + /// Set to `true` only if you need to power off immediately after download. + pub sync_all: bool, + /// Write buffer size in bytes. Recommended: `16 * 1024 * 1024` + /// + /// - Only effective for [`WriteMethod::Std`]. Reduces the number of `write` syscalls + /// by batching small writes into larger ones via `BufWriter`. + /// - Not used for [`WriteMethod::Mmap`], as the buffer is managed by the OS. + pub write_buffer_size: usize, + /// Cache high watermark in bytes. Recommended: `16 * 1024 * 1024` + /// + /// When the byte merge buffer reaches this size, a merge flush is triggered + /// to reduce the buffer to `cache_low_watermark` or below. + /// + /// - Only effective for [`WriteMethod::Std`]. + /// - Not used for [`WriteMethod::Mmap`]. + pub cache_high_watermark: usize, + /// Cache low watermark in bytes. Recommended: `8 * 1024 * 1024` + /// + /// After a merge flush, the byte merge buffer size is reduced to this level or below. + /// + /// - Only effective for [`WriteMethod::Std`]. + /// - Not used for [`WriteMethod::Mmap`]. + pub cache_low_watermark: usize, + /// Write queue capacity. Recommended: `10240` + /// + /// If download threads fill the write queue, backpressure is applied to + /// slow down downloads and prevent excessive memory usage. + pub write_queue_cap: usize, + /// Default retry interval after a request failure. Recommended: `500ms` + /// + /// If the server returns a `Retry-After` header, that value takes precedence. + pub retry_gap: Duration, + /// Pull timeout. Recommended: `5000ms` + /// + /// If no bytes are received within `pull_timeout` after sending the request, + /// the connection is dropped and re-established. This helps TCP detect + /// congestion and can improve download speed. + pub pull_timeout: Duration, + /// Whether to accept invalid certificates (dangerous). Recommended: `false` + pub accept_invalid_certs: bool, + /// Whether to accept invalid hostnames (dangerous). Recommended: `false` + pub accept_invalid_hostnames: bool, + /// Write method. Recommended: [`WriteMethod::Mmap`] + /// + /// - [`WriteMethod::Mmap`] is fastest — it delegates writes to the OS, but: + /// 1. On 32-bit systems, the maximum file size is 4 GB, so it automatically + /// falls back to [`WriteMethod::Std`]. + /// 2. Mmap requires the file size to be known and byte-range support from + /// the server; when the `fast_download` flag (set during prefetch) is false, + /// it falls back to [`WriteMethod::Std`]. + /// 3. In rare cases, the OS may cache all data in memory and flush it all + /// at once after the download completes, causing a long post-download delay. + /// - [`WriteMethod::Std`] has the best compatibility. Out-of-order chunks are + /// re-ordered into sequential order by the cache layer before being written. + pub write_method: WriteMethod, + /// Number of retries for fetching metadata. Recommended: `10`. Note: this is not + /// the retry count during download. + pub retry_times: usize, + /// Local IP addresses to bind for outgoing requests. Recommended: `Vec::new()` + /// + /// If you have multiple network interfaces, you can provide their IP addresses; + /// each time the puller is cloned (e.g. on retry or work-stealing) the next + /// address in the list is used. This may not always improve speed. + pub local_address: Vec, + /// Maximum number of speculative workers. Recommended: `3` + /// + /// When the remaining chunk is smaller than `min_chunk_size` and cannot be split, + /// speculative mode is used. Up to `max_speculative` workers compete on the same + /// chunk to prevent the download from stalling near 99%. + pub max_speculative: usize, + /// Already downloaded chunks. Pass `Vec::new()` to download the entire file. + pub downloaded_chunk: Arc>>, + /// Smoothing window for downloaded chunks in bytes. Recommended: `8 * 1024` + /// + /// Filters out small gaps in `downloaded_chunk` that are smaller than + /// `chunk_window` to reduce the number of HTTP requests. + pub chunk_window: u64, + /// Maximum number of redirects. Recommended value: `20` + pub max_redirects: usize, + /// Enable cookie store. When `true`, the client will automatically save + /// `Set-Cookie` headers from responses and send matching cookies in + /// subsequent requests (including across redirects). + /// + /// Requires the `cookie-store` feature. When the feature is disabled, + /// setting this to `true` has no effect. + pub cookie_store: bool, +} + +impl Default for Config { + fn default() -> Self { + Self { + retry_times: 10, + threads: 32, + proxy: Proxy::System, + headers: HashMap::new(), + sync_all: false, + min_chunk_size: 8 * 1024 * 1024, + write_buffer_size: 16 * 1024 * 1024, + cache_high_watermark: 16 * 1024 * 1024, + cache_low_watermark: 8 * 1024 * 1024, + write_queue_cap: 10240, + retry_gap: Duration::from_millis(500), + pull_timeout: Duration::from_secs(5), + accept_invalid_certs: false, + accept_invalid_hostnames: false, + local_address: Vec::new(), + max_speculative: 3, + write_method: WriteMethod::Mmap, + downloaded_chunk: Arc::default(), + chunk_window: 8 * 1024, + max_redirects: 20, + cookie_store: false, + } + } +} diff --git a/crates/fast-down-api/src/core/download.rs b/crates/fast-down-api/src/core/download.rs new file mode 100644 index 0000000..66a45f6 --- /dev/null +++ b/crates/fast-down-api/src/core/download.rs @@ -0,0 +1,35 @@ +use crate::{Config, Tx, prefetch::prefetch, utils::build_header}; +use fast_down::{fast_puller::build_client, handle::SharedHandle}; +use tokio_util::sync::CancellationToken; +use url::Url; + +pub struct DownloadHandle { + handle: SharedHandle<()>, +} + +impl DownloadHandle { + /// # Errors + pub fn new( + url: Url, + config: Config, + tx: Tx, + cancel_token: CancellationToken, + ) -> anyhow::Result { + let client = build_client( + build_header(&config.headers), + config.proxy.as_deref(), + config.accept_invalid_certs, + config.accept_invalid_hostnames, + config.cookie_store, + config.local_address.first().copied(), + config.max_redirects, + )?; + let handle = tokio::spawn(async move { + let Some((url_info, resp)) = prefetch(&url, &config, &client, &tx).await else { + return; + }; + }); + let handle = SharedHandle::new(handle); + Ok(Self { handle }) + } +} diff --git a/crates/fast-down-api/src/core/mod.rs b/crates/fast-down-api/src/core/mod.rs new file mode 100644 index 0000000..65e12f6 --- /dev/null +++ b/crates/fast-down-api/src/core/mod.rs @@ -0,0 +1,4 @@ +mod download; +pub(crate) mod prefetch; + +pub use download::*; diff --git a/crates/fast-down-api/src/core/prefetch.rs b/crates/fast-down-api/src/core/prefetch.rs new file mode 100644 index 0000000..9fb194c --- /dev/null +++ b/crates/fast-down-api/src/core/prefetch.rs @@ -0,0 +1,26 @@ +use crate::{Config, Event, Tx}; +use fast_down::{UrlInfo, http::Prefetch, reqwest::SmartRedirectClient}; +use reqwest::Response; +use url::Url; + +pub async fn prefetch( + url: &Url, + config: &Config, + client: &SmartRedirectClient, + tx: &Tx, +) -> Option<(UrlInfo, Response)> { + let mut retry_count = 0; + loop { + match client.prefetch(url.clone()).await { + Ok(t) => break Some(t), + Err((e, t)) => { + let _ = tx.send(Event::PrefetchError(e.into())); + retry_count += 1; + if retry_count >= config.retry_times { + return None; + } + tokio::time::sleep(t.unwrap_or(config.retry_gap)).await; + } + } + } +} diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs new file mode 100644 index 0000000..90a8830 --- /dev/null +++ b/crates/fast-down-api/src/event.rs @@ -0,0 +1,3 @@ +pub enum Event { + PrefetchError(anyhow::Error), +} diff --git a/crates/fast-down-api/src/lib.rs b/crates/fast-down-api/src/lib.rs new file mode 100644 index 0000000..c0afe7d --- /dev/null +++ b/crates/fast-down-api/src/lib.rs @@ -0,0 +1,42 @@ +#![doc = include_str!("../README.md")] + +mod config; +mod core; +mod event; +pub(crate) mod utils; + +pub use config::*; +pub use core::*; +pub use event::*; + +pub use fast_down; +pub use fast_down::{ + AnyError, BoxPusher, CacheDirectPusher, CacheMergePusher, CacheSeqPusher, DownloadResult, + Event as RawEvent, FileId, InvertIter, Merge, ProgressEntry, Proxy, PullResult, PullStream, + Puller, PullerError, Pusher, Total, UrlInfo, WorkerId, fast_puller, file, getifaddrs, handle, + http, invert, mem, mock, multi, reqwest as reqwest_adapter, single, +}; + +use tokio_util::sync::CancellationToken; + +/// Sender half of the event channel, used to push [`Event`]s from the download task. +pub type Tx = crossfire::MTx>; +/// Receiver half of the event channel, used to receive [`Event`]s from the download task. +pub type Rx = crossfire::MAsyncRx>; + +/// Create a new unbounded event channel for receiving download progress events. +/// +/// Returns a sender (`Tx`) and receiver (`Rx`) pair. +#[must_use] +pub fn create_channel() -> (Tx, Rx) { + crossfire::mpmc::unbounded_async() +} + +/// Create a new cancellation token for use with download tasks. +/// +/// Pass the token to any `DownloadTask` method (`start`, `start_with_pusher`, +/// or `start_in_memory`) to cancel the download at any time. +#[must_use] +pub fn create_cancellation_token() -> CancellationToken { + CancellationToken::new() +} diff --git a/crates/fast-down-api/src/utils/build_header.rs b/crates/fast-down-api/src/utils/build_header.rs new file mode 100644 index 0000000..30a5857 --- /dev/null +++ b/crates/fast-down-api/src/utils/build_header.rs @@ -0,0 +1,12 @@ +use reqwest::header::{HeaderMap, HeaderName}; +use std::{collections::HashMap, str::FromStr}; + +pub fn build_header(headers: &HashMap) -> HeaderMap { + let mut result = HeaderMap::with_capacity(headers.len()); + for (k, v) in headers { + if let (Ok(k), Ok(v)) = (HeaderName::from_str(k), v.parse()) { + result.insert(k, v); + } + } + result +} diff --git a/crates/fast-down-api/src/utils/gen_path.rs b/crates/fast-down-api/src/utils/gen_path.rs new file mode 100644 index 0000000..44f8971 --- /dev/null +++ b/crates/fast-down-api/src/utils/gen_path.rs @@ -0,0 +1,5 @@ +use crate::Config; +use fast_down::UrlInfo; +use std::path::PathBuf; + +pub fn gen_path(url_info: UrlInfo, config: Config) -> PathBuf {} diff --git a/crates/fast-down-api/src/utils/mod.rs b/crates/fast-down-api/src/utils/mod.rs new file mode 100644 index 0000000..15c0650 --- /dev/null +++ b/crates/fast-down-api/src/utils/mod.rs @@ -0,0 +1,5 @@ +mod build_header; +mod gen_path; + +pub use build_header::*; +pub use gen_path::*; From a7d2a61f45252b2d913bfc6b49817883405d4d0f Mon Sep 17 00:00:00 2001 From: share121 Date: Mon, 20 Jul 2026 23:28:47 +0800 Subject: [PATCH 02/26] =?UTF-8?q?feat:=20=E5=88=9D=E6=AD=A5=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=E4=B8=8B=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/Cargo.toml | 25 +- crates/fast-down-api/src/config.rs | 90 +++--- crates/fast-down-api/src/core/download.rs | 280 +++++++++++++++++- crates/fast-down-api/src/core/mod.rs | 4 +- crates/fast-down-api/src/core/state.rs | 80 +++++ crates/fast-down-api/src/event.rs | 26 ++ crates/fast-down-api/src/lib.rs | 6 - .../src/utils/filename_template.rs | 36 +++ crates/fast-down-api/src/utils/force_send.rs | 29 ++ crates/fast-down-api/src/utils/gen_path.rs | 38 ++- crates/fast-down-api/src/utils/mod.rs | 5 + crates/fast-down-api/src/utils/tx_err.rs | 13 + 12 files changed, 572 insertions(+), 60 deletions(-) create mode 100644 crates/fast-down-api/src/core/state.rs create mode 100644 crates/fast-down-api/src/utils/filename_template.rs create mode 100644 crates/fast-down-api/src/utils/force_send.rs create mode 100644 crates/fast-down-api/src/utils/tx_err.rs diff --git a/crates/fast-down-api/Cargo.toml b/crates/fast-down-api/Cargo.toml index 35ae02c..ee4c61c 100644 --- a/crates/fast-down-api/Cargo.toml +++ b/crates/fast-down-api/Cargo.toml @@ -13,13 +13,13 @@ keywords = ["concurrency", "downloader", "parallel", "performance", "thread"] categories = ["network-programming"] [dependencies] -fast-down = { workspace = true, features = ["cookie-store", "fast-puller", "file", "getifaddrs", "mem", "reqwest-tls", "serde"] } -parking_lot = { version = "0.12", features = ["serde"] } -serde = "1.0.156" -tokio = "1.39" +fast-down = { workspace = true, features = ["cookie-store", "fast-puller", "file", "getifaddrs", "mem", "reqwest-tls"] } +parking_lot.workspace = true +tokio.workspace = true tokio-util = "0.7" -url = "2.5.8" -reqwest = { version = "0.13.2", default-features = false, features = [ +url.workspace = true +serde = { version = "1.0.228", features = ["serde_derive"] } +reqwest = { version = "0.13.4", default-features = false, features = [ "brotli", "default-tls", "deflate", @@ -30,11 +30,16 @@ reqwest = { version = "0.13.2", default-features = false, features = [ "zstd", "cookies", ] } -thiserror = "2.0" -crossfire = "3.0" -toml = "1.1.2" -anyhow = "1.0.102" +thiserror.workspace = true +crossfire.workspace = true +anyhow = "1.0.103" mime_guess = "2.0.5" +path_helper = { version = "0.1.3", features = ["auto_ext", "sanitize", "tokio"], path = "../../../../path_helper" } +chrono = "0.4.45" +urlencoding = "2.1.3" +soft-canonicalize = { version = "0.5.6", features = ["dunce"] } +inherit-config = "0.2.2" +toml = "1.1.2" [lints] workspace = true diff --git a/crates/fast-down-api/src/config.rs b/crates/fast-down-api/src/config.rs index 6d582b9..bfe750c 100644 --- a/crates/fast-down-api/src/config.rs +++ b/crates/fast-down-api/src/config.rs @@ -1,7 +1,7 @@ use fast_down::{ProgressEntry, Proxy}; -use parking_lot::Mutex; +use inherit_config::InheritConfig; use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, net::IpAddr, sync::Arc, time::Duration}; +use std::{collections::HashMap, net::IpAddr, path::PathBuf, time::Duration}; /// File write method for downloaded data. /// @@ -17,30 +17,51 @@ pub enum WriteMethod { /// Configuration for a download task. /// /// All fields have sensible defaults; see [`Config::default`] for values. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, InheritConfig)] #[allow(clippy::struct_excessive_bools)] pub struct Config { + /// 保存的文件夹 + pub save_dir: PathBuf, + + /// 文件名解析 + pub parse_filename: bool, + + /// 文件名 + pub filename: String, + + /// 用于在 prefetch 阶段生成占位文件名 + pub gid: String, + /// Number of threads. Recommended: `32` / `16` / `8`. More threads does not always mean faster. + #[config(default = 32)] pub threads: usize, + /// Proxy setting. Supports https, http, and socks5 proxies. pub proxy: Proxy, + /// Custom request headers. pub headers: HashMap, + /// Minimum chunk size in bytes. Recommended: `8 * 1024 * 1024` /// /// - Chunks that are too small may cause heavy contention. /// - When chunking is no longer possible, speculative mode is used. + #[config(default = 8 * 1024 * 1024)] pub min_chunk_size: u64, + /// Whether to ensure data is fully flushed to disk. Recommended: `false` /// /// Set to `true` only if you need to power off immediately after download. pub sync_all: bool, + /// Write buffer size in bytes. Recommended: `16 * 1024 * 1024` /// /// - Only effective for [`WriteMethod::Std`]. Reduces the number of `write` syscalls /// by batching small writes into larger ones via `BufWriter`. /// - Not used for [`WriteMethod::Mmap`], as the buffer is managed by the OS. + #[config(default = 16 * 1024 * 1024)] pub write_buffer_size: usize, + /// Cache high watermark in bytes. Recommended: `16 * 1024 * 1024` /// /// When the byte merge buffer reaches this size, a merge flush is triggered @@ -48,33 +69,45 @@ pub struct Config { /// /// - Only effective for [`WriteMethod::Std`]. /// - Not used for [`WriteMethod::Mmap`]. + #[config(default = 16 * 1024 * 1024)] pub cache_high_watermark: usize, + /// Cache low watermark in bytes. Recommended: `8 * 1024 * 1024` /// /// After a merge flush, the byte merge buffer size is reduced to this level or below. /// /// - Only effective for [`WriteMethod::Std`]. /// - Not used for [`WriteMethod::Mmap`]. + #[config(default = 8 * 1024 * 1024)] pub cache_low_watermark: usize, + /// Write queue capacity. Recommended: `10240` /// /// If download threads fill the write queue, backpressure is applied to /// slow down downloads and prevent excessive memory usage. + #[config(default = 10240)] pub write_queue_cap: usize, + /// Default retry interval after a request failure. Recommended: `500ms` /// /// If the server returns a `Retry-After` header, that value takes precedence. + #[config(default = Duration::from_millis(500))] pub retry_gap: Duration, - /// Pull timeout. Recommended: `5000ms` + + /// Pull timeout. Recommended: `5s` /// /// If no bytes are received within `pull_timeout` after sending the request, /// the connection is dropped and re-established. This helps TCP detect /// congestion and can improve download speed. + #[config(default = Duration::from_secs(5))] pub pull_timeout: Duration, + /// Whether to accept invalid certificates (dangerous). Recommended: `false` pub accept_invalid_certs: bool, + /// Whether to accept invalid hostnames (dangerous). Recommended: `false` pub accept_invalid_hostnames: bool, + /// Write method. Recommended: [`WriteMethod::Mmap`] /// /// - [`WriteMethod::Mmap`] is fastest — it delegates writes to the OS, but: @@ -88,63 +121,50 @@ pub struct Config { /// - [`WriteMethod::Std`] has the best compatibility. Out-of-order chunks are /// re-ordered into sequential order by the cache layer before being written. pub write_method: WriteMethod, + /// Number of retries for fetching metadata. Recommended: `10`. Note: this is not /// the retry count during download. + #[config(default = 10)] pub retry_times: usize, + /// Local IP addresses to bind for outgoing requests. Recommended: `Vec::new()` /// /// If you have multiple network interfaces, you can provide their IP addresses; /// each time the puller is cloned (e.g. on retry or work-stealing) the next /// address in the list is used. This may not always improve speed. pub local_address: Vec, + /// Maximum number of speculative workers. Recommended: `3` /// /// When the remaining chunk is smaller than `min_chunk_size` and cannot be split, /// speculative mode is used. Up to `max_speculative` workers compete on the same /// chunk to prevent the download from stalling near 99%. + #[config(default = 3)] pub max_speculative: usize, + /// Already downloaded chunks. Pass `Vec::new()` to download the entire file. - pub downloaded_chunk: Arc>>, + pub downloaded_chunk: Vec, + /// Smoothing window for downloaded chunks in bytes. Recommended: `8 * 1024` /// /// Filters out small gaps in `downloaded_chunk` that are smaller than /// `chunk_window` to reduce the number of HTTP requests. + #[config(default = 8 * 1024)] pub chunk_window: u64, + /// Maximum number of redirects. Recommended value: `20` + #[config(default = 20)] pub max_redirects: usize, + /// Enable cookie store. When `true`, the client will automatically save /// `Set-Cookie` headers from responses and send matching cookies in /// subsequent requests (including across redirects). - /// - /// Requires the `cookie-store` feature. When the feature is disabled, - /// setting this to `true` has no effect. pub cookie_store: bool, -} -impl Default for Config { - fn default() -> Self { - Self { - retry_times: 10, - threads: 32, - proxy: Proxy::System, - headers: HashMap::new(), - sync_all: false, - min_chunk_size: 8 * 1024 * 1024, - write_buffer_size: 16 * 1024 * 1024, - cache_high_watermark: 16 * 1024 * 1024, - cache_low_watermark: 8 * 1024 * 1024, - write_queue_cap: 10240, - retry_gap: Duration::from_millis(500), - pull_timeout: Duration::from_secs(5), - accept_invalid_certs: false, - accept_invalid_hostnames: false, - local_address: Vec::new(), - max_speculative: 3, - write_method: WriteMethod::Mmap, - downloaded_chunk: Arc::default(), - chunk_window: 8 * 1024, - max_redirects: 20, - cookie_store: false, - } - } + /// 是否尝试断点续传，推荐值: `true` + #[config(default = true)] + pub resume: bool, + + /// 是否覆盖已存在的文件，推荐值: `false` + pub overwrite: bool, } diff --git a/crates/fast-down-api/src/core/download.rs b/crates/fast-down-api/src/core/download.rs index 66a45f6..81a1295 100644 --- a/crates/fast-down-api/src/core/download.rs +++ b/crates/fast-down-api/src/core/download.rs @@ -1,5 +1,26 @@ -use crate::{Config, Tx, prefetch::prefetch, utils::build_header}; -use fast_down::{fast_puller::build_client, handle::SharedHandle}; +use crate::{ + Config, DownloadState, Event, PartialConfig, Tx, WriteMethod, + prefetch::prefetch, + tx_err, + utils::{ForceSendExt, build_header, gen_path}, +}; +use fast_down::{ + BoxPusher, UrlInfo, + fast_puller::{FastDownPuller, FastDownPullerOptions, build_client}, + file::{CacheFilePusher, MmapFilePusher}, + handle::SharedHandle, + multi::download_multi, + single::download_single, +}; +use inherit_config::ConfigLayer; +use parking_lot::Mutex; +use path_helper::FileStemExt; +use reqwest::Response; +use std::sync::Arc; +use tokio::{ + fs::{self, File, OpenOptions}, + select, +}; use tokio_util::sync::CancellationToken; use url::Url; @@ -9,12 +30,13 @@ pub struct DownloadHandle { impl DownloadHandle { /// # Errors - pub fn new( + pub fn download( url: Url, - config: Config, + mut partial_config: PartialConfig, tx: Tx, cancel_token: CancellationToken, ) -> anyhow::Result { + let config = partial_config.clone().build(); let client = build_client( build_header(&config.headers), config.proxy.as_deref(), @@ -25,11 +47,259 @@ impl DownloadHandle { config.max_redirects, )?; let handle = tokio::spawn(async move { - let Some((url_info, resp)) = prefetch(&url, &config, &client, &tx).await else { + let Some((info, resp)) = prefetch(&url, &config, &client, &tx).await else { return; }; + let origin_final_path = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError); + let mut tmp_path = origin_final_path.with_added_extension("part"); + let mut config_path = origin_final_path.with_added_extension("fd"); + let can_resume = config.resume && info.fast_download; + let mut no_create_option = OpenOptions::new(); + no_create_option + .read(true) + .write(true) + .truncate(false) + .create(false); + let mut create_option = OpenOptions::new(); + create_option + .read(true) + .write(true) + .truncate(false) + .create(true); + let mut only_create_option = OpenOptions::new(); + only_create_option + .read(true) + .write(true) + .truncate(false) + .create_new(true); + if config.overwrite { + if can_resume + && let (Ok(file), Ok(state)) = tokio::join!( + no_create_option.open(&tmp_path), + DownloadState::load(&config_path) + ) + { + if let Some(config) = &state.config { + partial_config.inherit_from(config); + } + let _ = tx.send(Event::Start { + tmp_path: tmp_path.clone(), + config_path, + url_info: info.clone(), + parsed_config: partial_config.clone(), + }); + Self::overwrite( + file, + url, + partial_config.build(), + info, + Some(Arc::new(Mutex::new(Some(resp)))), + tx.clone(), + cancel_token, + ) + .force_send() + .await; + } else { + let file = tx_err!(create_option.open(&tmp_path).await, tx, BuildPusherError); + let _ = tx.send(Event::Start { + tmp_path: tmp_path.clone(), + config_path, + url_info: info.clone(), + parsed_config: partial_config, + }); + Self::overwrite( + file, + url, + config, + info, + Some(Arc::new(Mutex::new(Some(resp)))), + tx.clone(), + cancel_token, + ) + .force_send() + .await; + } + tx_err!( + fs::rename(tmp_path, origin_final_path).await, + tx, + RenameFailed + ); + return; + } + let mut i = 0; + let mut final_path = origin_final_path.clone(); + loop { + if can_resume + && let (Ok(file), Ok(state)) = tokio::join!( + no_create_option.open(&tmp_path), + DownloadState::load(&config_path) + ) + { + if let Some(config) = &state.config { + partial_config.inherit_from(config); + } + let _ = tx.send(Event::Start { + tmp_path: tmp_path.clone(), + config_path, + url_info: info.clone(), + parsed_config: partial_config.clone(), + }); + Self::overwrite( + file, + url, + partial_config.build(), + info, + Some(Arc::new(Mutex::new(Some(resp)))), + tx.clone(), + cancel_token, + ) + .force_send() + .await; + tx_err!(fs::rename(tmp_path, final_path).await, tx, RenameFailed); + return; + } else if let Ok(file) = only_create_option.open(&tmp_path).await { + let _ = tx.send(Event::Start { + tmp_path: tmp_path.clone(), + config_path, + url_info: info.clone(), + parsed_config: partial_config, + }); + Self::overwrite( + file, + url, + config, + info, + Some(Arc::new(Mutex::new(Some(resp)))), + tx.clone(), + cancel_token, + ) + .force_send() + .await; + tx_err!(fs::rename(tmp_path, final_path).await, tx, RenameFailed); + return; + } + i += 1; + final_path = origin_final_path.with_added_file_stem_prefix(format!(" {i}")); + tmp_path = final_path.with_added_extension("part"); + config_path = final_path.with_added_extension("fd"); + } }); let handle = SharedHandle::new(handle); Ok(Self { handle }) } + + // pub fn resume() {} + + async fn overwrite( + file: File, + url: Url, + config: Config, + info: UrlInfo, + resp: Option>>>, + tx: Tx, + cancel_token: CancellationToken, + ) { + let res = cancel_token + .run_until_cancelled(async move { + let puller = FastDownPuller::new(FastDownPullerOptions { + url, + headers: build_header(&config.headers).into(), + proxy: config.proxy.as_deref(), + accept_invalid_certs: config.accept_invalid_certs, + accept_invalid_hostnames: config.accept_invalid_hostnames, + cookie_store: config.cookie_store, + file_id: info.file_id.clone(), + resp, + available_ips: config.local_address.into(), + max_redirects: config.max_redirects, + }) + .map_err(Event::BuildClientError)?; + + let pusher = if cfg!(target_pointer_width = "64") + && info.fast_download + && config.write_method == WriteMethod::Mmap + { + MmapFilePusher::new(&file, info.size, config.sync_all) + .await + .map(BoxPusher::new) + } else { + CacheFilePusher::new( + file, + info.size, + config.sync_all, + config.cache_high_watermark, + config.cache_low_watermark, + config.write_buffer_size, + ) + .await + .map(BoxPusher::new) + } + .map_err(Event::BuildPusherError)?; + + Ok::<_, Event>((puller, pusher)) + }) + .await; + let (puller, pusher) = match res { + Some(Ok(res)) => res, + Some(Err(e)) => { + let _ = tx.send(e); + return; + } + None => return, + }; + + let res = if info.fast_download { + download_multi( + puller, + pusher, + fast_down::multi::DownloadOptions { + download_chunks: config.downloaded_chunk.into_iter(), + concurrent: config.threads, + retry_gap: config.retry_gap, + pull_timeout: config.pull_timeout, + push_queue_cap: config.write_queue_cap, + min_chunk_size: config.min_chunk_size, + max_speculative: config.max_speculative, + }, + ) + } else { + download_single( + puller, + pusher, + fast_down::single::DownloadOptions { + retry_gap: config.retry_gap, + push_queue_cap: config.write_queue_cap, + }, + ) + }; + + loop { + select! { + e = res.event_chain.recv() => { + match e { + Ok(e) => { + let _ = match e { + fast_down::Event::Pulling(id) => tx.send(Event::Pulling(id)), + fast_down::Event::PullError(id, e) => tx.send(Event::PullError(id, anyhow::anyhow!(e))), + fast_down::Event::PullTimeout(id) => tx.send(Event::PullTimeout(id)), + fast_down::Event::PullProgress(id, range) => tx.send(Event::PullProgress(id, range)), + fast_down::Event::Pushing(id, range) => tx.send(Event::Pushing(id, range)), + fast_down::Event::PushError(id, range, e) => tx.send(Event::PushError(id, range, anyhow::anyhow!(e))), + fast_down::Event::PushProgress(id, range) => tx.send(Event::PushProgress(id, range)), + fast_down::Event::Flushing => tx.send(Event::Flushing), + fast_down::Event::FlushError(e) => tx.send(Event::FlushError(anyhow::anyhow!(e))), + fast_down::Event::Finished(id) => tx.send(Event::Finished(id)), + }; + }, + Err(_) => break, + } + } + () = cancel_token.cancelled() => break, + } + } + + if let Err(e) = res.join().await { + let _ = tx.send(Event::JoinError(e)); + } + } } diff --git a/crates/fast-down-api/src/core/mod.rs b/crates/fast-down-api/src/core/mod.rs index 65e12f6..4529374 100644 --- a/crates/fast-down-api/src/core/mod.rs +++ b/crates/fast-down-api/src/core/mod.rs @@ -1,4 +1,6 @@ mod download; -pub(crate) mod prefetch; +pub mod prefetch; +mod state; pub use download::*; +pub use state::*; diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs new file mode 100644 index 0000000..3fb8a40 --- /dev/null +++ b/crates/fast-down-api/src/core/state.rs @@ -0,0 +1,80 @@ +use crate::{Config, PartialConfig}; +use fast_down::{ProgressEntry, UrlInfo}; +use inherit_config::{ConfigLayer, InheritConfig}; +use path_helper::tokio::safe_replace; +use std::{ + ops::Deref, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::fs; +use url::Url; + +#[derive(Debug, Clone, InheritConfig)] +pub struct DownloadStateInner { + #[config(default = Url::parse("").unwrap())] + pub url: Url, + pub etag: Option>, + pub last_modified: Option>, + #[config(nest)] + pub config: Config, + pub progress: Vec, +} + +#[derive(Debug)] +pub struct DownloadState { + inner: PartialDownloadStateInner, + is_dirty: bool, + config_path: PathBuf, +} + +impl DownloadState { + #[must_use] + pub fn new(url: &Url, url_info: &UrlInfo, config: &PartialConfig, config_path: &Path) -> Self { + Self { + inner: PartialDownloadStateInner { + url: Some(url.clone()), + etag: Some(url_info.file_id.etag.clone()), + last_modified: Some(url_info.file_id.last_modified.clone()), + config: Some(config.clone()), + progress: Some(Vec::new()), + }, + is_dirty: false, + config_path: config_path.to_path_buf(), + } + } + + pub async fn load(config_path: &Path) -> anyhow::Result { + let inner = fs::read(&config_path).await?; + let inner: PartialDownloadStateInner = toml::from_slice(&inner)?; + Ok(Self { + inner, + is_dirty: false, + config_path: config_path.to_path_buf(), + }) + } + + pub async fn store(&mut self) -> anyhow::Result<()> { + if self.is_dirty { + self.inner + .simplify_from(&PartialDownloadStateInner::default()); + let inner = toml::to_string_pretty(&self.inner)?; + safe_replace(&self.config_path, inner.as_bytes()).await?; + self.is_dirty = false; + } + Ok(()) + } + + pub fn update(&mut self, cb: impl FnOnce(&mut PartialDownloadStateInner)) { + cb(&mut self.inner); + self.is_dirty = true; + } +} + +impl Deref for DownloadState { + type Target = PartialDownloadStateInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs index 90a8830..2ae6cf4 100644 --- a/crates/fast-down-api/src/event.rs +++ b/crates/fast-down-api/src/event.rs @@ -1,3 +1,29 @@ +use crate::PartialConfig; +use fast_down::{ProgressEntry, UrlInfo, WorkerId}; +use std::{path::PathBuf, sync::Arc}; + pub enum Event { PrefetchError(anyhow::Error), + GenPathError(std::io::Error), + BuildClientError(reqwest::Error), + BuildPusherError(std::io::Error), + JoinError(Arc), + RenameFailed(std::io::Error), + Start { + tmp_path: PathBuf, + config_path: PathBuf, + url_info: UrlInfo, + parsed_config: PartialConfig, + }, + + Pulling(WorkerId), + PullError(WorkerId, anyhow::Error), + PullTimeout(WorkerId), + PullProgress(WorkerId, ProgressEntry), + Pushing(WorkerId, ProgressEntry), + PushError(WorkerId, ProgressEntry, anyhow::Error), + PushProgress(WorkerId, ProgressEntry), + Flushing, + FlushError(anyhow::Error), + Finished(WorkerId), } diff --git a/crates/fast-down-api/src/lib.rs b/crates/fast-down-api/src/lib.rs index c0afe7d..3f3d4a1 100644 --- a/crates/fast-down-api/src/lib.rs +++ b/crates/fast-down-api/src/lib.rs @@ -10,12 +10,6 @@ pub use core::*; pub use event::*; pub use fast_down; -pub use fast_down::{ - AnyError, BoxPusher, CacheDirectPusher, CacheMergePusher, CacheSeqPusher, DownloadResult, - Event as RawEvent, FileId, InvertIter, Merge, ProgressEntry, Proxy, PullResult, PullStream, - Puller, PullerError, Pusher, Total, UrlInfo, WorkerId, fast_puller, file, getifaddrs, handle, - http, invert, mem, mock, multi, reqwest as reqwest_adapter, single, -}; use tokio_util::sync::CancellationToken; diff --git a/crates/fast-down-api/src/utils/filename_template.rs b/crates/fast-down-api/src/utils/filename_template.rs new file mode 100644 index 0000000..5c23186 --- /dev/null +++ b/crates/fast-down-api/src/utils/filename_template.rs @@ -0,0 +1,36 @@ +use chrono::Local; +use path_helper::sanitize_filename; +use std::panic; +use url::Url; + +pub fn parse_filename_template(template: String, url: &Url, filename: &str) -> String { + let template = + panic::catch_unwind(|| Local::now().format(&template).to_string()).unwrap_or(template); + let host = sanitize_filename(url.host_str().unwrap_or("unknown"), 255); + let mut parent_path: Vec<_> = url + .path_segments() + .into_iter() + .flat_map(|segments| { + segments.map(|seg| { + let decoded = urlencoding::decode_binary(seg.as_bytes()); + sanitize_filename(String::from_utf8_lossy(&decoded), 255) + }) + }) + .collect(); + parent_path.pop(); + let parent_path = if parent_path.is_empty() { + ".".to_string() + } else { + parent_path.join(std::path::MAIN_SEPARATOR_STR) + }; + let (file_stem, file_ext) = filename + .rfind('.') + .map_or((filename, ""), |pos| (&filename[..pos], &filename[pos..])); + #[allow(clippy::literal_string_with_formatting_args)] + template + .replace("{host}", &host) + .replace("{parent_path}", &parent_path) + .replace("{file_name}", filename) + .replace("{file_stem}", file_stem) + .replace("{file_ext}", file_ext) +} diff --git a/crates/fast-down-api/src/utils/force_send.rs b/crates/fast-down-api/src/utils/force_send.rs new file mode 100644 index 0000000..1f8fe4f --- /dev/null +++ b/crates/fast-down-api/src/utils/force_send.rs @@ -0,0 +1,29 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; + +pub struct ForceSend(pub F); + +#[allow(clippy::non_send_fields_in_send_ty)] +unsafe impl Send for ForceSend {} + +impl Future for ForceSend { + type Output = F::Output; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + unsafe { self.map_unchecked_mut(|s| &mut s.0).poll(cx) } + } +} + +pub trait ForceSendExt { + fn force_send(self) -> ForceSend + where + Self: Sized, + { + ForceSend(self) + } +} + +impl ForceSendExt for F { + fn force_send(self) -> ForceSend { + ForceSend(self) + } +} diff --git a/crates/fast-down-api/src/utils/gen_path.rs b/crates/fast-down-api/src/utils/gen_path.rs index 44f8971..b5f2068 100644 --- a/crates/fast-down-api/src/utils/gen_path.rs +++ b/crates/fast-down-api/src/utils/gen_path.rs @@ -1,5 +1,37 @@ -use crate::Config; +use crate::{Config, utils::parse_filename_template}; use fast_down::UrlInfo; -use std::path::PathBuf; +use path_helper::{auto_ext, sanitize_filename, sanitize_path}; +use soft_canonicalize::soft_canonicalize; +use std::{borrow::Cow, path::PathBuf}; +use tokio::fs; +use url::Url; -pub fn gen_path(url_info: UrlInfo, config: Config) -> PathBuf {} +pub async fn gen_path(url: &Url, info: &UrlInfo, config: &Config) -> std::io::Result { + let mut filename = sanitize_filename( + if config.filename.is_empty() || config.parse_filename { + auto_ext(&info.raw_name, info.content_type.as_deref()) + } else { + Cow::Borrowed(config.filename.as_str()) + }, + 248, + ); + let mut save_dir = config.save_dir.clone(); + if config.parse_filename && !config.filename.is_empty() { + let path = PathBuf::from(parse_filename_template( + config.filename.clone(), + url, + &filename, + )); + if let Some(s) = path.file_name() { + filename = sanitize_filename(s.to_string_lossy(), 248); + } + if let Some(parent_path) = path.parent() + && let Ok(new_save_dir) = soft_canonicalize(save_dir.join(sanitize_path(parent_path))) + && new_save_dir.starts_with(&save_dir) + { + save_dir = new_save_dir; + } + } + fs::create_dir_all(&save_dir).await?; + Ok(save_dir.join(&filename)) +} diff --git a/crates/fast-down-api/src/utils/mod.rs b/crates/fast-down-api/src/utils/mod.rs index 15c0650..8938650 100644 --- a/crates/fast-down-api/src/utils/mod.rs +++ b/crates/fast-down-api/src/utils/mod.rs @@ -1,5 +1,10 @@ mod build_header; +mod filename_template; +mod force_send; mod gen_path; +mod tx_err; pub use build_header::*; +pub use filename_template::*; +pub use force_send::*; pub use gen_path::*; diff --git a/crates/fast-down-api/src/utils/tx_err.rs b/crates/fast-down-api/src/utils/tx_err.rs new file mode 100644 index 0000000..f5530ab --- /dev/null +++ b/crates/fast-down-api/src/utils/tx_err.rs @@ -0,0 +1,13 @@ +#[macro_export] +#[doc(hidden)] +macro_rules! tx_err { + ($x: expr, $tx: expr, $event: ident) => { + match $x { + Ok(r) => r, + Err(e) => { + let _ = $tx.send(Event::$event(e)); + return; + } + } + }; +} From 958475af47b19aafb0a3d7a4fa42b61734872ad5 Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 21 Jul 2026 23:28:32 +0800 Subject: [PATCH 03/26] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/Cargo.toml | 4 +- crates/fast-down-api/src/core/download.rs | 443 +++++++++++++--------- crates/fast-down-api/src/core/state.rs | 8 + crates/fast-down-api/src/event.rs | 6 + 4 files changed, 287 insertions(+), 174 deletions(-) diff --git a/crates/fast-down-api/Cargo.toml b/crates/fast-down-api/Cargo.toml index ee4c61c..c4020df 100644 --- a/crates/fast-down-api/Cargo.toml +++ b/crates/fast-down-api/Cargo.toml @@ -13,11 +13,11 @@ keywords = ["concurrency", "downloader", "parallel", "performance", "thread"] categories = ["network-programming"] [dependencies] -fast-down = { workspace = true, features = ["cookie-store", "fast-puller", "file", "getifaddrs", "mem", "reqwest-tls"] } +fast-down = { workspace = true, features = ["cookie-store", "fast-puller", "file", "getifaddrs", "mem", "reqwest-tls", "serde"] } parking_lot.workspace = true tokio.workspace = true tokio-util = "0.7" -url.workspace = true +url = { workspace = true, features = ["serde"] } serde = { version = "1.0.228", features = ["serde_derive"] } reqwest = { version = "0.13.4", default-features = false, features = [ "brotli", diff --git a/crates/fast-down-api/src/core/download.rs b/crates/fast-down-api/src/core/download.rs index 81a1295..1be4098 100644 --- a/crates/fast-down-api/src/core/download.rs +++ b/crates/fast-down-api/src/core/download.rs @@ -14,8 +14,9 @@ use fast_down::{ }; use inherit_config::ConfigLayer; use parking_lot::Mutex; -use path_helper::FileStemExt; +use path_helper::{FileStemExt, tokio::gen_unique_path}; use reqwest::Response; +use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::{ fs::{self, File, OpenOptions}, @@ -24,6 +25,24 @@ use tokio::{ use tokio_util::sync::CancellationToken; use url::Url; +/// Per-task context that stays constant across all resume/attempt iterations. +struct Ctx { + url: Url, + info: UrlInfo, + tx: Tx, + cancel_token: CancellationToken, +} + +/// Paths used by a single download attempt. +struct AttemptPaths<'a> { + tmp: &'a Path, + config_path: &'a Path, + final_path: &'a Path, + /// When true (unique mode), regenerate a fresh unique destination right + /// before rename so a concurrently-created final file is never overwritten. + unique: bool, +} + pub struct DownloadHandle { handle: SharedHandle<()>, } @@ -32,7 +51,7 @@ impl DownloadHandle { /// # Errors pub fn download( url: Url, - mut partial_config: PartialConfig, + partial_config: PartialConfig, tx: Tx, cancel_token: CancellationToken, ) -> anyhow::Result { @@ -50,142 +69,176 @@ impl DownloadHandle { let Some((info, resp)) = prefetch(&url, &config, &client, &tx).await else { return; }; - let origin_final_path = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError); - let mut tmp_path = origin_final_path.with_added_extension("part"); - let mut config_path = origin_final_path.with_added_extension("fd"); - let can_resume = config.resume && info.fast_download; - let mut no_create_option = OpenOptions::new(); - no_create_option - .read(true) - .write(true) - .truncate(false) - .create(false); - let mut create_option = OpenOptions::new(); - create_option - .read(true) - .write(true) - .truncate(false) - .create(true); - let mut only_create_option = OpenOptions::new(); - only_create_option - .read(true) - .write(true) - .truncate(false) - .create_new(true); + let origin_final = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError); + let ctx = Ctx { + url, + info, + tx, + cancel_token, + }; + let resp = Some(Arc::new(Mutex::new(Some(resp)))); if config.overwrite { - if can_resume - && let (Ok(file), Ok(state)) = tokio::join!( - no_create_option.open(&tmp_path), - DownloadState::load(&config_path) - ) - { - if let Some(config) = &state.config { - partial_config.inherit_from(config); - } - let _ = tx.send(Event::Start { - tmp_path: tmp_path.clone(), - config_path, - url_info: info.clone(), - parsed_config: partial_config.clone(), - }); - Self::overwrite( - file, - url, - partial_config.build(), - info, - Some(Arc::new(Mutex::new(Some(resp)))), - tx.clone(), - cancel_token, - ) - .force_send() - .await; - } else { - let file = tx_err!(create_option.open(&tmp_path).await, tx, BuildPusherError); - let _ = tx.send(Event::Start { - tmp_path: tmp_path.clone(), - config_path, - url_info: info.clone(), - parsed_config: partial_config, - }); - Self::overwrite( - file, - url, - config, - info, - Some(Arc::new(Mutex::new(Some(resp)))), - tx.clone(), - cancel_token, - ) - .force_send() - .await; + Self::run_overwrite(&ctx, config, partial_config, resp, origin_final).await; + } else { + Self::run_unique(&ctx, config, partial_config, resp, origin_final).await; + } + }); + let handle = SharedHandle::new(handle); + Ok(Self { handle }) + } + + async fn run_overwrite( + ctx: &Ctx, + config: Config, + partial_config: PartialConfig, + resp: Option>>>, + origin_final: PathBuf, + ) { + let can_resume = config.resume && ctx.info.fast_download; + let tmp = origin_final.with_added_extension("part"); + let cfg = origin_final.with_added_extension("fd"); + + if can_resume + && let no_create = Self::open_existing() + && let (Ok(file), Ok(state)) = + tokio::join!(no_create.open(&tmp), DownloadState::load(&cfg)) + { + let mut partial_config = partial_config; + if let Some(config) = &state.config { + partial_config.inherit_from(config); + } + let parsed = partial_config.clone(); + let effective = partial_config.build(); + let paths = AttemptPaths { + tmp: &tmp, + config_path: &cfg, + final_path: &origin_final, + unique: false, + }; + Self::attempt(ctx, file, &paths, effective, parsed, resp).await; + return; + } + let create = Self::open_create(); + let file = tx_err!(create.open(&tmp).await, ctx.tx, BuildPusherError); + let paths = AttemptPaths { + tmp: &tmp, + config_path: &cfg, + final_path: &origin_final, + unique: false, + }; + Self::attempt(ctx, file, &paths, config, partial_config, resp).await; + } + + async fn run_unique( + ctx: &Ctx, + config: Config, + partial_config: PartialConfig, + resp: Option>>>, + origin_final: PathBuf, + ) { + let can_resume = config.resume && ctx.info.fast_download; + let mut i = 0; + let mut final_path = origin_final.clone(); + let mut tmp = origin_final.with_added_extension("part"); + let mut cfg = origin_final.with_added_extension("fd"); + let no_create = Self::open_existing(); + let only_create = Self::open_create_new(); + + loop { + if can_resume + && let (Ok(file), Ok(state)) = + tokio::join!(no_create.open(&tmp), DownloadState::load(&cfg)) + { + let mut partial_config = partial_config; + if let Some(config) = &state.config { + partial_config.inherit_from(config); } - tx_err!( - fs::rename(tmp_path, origin_final_path).await, - tx, - RenameFailed - ); + let parsed = partial_config.clone(); + let effective = partial_config.build(); + let paths = AttemptPaths { + tmp: &tmp, + config_path: &cfg, + final_path: &final_path, + unique: true, + }; + Self::attempt(ctx, file, &paths, effective, parsed, resp).await; return; } - let mut i = 0; - let mut final_path = origin_final_path.clone(); - loop { - if can_resume - && let (Ok(file), Ok(state)) = tokio::join!( - no_create_option.open(&tmp_path), - DownloadState::load(&config_path) - ) - { - if let Some(config) = &state.config { - partial_config.inherit_from(config); - } - let _ = tx.send(Event::Start { - tmp_path: tmp_path.clone(), - config_path, - url_info: info.clone(), - parsed_config: partial_config.clone(), - }); - Self::overwrite( - file, - url, - partial_config.build(), - info, - Some(Arc::new(Mutex::new(Some(resp)))), - tx.clone(), - cancel_token, - ) - .force_send() - .await; - tx_err!(fs::rename(tmp_path, final_path).await, tx, RenameFailed); - return; - } else if let Ok(file) = only_create_option.open(&tmp_path).await { - let _ = tx.send(Event::Start { - tmp_path: tmp_path.clone(), - config_path, - url_info: info.clone(), - parsed_config: partial_config, - }); - Self::overwrite( - file, - url, - config, - info, - Some(Arc::new(Mutex::new(Some(resp)))), - tx.clone(), - cancel_token, - ) - .force_send() - .await; - tx_err!(fs::rename(tmp_path, final_path).await, tx, RenameFailed); - return; - } - i += 1; - final_path = origin_final_path.with_added_file_stem_prefix(format!(" {i}")); - tmp_path = final_path.with_added_extension("part"); - config_path = final_path.with_added_extension("fd"); + if let Ok(file) = only_create.open(&tmp).await { + let paths = AttemptPaths { + tmp: &tmp, + config_path: &cfg, + final_path: &final_path, + unique: true, + }; + Self::attempt( + ctx, + file, + &paths, + config.clone(), + partial_config.clone(), + resp, + ) + .await; + return; } + i += 1; + final_path = origin_final.with_added_file_stem_prefix(format!(" {i}")); + tmp = final_path.with_added_extension("part"); + cfg = final_path.with_added_extension("fd"); + } + } + + /// Emit `Event::Start`, run the actual download, then rename `.part` to its + /// final destination and emit `Event::Renamed` with the real landing path. + /// In unique mode the destination is regenerated via `gen_unique_path` right + /// before rename, so a final file created concurrently during the download is + /// never overwritten (it lands as `xxx (1).mp4` instead). `resp` is consumed + /// here exactly once. + async fn attempt( + ctx: &Ctx, + file: File, + paths: &AttemptPaths<'_>, + effective: Config, + parsed: PartialConfig, + resp: Option>>>, + ) { + let _ = ctx.tx.send(Event::Start { + tmp_path: paths.tmp.to_path_buf(), + config_path: paths.config_path.to_path_buf(), + url_info: ctx.info.clone(), + parsed_config: parsed, }); - let handle = SharedHandle::new(handle); - Ok(Self { handle }) + Self::overwrite( + file, + ctx.url.clone(), + effective, + ctx.info.clone(), + resp, + ctx.tx.clone(), + ctx.cancel_token.clone(), + ) + .force_send() + .await; + // In unique mode, reserve a fresh unique destination right before rename: + // the final file may have been created by someone else while we were + // downloading. `gen_unique_path` atomically creates an empty placeholder + // (create_new) which the rename below then replaces, closing the TOCTOU gap. + let dest = if paths.unique { + tx_err!(gen_unique_path(paths.final_path).await, ctx.tx, GenPathError) + } else { + paths.final_path.to_path_buf() + }; + if let Err(e) = fs::rename(paths.tmp, &dest).await { + // Best-effort: drop the empty placeholder we just reserved so a failed + // rename doesn't leave an orphan `xxx (1).mp4` behind. + if paths.unique { + let _ = fs::remove_file(&dest).await; + } + let _ = ctx.tx.send(Event::RenameFailed(e)); + return; + } + let _ = ctx.tx.send(Event::Renamed(dest)); } // pub fn resume() {} @@ -199,55 +252,83 @@ impl DownloadHandle { tx: Tx, cancel_token: CancellationToken, ) { - let res = cancel_token + let url_ref = &url; + let config_ref = &config; + let info_ref = &info; + let built = cancel_token .run_until_cancelled(async move { - let puller = FastDownPuller::new(FastDownPullerOptions { - url, - headers: build_header(&config.headers).into(), - proxy: config.proxy.as_deref(), - accept_invalid_certs: config.accept_invalid_certs, - accept_invalid_hostnames: config.accept_invalid_hostnames, - cookie_store: config.cookie_store, - file_id: info.file_id.clone(), - resp, - available_ips: config.local_address.into(), - max_redirects: config.max_redirects, - }) - .map_err(Event::BuildClientError)?; - - let pusher = if cfg!(target_pointer_width = "64") - && info.fast_download - && config.write_method == WriteMethod::Mmap - { - MmapFilePusher::new(&file, info.size, config.sync_all) - .await - .map(BoxPusher::new) - } else { - CacheFilePusher::new( - file, - info.size, - config.sync_all, - config.cache_high_watermark, - config.cache_low_watermark, - config.write_buffer_size, - ) - .await - .map(BoxPusher::new) - } - .map_err(Event::BuildPusherError)?; - - Ok::<_, Event>((puller, pusher)) + let puller = Self::build_puller(url_ref, config_ref, info_ref, resp)?; + let pusher = Self::build_pusher(file, config_ref, info_ref).await?; + Ok::<_, Box>((puller, pusher)) }) .await; - let (puller, pusher) = match res { - Some(Ok(res)) => res, + let (puller, pusher) = match built { + Some(Ok(built)) => built, Some(Err(e)) => { - let _ = tx.send(e); + let _ = tx.send(*e); return; } None => return, }; + Self::run_download(puller, pusher, config, info, tx, cancel_token).await; + } + + fn build_puller( + url: &Url, + config: &Config, + info: &UrlInfo, + resp: Option>>>, + ) -> Result> { + FastDownPuller::new(FastDownPullerOptions { + url: url.clone(), + headers: build_header(&config.headers).into(), + proxy: config.proxy.as_deref(), + accept_invalid_certs: config.accept_invalid_certs, + accept_invalid_hostnames: config.accept_invalid_hostnames, + cookie_store: config.cookie_store, + file_id: info.file_id.clone(), + resp, + available_ips: config.local_address.clone().into(), + max_redirects: config.max_redirects, + }) + .map_err(|e| Box::new(Event::BuildClientError(e))) + } + async fn build_pusher( + file: File, + config: &Config, + info: &UrlInfo, + ) -> Result> { + if cfg!(target_pointer_width = "64") + && info.fast_download + && config.write_method == WriteMethod::Mmap + { + MmapFilePusher::new(&file, info.size, config.sync_all) + .await + .map(BoxPusher::new) + } else { + CacheFilePusher::new( + file, + info.size, + config.sync_all, + config.cache_high_watermark, + config.cache_low_watermark, + config.write_buffer_size, + ) + .await + .map(BoxPusher::new) + } + .map_err(|e| Box::new(Event::BuildPusherError(e))) + } + + async fn run_download( + puller: FastDownPuller, + pusher: BoxPusher, + config: Config, + info: UrlInfo, + tx: Tx, + cancel_token: CancellationToken, + ) { let res = if info.fast_download { download_multi( puller, @@ -302,4 +383,22 @@ impl DownloadHandle { let _ = tx.send(Event::JoinError(e)); } } + + fn open_existing() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(false); + o + } + + fn open_create() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(true); + o + } + + fn open_create_new() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create_new(true); + o + } } diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs index 3fb8a40..3247441 100644 --- a/crates/fast-down-api/src/core/state.rs +++ b/crates/fast-down-api/src/core/state.rs @@ -44,6 +44,10 @@ impl DownloadState { } } + /// Load a download state from disk. + /// + /// # Errors + /// Returns an error if the file cannot be read or deserialized. pub async fn load(config_path: &Path) -> anyhow::Result { let inner = fs::read(&config_path).await?; let inner: PartialDownloadStateInner = toml::from_slice(&inner)?; @@ -54,6 +58,10 @@ impl DownloadState { }) } + /// Persist the download state to disk when it is dirty. + /// + /// # Errors + /// Returns an error if serializing or writing the state fails. pub async fn store(&mut self) -> anyhow::Result<()> { if self.is_dirty { self.inner diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs index 2ae6cf4..3b63e32 100644 --- a/crates/fast-down-api/src/event.rs +++ b/crates/fast-down-api/src/event.rs @@ -2,6 +2,7 @@ use crate::PartialConfig; use fast_down::{ProgressEntry, UrlInfo, WorkerId}; use std::{path::PathBuf, sync::Arc}; +#[allow(clippy::large_enum_variant)] pub enum Event { PrefetchError(anyhow::Error), GenPathError(std::io::Error), @@ -9,6 +10,11 @@ pub enum Event { BuildPusherError(std::io::Error), JoinError(Arc), RenameFailed(std::io::Error), + /// Emitted after the `.part` file is successfully renamed to its final + /// destination. Carries the actual landing path, which in unique mode may + /// differ from the originally-planned name (e.g. `xxx (1).mp4`) when the + /// target got occupied during the download. + Renamed(PathBuf), Start { tmp_path: PathBuf, config_path: PathBuf, From ff15cb6c1f304f6772d833cd513901fed7e5d9b5 Mon Sep 17 00:00:00 2001 From: share121 Date: Fri, 24 Jul 2026 20:42:49 +0800 Subject: [PATCH 04/26] =?UTF-8?q?feat:=20=E4=BF=AE=E5=A4=8D=20resume=20?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- Cargo.lock | 476 +++++++++++++- Cargo.toml | 1 + crates/fast-down-api/Cargo.toml | 26 +- crates/fast-down-api/src/core/download.rs | 762 ++++++++++++---------- crates/fast-down-api/src/core/state.rs | 63 +- crates/fast-down-api/src/event.rs | 33 +- crates/fast-down-api/tests/resume.rs | 668 +++++++++++++++++++ 7 files changed, 1693 insertions(+), 336 deletions(-) create mode 100644 crates/fast-down-api/tests/resume.rs diff --git a/Cargo.lock b/Cargo.lock index 43a4bca..8a3b047 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -11,6 +17,36 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -21,12 +57,30 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "atomic-waker" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "aws-lc-rs" version = "1.17.3" @@ -62,6 +116,27 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -109,6 +184,19 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "cmake" version = "0.1.58" @@ -137,6 +225,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "cookie" version = "0.18.1" @@ -166,6 +274,16 @@ dependencies = [ "url", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -191,6 +309,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-utils" version = "0.8.22" @@ -269,11 +396,39 @@ dependencies = [ "httpdate", "mockito", "parking_lot", - "path_helper", + "path_helper 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "reqwest", + "serde", + "thiserror", + "tokio", + "url", + "urlencoding", +] + +[[package]] +name = "fast-down-api" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "crossfire", + "fast-down", + "futures", + "http-body-util", + "hyper", + "hyper-util", + "inherit-config", + "mime_guess", + "parking_lot", + "path_helper 0.1.8", "reqwest", "serde", + "soft-canonicalize", "thiserror", "tokio", + "tokio-util", + "toml", "url", "urlencoding", ] @@ -313,6 +468,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -597,9 +762,35 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", ] [[package]] @@ -715,6 +906,26 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inherit-config" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590c82229e5e5335b001ce25020931c800c382089234890f215e2a4d7f695201" +dependencies = [ + "inherit-config-derive", +] + +[[package]] +name = "inherit-config-derive" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcef06b7bbce7e228d5bf17f3718aa7141ce92bf5cd2ab378ce96c6dd80c0066" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -857,6 +1068,32 @@ dependencies = [ "libc", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -899,6 +1136,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -934,6 +1180,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "path_helper" +version = "0.1.8" +dependencies = [ + "mime_guess", + "sanitize-filename", + "tokio", +] + [[package]] name = "path_helper" version = "0.1.8" @@ -997,6 +1252,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-canonicalize" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e8ce98d2141039e100a63f7ab04ee0dec503608d72dd6dbc9d56d7532da3ec" +dependencies = [ + "dunce", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1204,6 +1468,7 @@ dependencies = [ "cookie", "cookie_store", "futures-core", + "h2", "http", "http-body", "http-body-util", @@ -1314,7 +1579,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni", "log", @@ -1399,7 +1664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -1464,6 +1729,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1482,6 +1756,22 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simd_cesu8" version = "1.2.0" @@ -1526,6 +1816,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "soft-canonicalize" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80111c43bbe801bcec3679d2ccba588a4618c11de4024abbd028ff50e33436dd" +dependencies = [ + "dunce", + "proc-canonicalize", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1580,6 +1880,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -1679,6 +2000,7 @@ dependencies = [ "mio", "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -1719,6 +2041,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + [[package]] name = "tower" version = "0.5.3" @@ -1740,12 +2101,17 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ + "async-compression", "bitflags", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -1789,6 +2155,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1959,12 +2331,76 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2047,6 +2483,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -2167,3 +2609,31 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 01e2de3..5954ef9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/fast-down", "crates/fast-pull", "crates/fast-steal", + "crates/fast-down-api", ] [workspace.package] diff --git a/crates/fast-down-api/Cargo.toml b/crates/fast-down-api/Cargo.toml index c4020df..5de7bd3 100644 --- a/crates/fast-down-api/Cargo.toml +++ b/crates/fast-down-api/Cargo.toml @@ -13,7 +13,15 @@ keywords = ["concurrency", "downloader", "parallel", "performance", "thread"] categories = ["network-programming"] [dependencies] -fast-down = { workspace = true, features = ["cookie-store", "fast-puller", "file", "getifaddrs", "mem", "reqwest-tls", "serde"] } +fast-down = { workspace = true, features = [ + "cookie-store", + "fast-puller", + "file", + "getifaddrs", + "mem", + "reqwest-tls", + "serde", +] } parking_lot.workspace = true tokio.workspace = true tokio-util = "0.7" @@ -34,12 +42,26 @@ thiserror.workspace = true crossfire.workspace = true anyhow = "1.0.103" mime_guess = "2.0.5" -path_helper = { version = "0.1.3", features = ["auto_ext", "sanitize", "tokio"], path = "../../../../path_helper" } +path_helper = { version = "0.1.3", features = [ + "auto_ext", + "sanitize", + "tokio", +], path = "../../../../path_helper" } chrono = "0.4.45" urlencoding = "2.1.3" soft-canonicalize = { version = "0.5.6", features = ["dunce"] } inherit-config = "0.2.2" toml = "1.1.2" +[dev-dependencies] +url = { workspace = true } +bytes = { workspace = true } +futures = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tokio-util = "0.7" +hyper = { version = "1", features = ["server", "http1"] } +hyper-util = { version = "0.1", features = ["tokio", "server", "http1"] } +http-body-util = "0.1" + [lints] workspace = true diff --git a/crates/fast-down-api/src/core/download.rs b/crates/fast-down-api/src/core/download.rs index 1be4098..a122d13 100644 --- a/crates/fast-down-api/src/core/download.rs +++ b/crates/fast-down-api/src/core/download.rs @@ -1,59 +1,114 @@ +#![allow(clippy::too_many_lines)] + use crate::{ - Config, DownloadState, Event, PartialConfig, Tx, WriteMethod, + Config, DownloadState, Event, PartialConfig, ResumeError, Tx, WriteMethod, prefetch::prefetch, tx_err, utils::{ForceSendExt, build_header, gen_path}, }; use fast_down::{ - BoxPusher, UrlInfo, + BoxPusher, Merge, fast_puller::{FastDownPuller, FastDownPullerOptions, build_client}, file::{CacheFilePusher, MmapFilePusher}, handle::SharedHandle, + invert, multi::download_multi, + reqwest::SmartRedirectClient, single::download_single, }; use inherit_config::ConfigLayer; use parking_lot::Mutex; use path_helper::{FileStemExt, tokio::gen_unique_path}; -use reqwest::Response; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::{ fs::{self, File, OpenOptions}, select, + task::JoinError, }; use tokio_util::sync::CancellationToken; use url::Url; -/// Per-task context that stays constant across all resume/attempt iterations. -struct Ctx { - url: Url, - info: UrlInfo, - tx: Tx, - cancel_token: CancellationToken, -} - -/// Paths used by a single download attempt. -struct AttemptPaths<'a> { - tmp: &'a Path, - config_path: &'a Path, - final_path: &'a Path, - /// When true (unique mode), regenerate a fresh unique destination right - /// before rename so a concurrently-created final file is never overwritten. - unique: bool, -} +/// Persist the download state after at least this many freshly-written bytes. +const STATE_STORE_BYTES: u64 = 16 * 1024 * 1024; +/// Persist the download state after at least this many `PushProgress` events. +const STATE_STORE_EVENTS: usize = 512; pub struct DownloadHandle { handle: SharedHandle<()>, } impl DownloadHandle { + /// Wait for the download to complete. + /// + /// # Panics + /// Panics if the background download task exits unexpectedly. + /// + /// # Errors + /// Returns `Arc` if the download task itself panics or is + /// cancelled. + pub async fn join(&self) -> Result<(), Arc> { + self.handle.join().await + } + /// # Errors pub fn download( url: Url, partial_config: PartialConfig, tx: Tx, cancel_token: CancellationToken, + ) -> anyhow::Result { + Self::spawn(url, partial_config, tx, cancel_token, None) + } + + /// Explicitly resume an interrupted download from the given temporary file. + /// + /// `tmp_path` is the `.part` file left behind by a previous (cancelled or + /// crashed) download. + /// + /// - When `tmp_path` **exists**, the download continues from it + /// (`force_resume = true`, and the `resume` config flag is forced on). + /// - When `tmp_path` does **not** exist, the call falls back to a fresh + /// [`DownloadHandle::download`] (`force_resume = false`) and starts writing + /// at `tmp_path` from scratch. + /// + /// Note: even when `tmp_path` exists, if the `.fd` state file is missing or + /// the remote file changed, an [`Event::ResumeError`] is still emitted — + /// this is the `resume` contract (unlike `download`, it never silently + /// falls back to a full re-download). + /// + /// # Errors + pub fn resume( + tmp_path: impl AsRef, + url: Url, + partial_config: PartialConfig, + tx: Tx, + cancel_token: CancellationToken, + ) -> anyhow::Result { + // Hand the path down verbatim; `run` probes its existence asynchronously + // via `tokio::fs::metadata` (never a synchronous `Path::exists()`, which + // would block the calling executor thread on a slow/network mount). + Self::spawn( + url, + partial_config, + tx, + cancel_token, + Some(tmp_path.as_ref().to_path_buf()), + ) + } + + /// Shared entry point for [`DownloadHandle::download`] and + /// [`DownloadHandle::resume`]. Builds the HTTP client (errors here propagate + /// to the caller via `?`), then spawns the background task that runs the + /// entire download pipeline in [`Self::run`]. + /// + /// # Errors + fn spawn( + url: Url, + partial_config: PartialConfig, + tx: Tx, + cancel_token: CancellationToken, + tmp_path: Option, ) -> anyhow::Result { let config = partial_config.clone().build(); let client = build_client( @@ -65,340 +120,391 @@ impl DownloadHandle { config.local_address.first().copied(), config.max_redirects, )?; - let handle = tokio::spawn(async move { - let Some((info, resp)) = prefetch(&url, &config, &client, &tx).await else { - return; - }; - let origin_final = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError); - let ctx = Ctx { - url, - info, - tx, - cancel_token, - }; - let resp = Some(Arc::new(Mutex::new(Some(resp)))); - if config.overwrite { - Self::run_overwrite(&ctx, config, partial_config, resp, origin_final).await; - } else { - Self::run_unique(&ctx, config, partial_config, resp, origin_final).await; - } - }); + let handle = tokio::spawn(Self::run( + url, + config, + partial_config, + client, + tx, + cancel_token, + tmp_path, + )); let handle = SharedHandle::new(handle); Ok(Self { handle }) } - async fn run_overwrite( - ctx: &Ctx, - config: Config, + /// The entire download pipeline, consolidated into a single function: + /// + /// 1. prefetch + resolve the destination path, + /// 2. try to resume from an existing `.part`/`.fd` pair (or start fresh), + /// 3. build the puller/pusher and run the transfer (multi or single), + /// 4. forward engine events, persist the resume state with debounce, + /// 5. on cancel keep `.part`+`.fd`; otherwise rename into place and drop + /// the state file. + /// + /// In `overwrite` mode (`unique = false`) a single create attempt is made; + /// in `unique` mode (`unique = true`) a fresh `create_new` is retried with a + /// regenerated stem (`xxx (1).mp4`, …) until a free path is found. + /// + /// The build step and the transfer step are each wrapped in `force_send` + /// because the futures involved are not provably `Send` (they move the file + /// handle / response into the engine builders, and the engine's event- + /// channel receiver is `!Send`). They still run on this task, so the + /// `unsafe impl Send` assertions are sound. The transfer step is kept + /// *outside* `run_until_cancelled` so a mid-download cancel is handled by + /// its own event loop + final state store, rather than being silently + /// dropped by `run_until_cancelled`. + async fn run( + url: Url, + mut config: Config, partial_config: PartialConfig, - resp: Option>>>, - origin_final: PathBuf, + client: SmartRedirectClient, + tx: Tx, + cancel_token: CancellationToken, + tmp_path: Option, ) { - let can_resume = config.resume && ctx.info.fast_download; - let tmp = origin_final.with_added_extension("part"); - let cfg = origin_final.with_added_extension("fd"); + let Some((info, resp)) = prefetch(&url, &config, &client, &tx).await else { + return; + }; + // Probe the caller-provided `tmp_path` asynchronously so we never block + // the executor thread with a synchronous `Path::exists()` (which would + // stall a `current_thread` runtime or a slow network mount on the + // calling worker). `try_exists` is the async, error-distinguishing + // equivalent of `std::path::Path::exists`; we treat an undeterminable + // probe (`Err`) the same as "absent" and fall back to a fresh download. + // A present `.part` means we must force-resume this exact target. + let tmp_exists = match &tmp_path { + Some(p) => tokio::fs::try_exists(p).await.unwrap_or(false), + None => false, + }; + if tmp_exists { + config.resume = true; + } + // When an explicit final path is given (resume with a caller-provided + // `tmp_path`), use it verbatim and never auto-regenerate a unique name. + let (origin_final, unique) = if tmp_exists { + let base = tmp_path.as_ref().unwrap().with_extension(""); + (base, false) + } else { + ( + tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError), + !config.overwrite, + ) + }; + let can_resume = config.resume && info.fast_download; - if can_resume - && let no_create = Self::open_existing() - && let (Ok(file), Ok(state)) = - tokio::join!(no_create.open(&tmp), DownloadState::load(&cfg)) - { - let mut partial_config = partial_config; - if let Some(config) = &state.config { - partial_config.inherit_from(config); - } - let parsed = partial_config.clone(); - let effective = partial_config.build(); - let paths = AttemptPaths { - tmp: &tmp, - config_path: &cfg, - final_path: &origin_final, - unique: false, - }; - Self::attempt(ctx, file, &paths, effective, parsed, resp).await; + if tmp_exists && !info.fast_download { + let _ = tx.send(Event::ResumeError(ResumeError::NotResumable)); return; } - let create = Self::open_create(); - let file = tx_err!(create.open(&tmp).await, ctx.tx, BuildPusherError); - let paths = AttemptPaths { - tmp: &tmp, - config_path: &cfg, - final_path: &origin_final, - unique: false, + + let open_existing = { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(false); + o + }; + let open_create = { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(true); + o + }; + let open_create_new = { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create_new(true); + o }; - Self::attempt(ctx, file, &paths, config, partial_config, resp).await; - } - async fn run_unique( - ctx: &Ctx, - config: Config, - partial_config: PartialConfig, - resp: Option>>>, - origin_final: PathBuf, - ) { - let can_resume = config.resume && ctx.info.fast_download; - let mut i = 0; let mut final_path = origin_final.clone(); - let mut tmp = origin_final.with_added_extension("part"); - let mut cfg = origin_final.with_added_extension("fd"); - let no_create = Self::open_existing(); - let only_create = Self::open_create_new(); - + let mut i = 0usize; loop { - if can_resume - && let (Ok(file), Ok(state)) = - tokio::join!(no_create.open(&tmp), DownloadState::load(&cfg)) - { - let mut partial_config = partial_config; - if let Some(config) = &state.config { - partial_config.inherit_from(config); + let tmp = final_path.with_added_extension("part"); + let cfg = final_path.with_added_extension("fd"); + + // ---- 1. Try to resume from an existing `.part`/`.fd` pair ---- + let resumed: Option<(File, Config, PartialConfig, DownloadState)> = if can_resume { + let (open_res, load_res) = + tokio::join!(open_existing.open(&tmp), DownloadState::load(&cfg)); + match (open_res, load_res) { + (Ok(file), Ok(state)) if state.validate(&info) => { + let mut pc = partial_config.clone(); + if let Some(c) = &state.config { + pc.inherit_from(c); + } + let parsed = pc.clone(); + let effective = pc.build(); + Some((file, effective, parsed, state)) + } + (Ok(_), Ok(_)) => { + if tmp_exists { + let _ = tx.send(Event::ResumeError(ResumeError::FileChanged)); + return; + } + // Stale state: discard and start a fresh download. + let _ = fs::remove_file(&tmp).await; + let _ = fs::remove_file(&cfg).await; + None + } + _ => { + if tmp_exists { + let _ = tx.send(Event::ResumeError(ResumeError::NoStateFile)); + return; + } + None + } } - let parsed = partial_config.clone(); - let effective = partial_config.build(); - let paths = AttemptPaths { - tmp: &tmp, - config_path: &cfg, - final_path: &final_path, - unique: true, + } else { + None + }; + + // ---- 2. Pick the file to write + matching resume state ---- + let (file, mut effective, parsed, resume_state): ( + File, + Config, + PartialConfig, + Option, + ) = if let Some((f, eff, par, st)) = resumed { + (f, eff, par, Some(st)) + } else { + let f = if unique { + open_create_new.open(&tmp).await.ok() + } else { + Some(tx_err!(open_create.open(&tmp).await, tx, BuildPusherError)) }; - Self::attempt(ctx, file, &paths, effective, parsed, resp).await; - return; + if let Some(f) = f { + let parsed = partial_config.clone(); + (f, config.clone(), parsed, None) + } else { + if !unique { + return; + } + // Collision in unique mode: regenerate the stem and retry. + i += 1; + final_path = origin_final.with_added_file_stem_prefix(format!(" {i}")); + continue; + } + }; + + // ===== attempt: start, download, persist, rename ===== + let is_resume = resume_state.is_some(); + let mut state = + resume_state.unwrap_or_else(|| DownloadState::new(&url, &info, &parsed, &cfg)); + + // Fold the saved progress into the engine's "already downloaded" set + // so it only fetches the remaining bytes. + if is_resume { + if let Some(progress) = state.progress.clone() { + for p in progress { + effective.downloaded_chunk.merge_progress(p); + } + } + let _ = tx.send(Event::Resumed { + config_path: cfg.clone(), + progress: state.progress.clone().unwrap_or_default(), + size: info.size, + }); } - if let Ok(file) = only_create.open(&tmp).await { - let paths = AttemptPaths { - tmp: &tmp, - config_path: &cfg, - final_path: &final_path, - unique: true, - }; - Self::attempt( - ctx, - file, - &paths, - config.clone(), - partial_config.clone(), - resp, - ) + let _ = tx.send(Event::Start { + tmp_path: tmp.clone(), + config_path: cfg.clone(), + url_info: info.clone(), + parsed_config: parsed, + }); + + // ---- 3a. Build the puller + pusher ---- + // Wrapped in `force_send` (not provably `Send`). The build itself is + // fast and non-blocking, so a cancel landing here is rare and simply + // ends the task without a transfer. + let url_ref = &url; + let config_ref = &effective; + let info_ref = &info; + let ct = cancel_token.clone(); + let resp = Some(Arc::new(Mutex::new(Some(resp)))); + let built = ct + .run_until_cancelled(async move { + let puller = FastDownPuller::new(FastDownPullerOptions { + url: url_ref.clone(), + headers: build_header(&config_ref.headers).into(), + proxy: config_ref.proxy.as_deref(), + accept_invalid_certs: config_ref.accept_invalid_certs, + accept_invalid_hostnames: config_ref.accept_invalid_hostnames, + cookie_store: config_ref.cookie_store, + file_id: info_ref.file_id.clone(), + resp, + available_ips: config_ref.local_address.clone().into(), + max_redirects: config_ref.max_redirects, + }) + .map_err(|e| Box::new(Event::BuildClientError(e)))?; + let pusher = if cfg!(target_pointer_width = "64") + && info_ref.fast_download + && config_ref.write_method == WriteMethod::Mmap + { + MmapFilePusher::new(&file, info_ref.size, config_ref.sync_all) + .await + .map(BoxPusher::new) + } else { + CacheFilePusher::new( + file, + info_ref.size, + config_ref.sync_all, + config_ref.cache_high_watermark, + config_ref.cache_low_watermark, + config_ref.write_buffer_size, + ) + .await + .map(BoxPusher::new) + } + .map_err(|e| Box::new(Event::BuildPusherError(e)))?; + Ok::<_, Box>((puller, pusher)) + }) + .force_send() .await; - return; - } - i += 1; - final_path = origin_final.with_added_file_stem_prefix(format!(" {i}")); - tmp = final_path.with_added_extension("part"); - cfg = final_path.with_added_extension("fd"); - } - } + let (puller, pusher) = match built { + Some(Ok(b)) => b, + Some(Err(e)) => { + let _ = tx.send(*e); + // Persist the (possibly empty) state so a later resume still + // finds a coherent `.fd`, then stop. + state.update(|_| {}); + let _ = state.store().await; + return; + } + None => { + // Cancelled before the transfer started: keep the partial + // files and persist state so a later `resume()` can continue. + state.update(|_| {}); + let _ = state.store().await; + return; + } + }; - /// Emit `Event::Start`, run the actual download, then rename `.part` to its - /// final destination and emit `Event::Renamed` with the real landing path. - /// In unique mode the destination is regenerated via `gen_unique_path` right - /// before rename, so a final file created concurrently during the download is - /// never overwritten (it lands as `xxx (1).mp4` instead). `resp` is consumed - /// here exactly once. - async fn attempt( - ctx: &Ctx, - file: File, - paths: &AttemptPaths<'_>, - effective: Config, - parsed: PartialConfig, - resp: Option>>>, - ) { - let _ = ctx.tx.send(Event::Start { - tmp_path: paths.tmp.to_path_buf(), - config_path: paths.config_path.to_path_buf(), - url_info: ctx.info.clone(), - parsed_config: parsed, - }); - Self::overwrite( - file, - ctx.url.clone(), - effective, - ctx.info.clone(), - resp, - ctx.tx.clone(), - ctx.cancel_token.clone(), - ) - .force_send() - .await; - // In unique mode, reserve a fresh unique destination right before rename: - // the final file may have been created by someone else while we were - // downloading. `gen_unique_path` atomically creates an empty placeholder - // (create_new) which the rename below then replaces, closing the TOCTOU gap. - let dest = if paths.unique { - tx_err!(gen_unique_path(paths.final_path).await, ctx.tx, GenPathError) - } else { - paths.final_path.to_path_buf() - }; - if let Err(e) = fs::rename(paths.tmp, &dest).await { - // Best-effort: drop the empty placeholder we just reserved so a failed - // rename doesn't leave an orphan `xxx (1).mp4` behind. - if paths.unique { - let _ = fs::remove_file(&dest).await; - } - let _ = ctx.tx.send(Event::RenameFailed(e)); - return; - } - let _ = ctx.tx.send(Event::Renamed(dest)); - } + // ---- 3b. Run the transfer ---- + // Multi-threaded when the server supports range requests, otherwise a + // single sequential stream. This is a *separate* `force_send` future + // (the engine's event-channel receiver is `!Send`) kept outside + // `run_until_cancelled`, so a mid-download cancel is handled by this + // future's own event loop + final state store — `run_until_cancelled` + // would otherwise drop the inner future and skip persisting `.fd`. + let res = if info.fast_download { + download_multi( + puller, + pusher, + fast_down::multi::DownloadOptions { + download_chunks: invert( + effective.downloaded_chunk.iter().cloned(), + info.size, + effective.chunk_window, + ), + concurrent: effective.threads, + retry_gap: effective.retry_gap, + pull_timeout: effective.pull_timeout, + push_queue_cap: effective.write_queue_cap, + min_chunk_size: effective.min_chunk_size, + max_speculative: effective.max_speculative, + }, + ) + } else { + download_single( + puller, + pusher, + fast_down::single::DownloadOptions { + retry_gap: effective.retry_gap, + push_queue_cap: effective.write_queue_cap, + }, + ) + }; - // pub fn resume() {} + let tx2 = tx.clone(); + let state_ref = &mut state; + let ct2 = cancel_token.clone(); + (async move { + // All events (including `PushProgress`) flow through the single + // `event_chain`. `PushProgress` also updates `state` with + // debounced store so the `.fd` always reflects the truth. + let mut store_events: usize = 0; + let mut store_bytes: u64 = 0; + loop { + select! { + e = res.event_chain.recv() => { + match e { + Ok(e) => { + if let fast_down::Event::PushProgress(range) = &e { + store_events += 1; + store_bytes += range.end - range.start; + state_ref.merge_progress(range.clone()); + } + let _ = match e { + fast_down::Event::Pulling(id) => tx2.send(Event::Pulling(id)), + fast_down::Event::PullError(id, e) => tx2.send(Event::PullError(id, anyhow::anyhow!(e))), + fast_down::Event::PullTimeout(id) => tx2.send(Event::PullTimeout(id)), + fast_down::Event::PullProgress(id, range) => tx2.send(Event::PullProgress(id, range)), + fast_down::Event::Pushing(id, range) => tx2.send(Event::Pushing(id, range)), + fast_down::Event::PushError(id, range, e) => tx2.send(Event::PushError(id, range, anyhow::anyhow!(e))), + fast_down::Event::PushProgress(range) => tx2.send(Event::PushProgress(range)), + fast_down::Event::Flushing => tx2.send(Event::Flushing), + fast_down::Event::FlushError(e) => tx2.send(Event::FlushError(anyhow::anyhow!(e))), + fast_down::Event::Finished(id) => tx2.send(Event::Finished(id)), + }; + if store_events >= STATE_STORE_EVENTS + || store_bytes >= STATE_STORE_BYTES + { + let _ = state_ref.store().await; + store_events = 0; + store_bytes = 0; + } + } + Err(_) => break, + } + } + () = ct2.cancelled() => break, + } + } - async fn overwrite( - file: File, - url: Url, - config: Config, - info: UrlInfo, - resp: Option>>>, - tx: Tx, - cancel_token: CancellationToken, - ) { - let url_ref = &url; - let config_ref = &config; - let info_ref = &info; - let built = cancel_token - .run_until_cancelled(async move { - let puller = Self::build_puller(url_ref, config_ref, info_ref, resp)?; - let pusher = Self::build_pusher(file, config_ref, info_ref).await?; - Ok::<_, Box>((puller, pusher)) + if let Err(e) = res.join().await { + let _ = tx2.send(Event::JoinError(e)); + } + // Force a final store so the `.fd` always reflects the truth + // (also covers the cancelled-before-finish case). + let _ = state_ref.store().await; }) + .force_send() .await; - let (puller, pusher) = match built { - Some(Ok(built)) => built, - Some(Err(e)) => { - let _ = tx.send(*e); + + // The `.fd` state file captures the current download progress (url, + // config, size, etag) so a later resume can validate the remote file + // identity. Called unconditionally even when cancelled before the + // transfer finished. + state.update(|_| {}); + let _ = state.store().await; + // If the download was cancelled, do NOT rename the `.part` and do NOT + // remove the `.fd` state file: keep both so a later `download()`/ + // `resume()` can continue from where it left off (design doc §8). + if cancel_token.is_cancelled() { return; } - None => return, - }; - Self::run_download(puller, pusher, config, info, tx, cancel_token).await; - } - - fn build_puller( - url: &Url, - config: &Config, - info: &UrlInfo, - resp: Option>>>, - ) -> Result> { - FastDownPuller::new(FastDownPullerOptions { - url: url.clone(), - headers: build_header(&config.headers).into(), - proxy: config.proxy.as_deref(), - accept_invalid_certs: config.accept_invalid_certs, - accept_invalid_hostnames: config.accept_invalid_hostnames, - cookie_store: config.cookie_store, - file_id: info.file_id.clone(), - resp, - available_ips: config.local_address.clone().into(), - max_redirects: config.max_redirects, - }) - .map_err(|e| Box::new(Event::BuildClientError(e))) - } - - async fn build_pusher( - file: File, - config: &Config, - info: &UrlInfo, - ) -> Result> { - if cfg!(target_pointer_width = "64") - && info.fast_download - && config.write_method == WriteMethod::Mmap - { - MmapFilePusher::new(&file, info.size, config.sync_all) - .await - .map(BoxPusher::new) - } else { - CacheFilePusher::new( - file, - info.size, - config.sync_all, - config.cache_high_watermark, - config.cache_low_watermark, - config.write_buffer_size, - ) - .await - .map(BoxPusher::new) - } - .map_err(|e| Box::new(Event::BuildPusherError(e))) - } - - async fn run_download( - puller: FastDownPuller, - pusher: BoxPusher, - config: Config, - info: UrlInfo, - tx: Tx, - cancel_token: CancellationToken, - ) { - let res = if info.fast_download { - download_multi( - puller, - pusher, - fast_down::multi::DownloadOptions { - download_chunks: config.downloaded_chunk.into_iter(), - concurrent: config.threads, - retry_gap: config.retry_gap, - pull_timeout: config.pull_timeout, - push_queue_cap: config.write_queue_cap, - min_chunk_size: config.min_chunk_size, - max_speculative: config.max_speculative, - }, - ) - } else { - download_single( - puller, - pusher, - fast_down::single::DownloadOptions { - retry_gap: config.retry_gap, - push_queue_cap: config.write_queue_cap, - }, - ) - }; - - loop { - select! { - e = res.event_chain.recv() => { - match e { - Ok(e) => { - let _ = match e { - fast_down::Event::Pulling(id) => tx.send(Event::Pulling(id)), - fast_down::Event::PullError(id, e) => tx.send(Event::PullError(id, anyhow::anyhow!(e))), - fast_down::Event::PullTimeout(id) => tx.send(Event::PullTimeout(id)), - fast_down::Event::PullProgress(id, range) => tx.send(Event::PullProgress(id, range)), - fast_down::Event::Pushing(id, range) => tx.send(Event::Pushing(id, range)), - fast_down::Event::PushError(id, range, e) => tx.send(Event::PushError(id, range, anyhow::anyhow!(e))), - fast_down::Event::PushProgress(id, range) => tx.send(Event::PushProgress(id, range)), - fast_down::Event::Flushing => tx.send(Event::Flushing), - fast_down::Event::FlushError(e) => tx.send(Event::FlushError(anyhow::anyhow!(e))), - fast_down::Event::Finished(id) => tx.send(Event::Finished(id)), - }; - }, - Err(_) => break, - } + // In unique mode, reserve a fresh unique destination right before + // rename: the final file may have been created by someone else while + // we were downloading. `gen_unique_path` atomically creates an empty + // placeholder (create_new) which the rename below then replaces, + // closing the TOCTOU gap. + let dest = if unique { + tx_err!(gen_unique_path(&final_path).await, tx, GenPathError) + } else { + final_path.clone() + }; + if let Err(e) = fs::rename(&tmp, &dest).await { + // Best-effort: drop the empty placeholder we just reserved so a + // failed rename doesn't leave an orphan `xxx (1).mp4` behind. + if unique { + let _ = fs::remove_file(&dest).await; } - () = cancel_token.cancelled() => break, + let _ = tx.send(Event::RenameFailed(e)); + return; } + let _ = tx.send(Event::Renamed(dest)); + // Success: the download is complete and renamed, so the state file is + // no longer needed. Best-effort cleanup only (kept on cancel). + let _ = fs::remove_file(&cfg).await; + return; } - - if let Err(e) = res.join().await { - let _ = tx.send(Event::JoinError(e)); - } - } - - fn open_existing() -> OpenOptions { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create(false); - o - } - - fn open_create() -> OpenOptions { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create(true); - o - } - - fn open_create_new() -> OpenOptions { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create_new(true); - o } } diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs index 3247441..d572b0c 100644 --- a/crates/fast-down-api/src/core/state.rs +++ b/crates/fast-down-api/src/core/state.rs @@ -1,5 +1,5 @@ use crate::{Config, PartialConfig}; -use fast_down::{ProgressEntry, UrlInfo}; +use fast_down::{FileId, Merge, ProgressEntry, UrlInfo}; use inherit_config::{ConfigLayer, InheritConfig}; use path_helper::tokio::safe_replace; use std::{ @@ -12,13 +12,19 @@ use url::Url; #[derive(Debug, Clone, InheritConfig)] pub struct DownloadStateInner { - #[config(default = Url::parse("").unwrap())] + #[config(default = Url::parse("http://localhost/").unwrap())] pub url: Url, pub etag: Option>, pub last_modified: Option>, #[config(nest)] pub config: Config, pub progress: Vec, + /// Total file size recorded at save time, compared against the server + /// `UrlInfo.size` during resume validation. + /// + /// An older `.fd` written without this field reads back as `None`, which + /// makes `validate` fail and safely fall back to a full re-download. + pub size: u64, } #[derive(Debug)] @@ -38,6 +44,7 @@ impl DownloadState { last_modified: Some(url_info.file_id.last_modified.clone()), config: Some(config.clone()), progress: Some(Vec::new()), + size: Some(url_info.size), }, is_dirty: false, config_path: config_path.to_path_buf(), @@ -77,6 +84,58 @@ impl DownloadState { cb(&mut self.inner); self.is_dirty = true; } + + /// Check whether the server-side file is still the same one this state was + /// saved for, so a resumed download continues from the correct offset. + /// + /// The comparison requires the recorded `size` to match and (unless both + /// sides are missing identity headers) the `FileId` (`etag` + + /// `last_modified`) to be equal. Resumable downloads also require the + /// server to support range requests. + #[must_use] + pub fn validate(&self, info: &UrlInfo) -> bool { + if self.size != Some(info.size) { + return false; + } + if !info.fast_download { + return false; + } + let saved = FileId { + etag: self.etag.clone().flatten(), + last_modified: self.last_modified.clone().flatten(), + }; + let both_missing = saved.etag.is_none() + && saved.last_modified.is_none() + && info.file_id.etag.is_none() + && info.file_id.last_modified.is_none(); + if !both_missing && saved != info.file_id { + return false; + } + true + } + + /// Merge a freshly-written byte range into the recorded progress. + /// + /// The progress list is the authoritative set of on-disk ranges; new ranges + /// are merged, de-duplicated and normalized. Marks the state dirty. + pub fn merge_progress(&mut self, range: ProgressEntry) { + self.inner + .progress + .get_or_insert_with(Vec::new) + .merge_progress(range); + self.is_dirty = true; + } + + /// Total number of bytes already downloaded, derived from `progress`. + /// + /// This value is intentionally not persisted; it is recomputed on demand + /// from the recorded ranges. + #[must_use] + pub fn downloaded_bytes(&self) -> u64 { + self.progress + .as_ref() + .map_or(0, |v| v.iter().map(|r| r.end - r.start).sum()) + } } impl Deref for DownloadState { diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs index 3b63e32..464d280 100644 --- a/crates/fast-down-api/src/event.rs +++ b/crates/fast-down-api/src/event.rs @@ -1,6 +1,25 @@ use crate::PartialConfig; use fast_down::{ProgressEntry, UrlInfo, WorkerId}; use std::{path::PathBuf, sync::Arc}; +use thiserror::Error; + +/// Errors that can occur when an explicit [`DownloadHandle::resume`](crate::DownloadHandle::resume) +/// cannot continue an interrupted download. +/// +/// These are surfaced through [`Event::ResumeError`] so the caller is notified +/// instead of silently falling back to a full re-download. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum ResumeError { + /// No `.fd` state file exists for this download, so there is nothing to resume from. + #[error("no .fd state file to resume from")] + NoStateFile, + /// The remote file changed (size / etag / last-modified mismatch), so resuming would corrupt the output. + #[error("remote file changed, cannot resume")] + FileChanged, + /// The server does not support resumable (range) downloads. + #[error("server does not support resumable download")] + NotResumable, +} #[allow(clippy::large_enum_variant)] pub enum Event { @@ -21,6 +40,18 @@ pub enum Event { url_info: UrlInfo, parsed_config: PartialConfig, }, + /// Emitted when a download resumes from a previously-saved state, before + /// [`Event::Start`]. Carries the progress that will be continued from and + /// the total file size, so a UI can show e.g. "resuming from 42%". + Resumed { + config_path: PathBuf, + progress: Vec, + size: u64, + }, + /// Emitted when an explicit `resume()` call cannot continue the download. + /// Unlike `download()` (which silently falls back to a full re-download), + /// `resume()` reports the failure so the caller can decide what to do. + ResumeError(ResumeError), Pulling(WorkerId), PullError(WorkerId, anyhow::Error), @@ -28,7 +59,7 @@ pub enum Event { PullProgress(WorkerId, ProgressEntry), Pushing(WorkerId, ProgressEntry), PushError(WorkerId, ProgressEntry, anyhow::Error), - PushProgress(WorkerId, ProgressEntry), + PushProgress(ProgressEntry), Flushing, FlushError(anyhow::Error), Finished(WorkerId), diff --git a/crates/fast-down-api/tests/resume.rs b/crates/fast-down-api/tests/resume.rs new file mode 100644 index 0000000..50a5825 --- /dev/null +++ b/crates/fast-down-api/tests/resume.rs @@ -0,0 +1,668 @@ +//! End-to-end integration tests for the resume (断点续传) feature of +//! `fast_down_api`. +//! +//! These tests stand up a minimal ranged HTTP server (built on `hyper`) and drive +//! the public `DownloadHandle::{download, resume}` API through real, resumable +//! scenarios: a successful resume after a mid-download cancel, a remote file +//! change (both the `download` silent-fallback and `resume` error-reporting +//! branches), a missing `.fd` state file, a server that does not support range +//! requests, and the core "cancel keeps `.part`/`.fd` so it can be resumed" +//! contract. +#![allow( + clippy::too_many_lines, + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::similar_names +)] + +use std::convert::Infallible; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use fast_down_api::{ + DownloadHandle, Event, PartialConfig, ResumeError, Rx, WriteMethod, create_cancellation_token, + create_channel, +}; +use futures::StreamExt; +use futures::stream::unfold; +use http_body_util::combinators::BoxBody; +use http_body_util::{BodyExt, StreamBody}; +use hyper::body::{Frame, Incoming}; +use hyper::header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, ETAG, LAST_MODIFIED, RANGE}; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use tokio::net::TcpListener; +use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; +use url::Url; + +/// Total size of the served file (5 MiB). Large enough that a throttled download +/// cannot finish before our cancel fires, so the partial state is always real. +const FILE_SIZE: usize = 5 * 1024 * 1024; +/// Bytes served per throttled chunk, with a small sleep between chunks, so the +/// download is slow enough for the cancel to land in the middle. +const THROTTLE_CHUNK: usize = 64 * 1024; +const THROTTLE_MS: u64 = 150; + +type RespBody = BoxBody; + +/// In-memory file served by [`TestServer`]. +struct FileData { + body: Vec, + etag: String, + last_modified: String, + supports_range: bool, +} + +#[derive(Clone)] +struct TestServer { + data: Arc>, +} + +impl TestServer { + /// Bind to a random loopback port and start serving. Returns the base URL. + async fn serve(&self) -> String { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind server socket"); + let addr = listener.local_addr().expect("resolve local addr"); + let server = self.clone(); + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.expect("accept connection"); + let conn_server = server.clone(); + tokio::spawn(async move { + let io = TokioIo::new(stream); + let service = service_fn(move |req| { + let s = conn_server.clone(); + handle(s, req) + }); + let _ = http1::Builder::new().serve_connection(io, service).await; + }); + } + }); + format!("http://{addr}") + } + + /// Swap the served content (and identity headers), e.g. to simulate the + /// remote file having changed between an interrupted and a resumed download. + async fn set_content(&self, body: Vec, etag: &str, last_modified: &str) { + let mut data = self.data.write().await; + data.body = body; + data.etag = etag.to_string(); + data.last_modified = last_modified.to_string(); + } +} + +/// The hyper request handler: serves the current [`FileData`], honouring range +/// requests (with throttling) when `supports_range` is set, and always answering +/// with `ETag`/`Last-Modified` so the prefetch can build a `FileId`. +async fn handle( + server: TestServer, + req: Request, +) -> Result, Infallible> { + let data = server.data.read().await; + let total = data.body.len(); + let supports_range = data.supports_range; + let etag = data.etag.clone(); + let last_modified = data.last_modified.clone(); + let body = data.body.clone(); + drop(data); + + let range = req + .headers() + .get(RANGE) + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + + if supports_range + && let Some(header) = range + && let Some((start, end)) = parse_range(&header, total) + { + let chunk = body[start..end].to_vec(); + let end_inclusive = end - 1; + let content_range = format!("bytes {start}-{end_inclusive}/{total}"); + return Ok(Response::builder() + .status(StatusCode::PARTIAL_CONTENT) + .header(CONTENT_RANGE, content_range) + .header(ACCEPT_RANGES, "bytes") + .header(CONTENT_LENGTH, chunk.len().to_string()) + .header(ETAG, etag.as_str()) + .header(LAST_MODIFIED, last_modified.as_str()) + .body(throttled_stream(chunk)) + .expect("build 206 response")); + } + + // Fallback: full body, no `Accept-Ranges`, so a `Range` probe (used by the + // prefetch to detect resumability) will not see a `Content-Range` header. + // Throttled the same way as the ranged branch so a fresh (non-resumed) + // download is still slow enough for a mid-flight cancel to land. + Ok(Response::builder() + .status(StatusCode::OK) + .header(CONTENT_LENGTH, body.len().to_string()) + .header(ETAG, etag.as_str()) + .header(LAST_MODIFIED, last_modified.as_str()) + .body(throttled_stream(body)) + .expect("build 200 response")) +} + +/// Parse a `bytes=START-END` (or `bytes=START-`) header into an exclusive +/// `[start, end)` range against `total` bytes. +fn parse_range(header: &str, total: usize) -> Option<(usize, usize)> { + let spec = header.trim().strip_prefix("bytes=")?; + let (start_s, end_s) = spec.split_once('-')?; + let start = if start_s.is_empty() { + 0 + } else { + start_s.trim().parse().ok()? + }; + let end = if end_s.is_empty() { + total + } else { + end_s.trim().parse::().ok()?.saturating_add(1) + }; + let end = end.min(total); + if start >= end { + return None; + } + Some((start, end)) +} + +/// Build a streaming body that emits `THROTTLE_CHUNK` slices of `data` with a +/// small `sleep` between them, keeping the connection open long enough for a +/// cancel to arrive mid-transfer. +fn throttled_stream(data: Vec) -> RespBody { + let stream = unfold((0usize, data), |(pos, data)| async move { + if pos >= data.len() { + return None; + } + tokio::time::sleep(Duration::from_millis(THROTTLE_MS)).await; + let end = (pos + THROTTLE_CHUNK).min(data.len()); + Some(( + Ok::<_, Infallible>(Bytes::copy_from_slice(&data[pos..end])), + (end, data), + )) + }); + let framed = stream.map(|result| result.map(Frame::data)); + BodyExt::boxed(StreamBody::new(framed)) +} + +/// Start a ranged server serving `body` under the given identity headers. +async fn start_server( + body: Vec, + etag: &str, + last_modified: &str, + supports_range: bool, +) -> (TestServer, String) { + let server = TestServer { + data: Arc::new(RwLock::new(FileData { + body, + etag: etag.to_string(), + last_modified: last_modified.to_string(), + supports_range, + })), + }; + let url = server.serve().await; + (server, url) +} + +/// A fresh, unique, empty temp directory per test (runs in parallel safe). +fn temp_dir(name: &str) -> PathBuf { + static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("fast_down_api_resume_{name}_{n}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir +} + +/// Build a deterministic config that targets `/out.bin` and downloads with +/// a single thread and small chunks so the cancel lands predictably mid-flight. +fn make_config(save_dir: &Path) -> PartialConfig { + PartialConfig { + save_dir: Some(save_dir.to_path_buf()), + filename: Some("out.bin".to_string()), + parse_filename: Some(false), + overwrite: Some(false), + write_method: Some(WriteMethod::Mmap), + min_chunk_size: Some(1024 * 1024), + threads: Some(1), + ..Default::default() + } +} + +/// Drain all events from `rx` until the task ends (channel closed). +async fn drain(rx: Rx) -> Vec { + let mut events = Vec::new(); + while let Ok(e) = rx.recv().await { + events.push(e); + } + events +} + +/// Start a normal `download()` and cancel it a fixed time after `Event::Start`, +/// leaving a real partial `.part`/`.fd` on disk. Returns the collected events. +/// `DownloadHelper` 的 `partial_download_via_cancel` 版本——开始下载后收到 `Start` 立即取消。 +async fn partial_download_via_cancel( + url: &str, + save_dir: &Path, + cancel: CancellationToken, +) -> Vec { + let cfg = make_config(save_dir); + let (tx, rx) = create_channel(); + let _handle = + DownloadHandle::download(Url::parse(url).expect("valid url"), cfg, tx, cancel.clone()) + .expect("spawn download"); + + let mut events = Vec::new(); + let mut started = false; + while let Ok(e) = rx.recv().await { + if matches!(e, Event::Start { .. }) && !started { + started = true; + cancel.cancel(); + } + events.push(e); + } + assert!(started, "expected Event::Start during the partial download"); + assert!( + !events.iter().any(|e| matches!(e, Event::Renamed(_))), + "a cancelled download must not rename the .part file" + ); + events +} + +fn original_bytes() -> Vec { + vec![0xAA; FILE_SIZE] +} + +fn new_bytes() -> Vec { + vec![0xBB; FILE_SIZE] +} + +/// Case 1 (+ Case 5 emphasis): cancel mid-download, then `resume()` finishes it. +#[tokio::test] +async fn test_resume_success() { + let dir = temp_dir("resume_success"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + assert!(part.exists(), ".part must exist after cancel"); + assert!(fd.exists(), ".fd must exist after cancel"); + assert!( + !final_path.exists(), + "final file must NOT exist after a cancelled download" + ); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let resume_cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + resume_cancel, + ) + .expect("spawn resume"); + let events = drain(rx).await; + + let (_, size) = events + .iter() + .find_map(|e| match e { + Event::Resumed { progress, size, .. } => Some((progress, *size)), + _ => None, + }) + .expect("expected Event::Resumed"); + assert_eq!( + size, FILE_SIZE as u64, + "Event::Resumed.size must equal file size" + ); + + let renamed = events + .iter() + .find_map(|e| match e { + Event::Renamed(p) => Some(p.clone()), + _ => None, + }) + .expect("expected Event::Renamed"); + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "resume() must not emit a ResumeError on a valid state" + ); + + let got = tokio::fs::read(&renamed).await.expect("read final file"); + assert_eq!(got, original_bytes(), "resumed file content mismatch"); + assert!( + !fd.exists(), + ".fd must be removed after a successful resume (contract §1)" + ); +} + +/// Case 2 (download branch): a stale `.fd` (remote file changed) makes +/// `download()` silently fall back to a full re-download of the NEW content. +#[tokio::test] +async fn test_file_changed_download_falls_back() { + let dir = temp_dir("file_changed_download"); + let (server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + // The `.fd` now records the ORIGINAL etag/size. + + server.set_content(new_bytes(), "new", "LM-B").await; + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let dl_cancel = create_cancellation_token(); + let _handle = + DownloadHandle::download(Url::parse(&url).expect("valid url"), cfg, tx, dl_cancel) + .expect("spawn download"); + let events = drain(rx).await; + + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "download() must complete (Renamed) after a silent fallback" + ); + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "download() must NOT emit a ResumeError" + ); + + let final_path = dir.join("out.bin"); + let got = tokio::fs::read(&final_path) + .await + .expect("final file should exist"); + assert_eq!(got, new_bytes(), "final file must be the NEW content"); + let fd = final_path.with_added_extension("fd"); + assert!( + !fd.exists(), + ".fd must be removed after a successful download (contract §1)" + ); +} + +/// Case 2 (resume branch): a stale `.fd` (remote file changed) makes `resume()` +/// report `ResumeError::FileChanged` and keep the partial files untouched. +#[tokio::test] +async fn test_file_changed_resume_reports_error() { + let dir = temp_dir("file_changed_resume"); + let (server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + server.set_content(new_bytes(), "new", "LM-B").await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let resume_cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + resume_cancel, + ) + .expect("spawn resume"); + let events = drain(rx).await; + + let err = events + .iter() + .find_map(|e| match e { + Event::ResumeError(r) => Some(r.clone()), + _ => None, + }) + .expect("expected Event::ResumeError"); + assert_eq!(err, ResumeError::FileChanged); + + assert!( + !events.iter().any(|e| matches!(e, Event::Renamed(_))), + "resume() must NOT rename on FileChanged" + ); + + assert!(part.exists(), ".part must be kept on resume error"); + assert!(fd.exists(), ".fd must be kept on resume error"); + assert!( + !final_path.exists(), + "final file must not exist on resume error" + ); +} + +/// Case 3: `resume()` with no `.fd` state file reports `ResumeError::NoStateFile`. +#[tokio::test] +async fn test_resume_no_state_file() { + let dir = temp_dir("resume_no_state_file"); + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + // `tmp_path` (.part) exists but the `.fd` state file is missing → `resume` + // must report `NoStateFile` (the resume contract), not silently fall back. + let _ = tokio::fs::remove_file(&fd).await; + let _ = std::fs::File::create(&part).expect("create .part"); + + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + cancel, + ) + .expect("spawn resume"); + let events = drain(rx).await; + + let err = events + .iter() + .find_map(|e| match e { + Event::ResumeError(r) => Some(r.clone()), + _ => None, + }) + .expect("expected Event::ResumeError"); + assert_eq!(err, ResumeError::NoStateFile); + assert!( + !events.iter().any(|e| matches!(e, Event::Renamed(_))), + "resume() must NOT rename when there is no .fd" + ); +} + +/// Case 4: `resume()` against a server that does not support range requests +/// reports `ResumeError::NotResumable`. +#[tokio::test] +async fn test_resume_not_resumable() { + let dir = temp_dir("resume_not_resumable"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", false).await; + // `tmp_path` (.part) must exist so `force_resume` is engaged; the + // non-range server then reports `NotResumable`. + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let _ = std::fs::File::create(&part).expect("create .part"); + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + cancel, + ) + .expect("spawn resume"); + let events = drain(rx).await; + + let err = events + .iter() + .find_map(|e| match e { + Event::ResumeError(r) => Some(r.clone()), + _ => None, + }) + .expect("expected Event::ResumeError"); + assert_eq!(err, ResumeError::NotResumable); +} + +/// Case 5 (explicit): the previous round's fix — a cancel must preserve `.part` +/// and `.fd` (not delete them), and a subsequent `resume()` must finish cleanly. +#[tokio::test] +async fn test_cancel_keeps_part_and_fd_then_resume() { + let dir = temp_dir("cancel_keeps_state"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + assert!( + part.exists() && fd.exists(), + "cancel must preserve both .part and .fd" + ); + assert!( + !final_path.exists(), + "cancel must NOT create the final file" + ); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let resume_cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + resume_cancel, + ) + .expect("spawn resume"); + let events = drain(rx).await; + + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "resume() must complete with Renamed after a cancel" + ); + assert!( + !fd.exists(), + ".fd must be removed after the resume completes" + ); + let got = tokio::fs::read(&final_path) + .await + .expect("final file should exist after resume"); + assert_eq!( + got, + original_bytes(), + "final content must match after resume" + ); +} + +/// Case 5b (explicit new behavior): when the provided `tmp_path` does **not** +/// exist, `resume()` must fall back to a fresh full download (`force_resume = +/// false`) at that path — it must complete (Renamed), produce the correct +/// content, and never emit a `ResumeError`. +#[tokio::test] +async fn test_resume_missing_tmp_path_falls_back_to_download() { + let dir = temp_dir("resume_missing_tmp"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + assert!( + !part.exists(), + "tmp_path must not exist before resume (the point of this test)" + ); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + cancel, + ) + .expect("spawn resume"); + let events = drain(rx).await; + + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "missing tmp_path must fall back to a full download (Renamed)" + ); + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "missing tmp_path must NOT emit a ResumeError" + ); + + let got = tokio::fs::read(&final_path) + .await + .expect("final file should exist after the fallback download"); + assert_eq!( + got, + original_bytes(), + "fallback download must produce correct content" + ); + assert!( + !part.exists(), + ".part must be renamed away after a successful fallback download" + ); +} + +/// Case 6 (fresh download, sanity guard): a brand-new `download()` with no +/// cancel and no resume must write the COMPLETE and CORRECT file. This is the +/// direct regression guard for BUG-1 — an empty `downloaded_chunk` (before the +/// fix) made `run_download` request zero chunks and produce a 0-byte file. +#[tokio::test] +async fn test_fresh_download_writes_full_file() { + let dir = temp_dir("fresh_download"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel = create_cancellation_token(); + let _handle = DownloadHandle::download(Url::parse(&url).expect("valid url"), cfg, tx, cancel) + .expect("spawn download"); + let events = drain(rx).await; + + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "a fresh download must complete with Renamed" + ); + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "a fresh download must NOT emit a ResumeError" + ); + + let final_path = dir.join("out.bin"); + assert!( + final_path.exists(), + "final file must exist after a fresh download" + ); + + let got = tokio::fs::read(&final_path).await.expect("read final file"); + assert_eq!( + got.len(), + FILE_SIZE, + "fresh download must write the full file (BUG-1 regression guard)" + ); + assert_eq!( + got, + original_bytes(), + "fresh download content must match source exactly" + ); + + let fd = final_path.with_added_extension("fd"); + assert!( + !fd.exists(), + ".fd must be removed after a successful fresh download" + ); +} From 8e73a3ab924f31f1d257e1502fe0f624008cee72 Mon Sep 17 00:00:00 2001 From: share121 Date: Fri, 24 Jul 2026 23:03:17 +0800 Subject: [PATCH 05/26] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20gen=5Fpath=20?= =?UTF-8?q?=E5=AF=B9=20config.save=5Fdir=20=E7=9A=84=E9=95=BF=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E5=A4=84=E7=90=86=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- Cargo.lock | 4 ---- crates/fast-down-api/Cargo.toml | 2 +- crates/fast-down-api/src/utils/gen_path.rs | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a3b047..3e6379f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1257,9 +1257,6 @@ name = "proc-canonicalize" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39e8ce98d2141039e100a63f7ab04ee0dec503608d72dd6dbc9d56d7532da3ec" -dependencies = [ - "dunce", -] [[package]] name = "proc-macro2" @@ -1822,7 +1819,6 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80111c43bbe801bcec3679d2ccba588a4618c11de4024abbd028ff50e33436dd" dependencies = [ - "dunce", "proc-canonicalize", ] diff --git a/crates/fast-down-api/Cargo.toml b/crates/fast-down-api/Cargo.toml index 5de7bd3..7b43491 100644 --- a/crates/fast-down-api/Cargo.toml +++ b/crates/fast-down-api/Cargo.toml @@ -49,7 +49,7 @@ path_helper = { version = "0.1.3", features = [ ], path = "../../../../path_helper" } chrono = "0.4.45" urlencoding = "2.1.3" -soft-canonicalize = { version = "0.5.6", features = ["dunce"] } +soft-canonicalize = "0.5.6" inherit-config = "0.2.2" toml = "1.1.2" diff --git a/crates/fast-down-api/src/utils/gen_path.rs b/crates/fast-down-api/src/utils/gen_path.rs index b5f2068..bb33483 100644 --- a/crates/fast-down-api/src/utils/gen_path.rs +++ b/crates/fast-down-api/src/utils/gen_path.rs @@ -15,7 +15,7 @@ pub async fn gen_path(url: &Url, info: &UrlInfo, config: &Config) -> std::io::Re }, 248, ); - let mut save_dir = config.save_dir.clone(); + let mut save_dir = soft_canonicalize::soft_canonicalize(&config.save_dir)?; if config.parse_filename && !config.filename.is_empty() { let path = PathBuf::from(parse_filename_template( config.filename.clone(), From 2c24c960d761609b3d459566925ebde9f502f2b4 Mon Sep 17 00:00:00 2001 From: share121 Date: Fri, 24 Jul 2026 23:03:46 +0800 Subject: [PATCH 06/26] =?UTF-8?q?chore:=20=E9=BB=98=E8=AE=A4=20url=20?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=20about:blank=20=E6=9B=B4=E5=90=88=E8=A7=84?= =?UTF-8?q?=E8=8C=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/src/core/state.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs index d572b0c..11b9aab 100644 --- a/crates/fast-down-api/src/core/state.rs +++ b/crates/fast-down-api/src/core/state.rs @@ -12,7 +12,7 @@ use url::Url; #[derive(Debug, Clone, InheritConfig)] pub struct DownloadStateInner { - #[config(default = Url::parse("http://localhost/").unwrap())] + #[config(default = Url::parse("about:blank").unwrap())] pub url: Url, pub etag: Option>, pub last_modified: Option>, @@ -104,11 +104,7 @@ impl DownloadState { etag: self.etag.clone().flatten(), last_modified: self.last_modified.clone().flatten(), }; - let both_missing = saved.etag.is_none() - && saved.last_modified.is_none() - && info.file_id.etag.is_none() - && info.file_id.last_modified.is_none(); - if !both_missing && saved != info.file_id { + if saved != info.file_id { return false; } true From 21745859a6e4d0711c781c467e1dcf4e0b28239f Mon Sep 17 00:00:00 2001 From: share121 Date: Fri, 24 Jul 2026 23:16:00 +0800 Subject: [PATCH 07/26] =?UTF-8?q?ci:=20=E4=BF=AE=E5=A4=8D=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E8=B7=AF=E5=BE=84=E5=AF=BC=E8=87=B4=E7=9A=84=20ci=20?= =?UTF-8?q?=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- Cargo.lock | 19 ++++++------------- crates/fast-down-api/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e6379f..0c37f00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -151,9 +151,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -396,7 +396,7 @@ dependencies = [ "httpdate", "mockito", "parking_lot", - "path_helper 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "path_helper", "reqwest", "serde", "thiserror", @@ -421,7 +421,7 @@ dependencies = [ "inherit-config", "mime_guess", "parking_lot", - "path_helper 0.1.8", + "path_helper", "reqwest", "serde", "soft-canonicalize", @@ -1180,22 +1180,15 @@ dependencies = [ "windows-link", ] -[[package]] -name = "path_helper" -version = "0.1.8" -dependencies = [ - "mime_guess", - "sanitize-filename", - "tokio", -] - [[package]] name = "path_helper" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8b6f3d03172dc1dac2005cfb2486d79d20f3e256383e14bcc6f23d73af9bc" dependencies = [ + "mime_guess", "sanitize-filename", + "tokio", ] [[package]] diff --git a/crates/fast-down-api/Cargo.toml b/crates/fast-down-api/Cargo.toml index 7b43491..9bcf95d 100644 --- a/crates/fast-down-api/Cargo.toml +++ b/crates/fast-down-api/Cargo.toml @@ -46,7 +46,7 @@ path_helper = { version = "0.1.3", features = [ "auto_ext", "sanitize", "tokio", -], path = "../../../../path_helper" } +] } chrono = "0.4.45" urlencoding = "2.1.3" soft-canonicalize = "0.5.6" From 9aa188115f24939fec353d7a0ce198e37bf4efdf Mon Sep 17 00:00:00 2001 From: share121 Date: Fri, 24 Jul 2026 23:40:53 +0800 Subject: [PATCH 08/26] =?UTF-8?q?wip:=20=E5=86=99=E4=B8=80=E5=8D=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- Cargo.lock | 1 - crates/fast-down-api/Cargo.toml | 3 --- crates/fast-down-api/README.md | 8 ++++---- crates/fast-down-api/src/core/download.rs | 3 +-- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c37f00..75bb1a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -419,7 +419,6 @@ dependencies = [ "hyper", "hyper-util", "inherit-config", - "mime_guess", "parking_lot", "path_helper", "reqwest", diff --git a/crates/fast-down-api/Cargo.toml b/crates/fast-down-api/Cargo.toml index 9bcf95d..cbeaf39 100644 --- a/crates/fast-down-api/Cargo.toml +++ b/crates/fast-down-api/Cargo.toml @@ -41,7 +41,6 @@ reqwest = { version = "0.13.4", default-features = false, features = [ thiserror.workspace = true crossfire.workspace = true anyhow = "1.0.103" -mime_guess = "2.0.5" path_helper = { version = "0.1.3", features = [ "auto_ext", "sanitize", @@ -54,11 +53,9 @@ inherit-config = "0.2.2" toml = "1.1.2" [dev-dependencies] -url = { workspace = true } bytes = { workspace = true } futures = { workspace = true } tokio = { workspace = true, features = ["full"] } -tokio-util = "0.7" hyper = { version = "1", features = ["server", "http1"] } hyper-util = { version = "0.1", features = ["tokio", "server", "http1"] } http-body-util = "0.1" diff --git a/crates/fast-down-api/README.md b/crates/fast-down-api/README.md index 325b1d1..43eed85 100644 --- a/crates/fast-down-api/README.md +++ b/crates/fast-down-api/README.md @@ -1,9 +1,9 @@ -# fast-down-ffi +# fast-down-api [![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) [![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) -[![Latest version](https://img.shields.io/crates/v/fast-down-ffi.svg)](https://crates.io/crates/fast-down-ffi) -[![Documentation](https://docs.rs/fast-down-ffi/badge.svg)](https://docs.rs/fast-down-ffi) -[![License](https://img.shields.io/crates/l/fast-down-ffi.svg)](https://github.com/fast-down/core/blob/main/LICENSE) +[![Latest version](https://img.shields.io/crates/v/fast-down-api.svg)](https://crates.io/crates/fast-down-api) +[![Documentation](https://docs.rs/fast-down-api/badge.svg)](https://docs.rs/fast-down-api) +[![License](https://img.shields.io/crates/l/fast-down-api.svg)](https://github.com/fast-down/core/blob/main/LICENSE) This library provides a convenient and easy-to-use wrapper around fast-down diff --git a/crates/fast-down-api/src/core/download.rs b/crates/fast-down-api/src/core/download.rs index a122d13..09e2dba 100644 --- a/crates/fast-down-api/src/core/download.rs +++ b/crates/fast-down-api/src/core/download.rs @@ -1,5 +1,3 @@ -#![allow(clippy::too_many_lines)] - use crate::{ Config, DownloadState, Event, PartialConfig, ResumeError, Tx, WriteMethod, prefetch::prefetch, @@ -154,6 +152,7 @@ impl DownloadHandle { /// *outside* `run_until_cancelled` so a mid-download cancel is handled by /// its own event loop + final state store, rather than being silently /// dropped by `run_until_cancelled`. + #[allow(clippy::too_many_lines)] async fn run( url: Url, mut config: Config, From 58a41ad0527b1c9306148a435e6b4577ab97af63 Mon Sep 17 00:00:00 2001 From: share121 Date: Sun, 26 Jul 2026 12:18:42 +0800 Subject: [PATCH 09/26] =?UTF-8?q?feat:=20=E6=8F=90=E4=BE=9B=E8=AF=A6?= =?UTF-8?q?=E7=BB=86=20prefetch=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/src/core/prefetch.rs | 2 +- crates/fast-down-api/src/event.rs | 4 +-- crates/fast-down/src/http/prefetch.rs | 12 +++----- crates/fast-down/src/reqwest/mod.rs | 36 +++++++++-------------- 4 files changed, 21 insertions(+), 33 deletions(-) diff --git a/crates/fast-down-api/src/core/prefetch.rs b/crates/fast-down-api/src/core/prefetch.rs index 9fb194c..0ee3378 100644 --- a/crates/fast-down-api/src/core/prefetch.rs +++ b/crates/fast-down-api/src/core/prefetch.rs @@ -14,7 +14,7 @@ pub async fn prefetch( match client.prefetch(url.clone()).await { Ok(t) => break Some(t), Err((e, t)) => { - let _ = tx.send(Event::PrefetchError(e.into())); + let _ = tx.send(Event::PrefetchError(e)); retry_count += 1; if retry_count >= config.retry_times { return None; diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs index 464d280..6ee199c 100644 --- a/crates/fast-down-api/src/event.rs +++ b/crates/fast-down-api/src/event.rs @@ -1,5 +1,5 @@ use crate::PartialConfig; -use fast_down::{ProgressEntry, UrlInfo, WorkerId}; +use fast_down::{ProgressEntry, UrlInfo, WorkerId, reqwest::ReqwestResponseError}; use std::{path::PathBuf, sync::Arc}; use thiserror::Error; @@ -23,7 +23,7 @@ pub enum ResumeError { #[allow(clippy::large_enum_variant)] pub enum Event { - PrefetchError(anyhow::Error), + PrefetchError(ReqwestResponseError), GenPathError(std::io::Error), BuildClientError(reqwest::Error), BuildPusherError(std::io::Error), diff --git a/crates/fast-down/src/http/prefetch.rs b/crates/fast-down/src/http/prefetch.rs index 827a568..c83e1e4 100644 --- a/crates/fast-down/src/http/prefetch.rs +++ b/crates/fast-down/src/http/prefetch.rs @@ -1,8 +1,8 @@ use crate::{ UrlInfo, http::{ - ContentDisposition, GetResponse, HttpClient, HttpError, HttpHeaders, HttpRequestBuilder, - HttpResponse, + ContentDisposition, GetRequestError, GetResponse, HttpClient, HttpError, HttpHeaders, + HttpRequestBuilder, HttpResponse, }, url_info::FileId, }; @@ -11,7 +11,7 @@ use url::Url; /// Result of a prefetch operation: the metadata ([`UrlInfo`]) and the initial HTTP response. pub type PrefetchResult = - Result<(UrlInfo, GetResponse), (HttpError, Option)>; + Result<(UrlInfo, GetResponse), (GetRequestError, Option)>; /// Trait for fetching resource metadata (size, filename, range support) from a URL. /// @@ -76,11 +76,7 @@ async fn prefetch_no_range( client: &Client, url: Url, ) -> PrefetchResult { - let resp = client - .get(url, None) - .send() - .await - .map_err(|(e, d)| (HttpError::Request(e), d))?; + let resp = client.get(url, None).send().await?; let headers = resp.headers(); let size = headers .get("content-length") diff --git a/crates/fast-down/src/reqwest/mod.rs b/crates/fast-down/src/reqwest/mod.rs index 2b56ace..009bec8 100644 --- a/crates/fast-down/src/reqwest/mod.rs +++ b/crates/fast-down/src/reqwest/mod.rs @@ -37,13 +37,13 @@ impl HttpRequestBuilder for RequestBuilder { let res = self .send() .await - .map_err(|e| (ReqwestResponseError::Reqwest(e), None))?; + .map_err(|e| (ReqwestResponseError::Request(e), None))?; let status = res.status(); if status.is_success() { Ok(res) } else { let retry_after = parse_retry_after(res.headers()); - Err((ReqwestResponseError::StatusCode(status), retry_after)) + Err((ReqwestResponseError::StatusCode(res), retry_after)) } } } @@ -81,9 +81,9 @@ pub enum ReqwestGetHeaderError { #[derive(thiserror::Error, Debug)] pub enum ReqwestResponseError { #[error("Reqwest error {0:?}")] - Reqwest(reqwest::Error), - #[error("Status code {0:?}")] - StatusCode(reqwest::StatusCode), + Request(reqwest::Error), + #[error("Url: {}, Status Code: {}, Headers: {:?}", .0.url(), .0.status(), .0.headers())] + StatusCode(Response), } /// Parse the `Retry-After` response header into a [`Duration`]. @@ -215,7 +215,7 @@ impl HttpRequestBuilder for ManualRedirectRequestBuilder { let resp = req .send() .await - .map_err(|e| (ReqwestResponseError::Reqwest(e), None))?; + .map_err(|e| (ReqwestResponseError::Request(e), None))?; // DEBUG ASSERT: If reqwest auto-followed redirects, resp.url() will differ // from the URL we sent the request to. This means the inner Client was NOT @@ -234,12 +234,12 @@ impl HttpRequestBuilder for ManualRedirectRequestBuilder { Ok(resp) } else { let retry_after = parse_retry_after(resp.headers()); - Err((ReqwestResponseError::StatusCode(status), retry_after)) + Err((ReqwestResponseError::StatusCode(resp), retry_after)) }; } if self.redirect_count >= self.max_redirects { let retry_after = parse_retry_after(resp.headers()); - return Err((ReqwestResponseError::StatusCode(status), retry_after)); + return Err((ReqwestResponseError::StatusCode(resp), retry_after)); } let location = if let Some(v) = resp.headers().get(header::LOCATION) && let Ok(s) = v.to_str() @@ -247,11 +247,11 @@ impl HttpRequestBuilder for ManualRedirectRequestBuilder { s } else { let retry_after = parse_retry_after(resp.headers()); - return Err((ReqwestResponseError::StatusCode(status), retry_after)); + return Err((ReqwestResponseError::StatusCode(resp), retry_after)); }; let Ok(mut next_url) = self.url.join(location) else { let retry_after = parse_retry_after(resp.headers()); - return Err((ReqwestResponseError::StatusCode(status), retry_after)); + return Err((ReqwestResponseError::StatusCode(resp), retry_after)); }; // RFC 9110 §10.2.2: If the Location header lacks a fragment, // inherit it from the original request URI. @@ -296,7 +296,7 @@ mod tests { )] use super::*; use crate::{ - http::{HttpError, HttpPuller, Prefetch}, + http::{HttpPuller, Prefetch}, url_info::FileId, }; use fast_pull::{ @@ -411,17 +411,9 @@ mod tests { match client.prefetch(url).await { Ok(info) => unreachable!("404 status code should not success: {info:?}"), Err((err, _)) => match err { - HttpError::Request(e) => match e { - ReqwestResponseError::Reqwest(error) => unreachable!("{error:?}"), - ReqwestResponseError::StatusCode(status_code) => { - assert_eq!(status_code, StatusCode::NOT_FOUND); - } - }, - HttpError::Chunk(_, _) | HttpError::Irrecoverable => { - unreachable!() - } - HttpError::MismatchedBody(file_id, _) => { - unreachable!("404 status code should not return mismatched body: {file_id:?}") + ReqwestResponseError::Request(error) => unreachable!("{error:?}"), + ReqwestResponseError::StatusCode(resp) => { + assert_eq!(resp.status(), StatusCode::NOT_FOUND); } }, } From bc8cb7c5ea5849a7e21b848d221890ba1db10832 Mon Sep 17 00:00:00 2001 From: share121 Date: Sun, 26 Jul 2026 20:29:42 +0800 Subject: [PATCH 10/26] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- Cargo.lock | 4 +- crates/fast-down-api/Cargo.toml | 2 +- crates/fast-down-api/src/core/download.rs | 682 +++++++++++++--------- crates/fast-down-api/src/core/prefetch.rs | 5 +- crates/fast-down-api/src/core/state.rs | 2 +- crates/fast-down-api/src/event.rs | 2 +- 6 files changed, 408 insertions(+), 289 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75bb1a4..73301ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1181,9 +1181,9 @@ dependencies = [ [[package]] name = "path_helper" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8b6f3d03172dc1dac2005cfb2486d79d20f3e256383e14bcc6f23d73af9bc" +checksum = "b1cf0534c5a78bd12bd2c8d9ad3231f22ef07fc2205b5093e63e40cab4c4a8fc" dependencies = [ "mime_guess", "sanitize-filename", diff --git a/crates/fast-down-api/Cargo.toml b/crates/fast-down-api/Cargo.toml index cbeaf39..28f9a01 100644 --- a/crates/fast-down-api/Cargo.toml +++ b/crates/fast-down-api/Cargo.toml @@ -41,7 +41,7 @@ reqwest = { version = "0.13.4", default-features = false, features = [ thiserror.workspace = true crossfire.workspace = true anyhow = "1.0.103" -path_helper = { version = "0.1.3", features = [ +path_helper = { version = "0.1.9", features = [ "auto_ext", "sanitize", "tokio", diff --git a/crates/fast-down-api/src/core/download.rs b/crates/fast-down-api/src/core/download.rs index 09e2dba..3916826 100644 --- a/crates/fast-down-api/src/core/download.rs +++ b/crates/fast-down-api/src/core/download.rs @@ -5,7 +5,7 @@ use crate::{ utils::{ForceSendExt, build_header, gen_path}, }; use fast_down::{ - BoxPusher, Merge, + BoxPusher, Merge, UrlInfo, fast_puller::{FastDownPuller, FastDownPullerOptions, build_client}, file::{CacheFilePusher, MmapFilePusher}, handle::SharedHandle, @@ -16,12 +16,12 @@ use fast_down::{ }; use inherit_config::ConfigLayer; use parking_lot::Mutex; -use path_helper::{FileStemExt, tokio::gen_unique_path}; +use path_helper::{IterStemExt, tokio::gen_unique_path}; +use reqwest::Response; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::{ fs::{self, File, OpenOptions}, - select, task::JoinError, }; use tokio_util::sync::CancellationToken; @@ -32,6 +32,57 @@ const STATE_STORE_BYTES: u64 = 16 * 1024 * 1024; /// Persist the download state after at least this many `PushProgress` events. const STATE_STORE_EVENTS: usize = 512; +/// Outcome of trying to acquire a writable `.part` file + the matching resume state. +#[allow(clippy::large_enum_variant)] +enum Acquire { + /// A file is ready to write, optionally carrying a previously-saved resume state. + /// + /// The large fields are boxed so the variant stays small (`large_enum_variant`). + Ready { + file: File, + effective: Config, + parsed: PartialConfig, + resume_state: Option, + }, + /// Unique-name collision: the caller should regenerate the stem and retry. + CollisionRetry, + /// An unrecoverable error was already reported via `tx`; the caller should stop. + Abort, +} + +/// Outcome of probing an existing `.part`/`.fd` pair for resume eligibility. +/// +/// The classification is pure (see [`classify_resume`]); this enum only names +/// the three possible results so the resume *contract* lives in one place. +#[allow(clippy::large_enum_variant)] +enum ResumeProbe { + /// File + state both present and the state still matches the remote file. + Valid { file: File, state: DownloadState }, + /// An explicit resume was requested but the pair is unusable (stale or + /// missing); the caller must report and stop. + GiveUp(ResumeError), + /// Stale or missing state on a plain download; the caller drops the partial + /// files and opens fresh. + Discard, +} + +/// Borrow the shared `OpenOptions` presets used to open the `.part` file. +fn open_existing() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(false); + o +} +fn open_create() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(true); + o +} +fn open_create_new() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create_new(true); + o +} + pub struct DownloadHandle { handle: SharedHandle<()>, } @@ -118,15 +169,18 @@ impl DownloadHandle { config.local_address.first().copied(), config.max_redirects, )?; - let handle = tokio::spawn(Self::run( - url, - config, - partial_config, - client, - tx, - cancel_token, - tmp_path, - )); + let handle = tokio::spawn( + Self::run( + url, + config, + partial_config, + client, + tx, + cancel_token, + tmp_path, + ) + .force_send(), + ); let handle = SharedHandle::new(handle); Ok(Self { handle }) } @@ -134,24 +188,25 @@ impl DownloadHandle { /// The entire download pipeline, consolidated into a single function: /// /// 1. prefetch + resolve the destination path, - /// 2. try to resume from an existing `.part`/`.fd` pair (or start fresh), - /// 3. build the puller/pusher and run the transfer (multi or single), - /// 4. forward engine events, persist the resume state with debounce, - /// 5. on cancel keep `.part`+`.fd`; otherwise rename into place and drop + /// 2. per-iteration: try to resume from an existing `.part`/`.fd` pair (or + /// start fresh), then build the puller/pusher and run the transfer + /// (multi or single), + /// 3. forward engine events, persist the resume state with debounce, + /// 4. on cancel keep `.part`+`.fd`; otherwise rename into place and drop /// the state file. /// /// In `overwrite` mode (`unique = false`) a single create attempt is made; /// in `unique` mode (`unique = true`) a fresh `create_new` is retried with a /// regenerated stem (`xxx (1).mp4`, …) until a free path is found. /// - /// The build step and the transfer step are each wrapped in `force_send` - /// because the futures involved are not provably `Send` (they move the file - /// handle / response into the engine builders, and the engine's event- - /// channel receiver is `!Send`). They still run on this task, so the - /// `unsafe impl Send` assertions are sound. The transfer step is kept - /// *outside* `run_until_cancelled` so a mid-download cancel is handled by - /// its own event loop + final state store, rather than being silently - /// dropped by `run_until_cancelled`. + /// The whole `run` future is wrapped in `force_send` at the `spawn` site + /// because the futures it drives are not provably `Send`: they move the + /// file handle / response into the engine builders, and the engine's + /// event-channel receiver is `!Send`. The future still runs on this task, + /// so the `unsafe impl Send` assertions are sound. The transfer event loop + /// is kept *outside* `run_until_cancelled` (only the puller/pusher build + /// uses it) so a mid-download cancel is handled by the loop's final state + /// store rather than being silently dropped. #[allow(clippy::too_many_lines)] async fn run( url: Url, @@ -165,137 +220,75 @@ impl DownloadHandle { let Some((info, resp)) = prefetch(&url, &config, &client, &tx).await else { return; }; - // Probe the caller-provided `tmp_path` asynchronously so we never block - // the executor thread with a synchronous `Path::exists()` (which would - // stall a `current_thread` runtime or a slow network mount on the - // calling worker). `try_exists` is the async, error-distinguishing - // equivalent of `std::path::Path::exists`; we treat an undeterminable - // probe (`Err`) the same as "absent" and fall back to a fresh download. - // A present `.part` means we must force-resume this exact target. - let tmp_exists = match &tmp_path { - Some(p) => tokio::fs::try_exists(p).await.unwrap_or(false), - None => false, + + // Resolve whether we are resuming from an explicit temp file (resume()) + // or starting a fresh download (download()). Never a synchronous + // `Path::exists()`, which would block the executor on a slow mount. + let (tmp_path, tmp_exists) = match tmp_path { + Some(path) => { + if fs::try_exists(&path).await.unwrap_or(false) { + config.resume = true; + (Some(path), true) + } else { + (None, false) + } + } + None => (None, false), }; - if tmp_exists { - config.resume = true; - } - // When an explicit final path is given (resume with a caller-provided - // `tmp_path`), use it verbatim and never auto-regenerate a unique name. - let (origin_final, unique) = if tmp_exists { - let base = tmp_path.as_ref().unwrap().with_extension(""); - (base, false) + + // Resolve the destination path + whether we run in unique-name mode. + let (origin_final, unique) = if let Some(path) = &tmp_path { + (path.with_extension(""), false) } else { - ( - tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError), - !config.overwrite, - ) + let p = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError); + (p, !config.overwrite) }; - let can_resume = config.resume && info.fast_download; if tmp_exists && !info.fast_download { let _ = tx.send(Event::ResumeError(ResumeError::NotResumable)); return; } + let can_resume = config.resume && info.fast_download; - let open_existing = { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create(false); - o - }; - let open_create = { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create(true); - o - }; - let open_create_new = { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create_new(true); - o - }; - - let mut final_path = origin_final.clone(); - let mut i = 0usize; - loop { + // `resp` is consumed exactly once, by `build_pipeline` on the iteration + // that actually starts a transfer (collision retries `continue` before it). + for final_path in origin_final.iter_stem() { let tmp = final_path.with_added_extension("part"); let cfg = final_path.with_added_extension("fd"); - // ---- 1. Try to resume from an existing `.part`/`.fd` pair ---- - let resumed: Option<(File, Config, PartialConfig, DownloadState)> = if can_resume { - let (open_res, load_res) = - tokio::join!(open_existing.open(&tmp), DownloadState::load(&cfg)); - match (open_res, load_res) { - (Ok(file), Ok(state)) if state.validate(&info) => { - let mut pc = partial_config.clone(); - if let Some(c) = &state.config { - pc.inherit_from(c); - } - let parsed = pc.clone(); - let effective = pc.build(); - Some((file, effective, parsed, state)) - } - (Ok(_), Ok(_)) => { - if tmp_exists { - let _ = tx.send(Event::ResumeError(ResumeError::FileChanged)); - return; - } - // Stale state: discard and start a fresh download. - let _ = fs::remove_file(&tmp).await; - let _ = fs::remove_file(&cfg).await; - None - } - _ => { - if tmp_exists { - let _ = tx.send(Event::ResumeError(ResumeError::NoStateFile)); - return; - } - None - } - } - } else { - None - }; - - // ---- 2. Pick the file to write + matching resume state ---- - let (file, mut effective, parsed, resume_state): ( - File, - Config, - PartialConfig, - Option, - ) = if let Some((f, eff, par, st)) = resumed { - (f, eff, par, Some(st)) - } else { - let f = if unique { - open_create_new.open(&tmp).await.ok() - } else { - Some(tx_err!(open_create.open(&tmp).await, tx, BuildPusherError)) - }; - if let Some(f) = f { - let parsed = partial_config.clone(); - (f, config.clone(), parsed, None) - } else { - if !unique { - return; - } - // Collision in unique mode: regenerate the stem and retry. - i += 1; - final_path = origin_final.with_added_file_stem_prefix(format!(" {i}")); - continue; - } + // ---- 1. Resume from / open the `.part` file (handles unique collisions) ---- + let acquired = try_acquire_target( + &tx, + can_resume, + tmp_exists, + unique, + &info, + &partial_config, + &config, + &tmp, + &cfg, + ) + .await; + let (file, effective, parsed, resume_state) = match acquired { + Acquire::CollisionRetry => continue, + Acquire::Abort => return, + Acquire::Ready { + file, + effective, + parsed, + resume_state, + } => (file, effective, parsed, resume_state), }; - // ===== attempt: start, download, persist, rename ===== + // ===== attempt: build state, emit events, transfer, persist, rename ===== let is_resume = resume_state.is_some(); let mut state = resume_state.unwrap_or_else(|| DownloadState::new(&url, &info, &parsed, &cfg)); - // Fold the saved progress into the engine's "already downloaded" set - // so it only fetches the remaining bytes. + // Announce a resumed download. The progress ranges were already folded + // into `effective.downloaded_chunk` inside `try_acquire_target`, so here + // we only notify the consumer. if is_resume { - if let Some(progress) = state.progress.clone() { - for p in progress { - effective.downloaded_chunk.merge_progress(p); - } - } let _ = tx.send(Event::Resumed { config_path: cfg.clone(), progress: state.progress.clone().unwrap_or_default(), @@ -305,91 +298,41 @@ impl DownloadHandle { let _ = tx.send(Event::Start { tmp_path: tmp.clone(), config_path: cfg.clone(), - url_info: info.clone(), parsed_config: parsed, }); - // ---- 3a. Build the puller + pusher ---- - // Wrapped in `force_send` (not provably `Send`). The build itself is - // fast and non-blocking, so a cancel landing here is rare and simply - // ends the task without a transfer. - let url_ref = &url; - let config_ref = &effective; - let info_ref = &info; - let ct = cancel_token.clone(); - let resp = Some(Arc::new(Mutex::new(Some(resp)))); - let built = ct - .run_until_cancelled(async move { - let puller = FastDownPuller::new(FastDownPullerOptions { - url: url_ref.clone(), - headers: build_header(&config_ref.headers).into(), - proxy: config_ref.proxy.as_deref(), - accept_invalid_certs: config_ref.accept_invalid_certs, - accept_invalid_hostnames: config_ref.accept_invalid_hostnames, - cookie_store: config_ref.cookie_store, - file_id: info_ref.file_id.clone(), - resp, - available_ips: config_ref.local_address.clone().into(), - max_redirects: config_ref.max_redirects, - }) - .map_err(|e| Box::new(Event::BuildClientError(e)))?; - let pusher = if cfg!(target_pointer_width = "64") - && info_ref.fast_download - && config_ref.write_method == WriteMethod::Mmap - { - MmapFilePusher::new(&file, info_ref.size, config_ref.sync_all) - .await - .map(BoxPusher::new) - } else { - CacheFilePusher::new( - file, - info_ref.size, - config_ref.sync_all, - config_ref.cache_high_watermark, - config_ref.cache_low_watermark, - config_ref.write_buffer_size, - ) - .await - .map(BoxPusher::new) - } - .map_err(|e| Box::new(Event::BuildPusherError(e)))?; - Ok::<_, Box>((puller, pusher)) - }) - .force_send() - .await; - let (puller, pusher) = match built { - Some(Ok(b)) => b, - Some(Err(e)) => { - let _ = tx.send(*e); - // Persist the (possibly empty) state so a later resume still - // finds a coherent `.fd`, then stop. - state.update(|_| {}); - let _ = state.store().await; - return; - } - None => { - // Cancelled before the transfer started: keep the partial - // files and persist state so a later `resume()` can continue. - state.update(|_| {}); - let _ = state.store().await; - return; - } + // ---- 3a. Build the puller + pusher (force_send; not provably Send) ---- + // The build itself is fast and non-blocking, so a cancel landing here + // is rare and simply ends the task without a transfer. + let Some((puller, pusher)) = build_pipeline( + &url, + &effective, + &info, + file, + resp, + cancel_token.clone(), + &tx, + ) + .await + else { + // Build failed (errored or cancelled before transfer): persist the + // (possibly empty) state so a later resume still finds a coherent .fd. + let _ = state.store().await; + return; }; - // ---- 3b. Run the transfer ---- + // ---- 3b. Run the transfer + forward engine events with debounced store ---- // Multi-threaded when the server supports range requests, otherwise a - // single sequential stream. This is a *separate* `force_send` future - // (the engine's event-channel receiver is `!Send`) kept outside - // `run_until_cancelled`, so a mid-download cancel is handled by this - // future's own event loop + final state store — `run_until_cancelled` - // would otherwise drop the inner future and skip persisting `.fd`. + // single sequential stream. Kept outside `run_until_cancelled` (see + // `run` doc) so a mid-download cancel is handled by this loop's final + // state store rather than being silently dropped. let res = if info.fast_download { download_multi( puller, pusher, fast_down::multi::DownloadOptions { download_chunks: invert( - effective.downloaded_chunk.iter().cloned(), + effective.downloaded_chunk.into_iter(), info.size, effective.chunk_window, ), @@ -412,67 +355,52 @@ impl DownloadHandle { ) }; - let tx2 = tx.clone(); - let state_ref = &mut state; - let ct2 = cancel_token.clone(); - (async move { - // All events (including `PushProgress`) flow through the single - // `event_chain`. `PushProgress` also updates `state` with - // debounced store so the `.fd` always reflects the truth. - let mut store_events: usize = 0; - let mut store_bytes: u64 = 0; - loop { - select! { - e = res.event_chain.recv() => { - match e { - Ok(e) => { - if let fast_down::Event::PushProgress(range) = &e { - store_events += 1; - store_bytes += range.end - range.start; - state_ref.merge_progress(range.clone()); - } - let _ = match e { - fast_down::Event::Pulling(id) => tx2.send(Event::Pulling(id)), - fast_down::Event::PullError(id, e) => tx2.send(Event::PullError(id, anyhow::anyhow!(e))), - fast_down::Event::PullTimeout(id) => tx2.send(Event::PullTimeout(id)), - fast_down::Event::PullProgress(id, range) => tx2.send(Event::PullProgress(id, range)), - fast_down::Event::Pushing(id, range) => tx2.send(Event::Pushing(id, range)), - fast_down::Event::PushError(id, range, e) => tx2.send(Event::PushError(id, range, anyhow::anyhow!(e))), - fast_down::Event::PushProgress(range) => tx2.send(Event::PushProgress(range)), - fast_down::Event::Flushing => tx2.send(Event::Flushing), - fast_down::Event::FlushError(e) => tx2.send(Event::FlushError(anyhow::anyhow!(e))), - fast_down::Event::Finished(id) => tx2.send(Event::Finished(id)), - }; - if store_events >= STATE_STORE_EVENTS - || store_bytes >= STATE_STORE_BYTES - { - let _ = state_ref.store().await; - store_events = 0; - store_bytes = 0; - } - } - Err(_) => break, - } - } - () = ct2.cancelled() => break, + // All events (including `PushProgress`) flow through the single + // `event_chain`. `PushProgress` also updates `state` with debounced + // store so the `.fd` always reflects the truth. + let mut store_events: usize = 0; + let mut store_bytes: u64 = 0; + while let Ok(e) = res.event_chain.recv().await { + if let fast_down::Event::PushProgress(range) = &e { + store_events += 1; + store_bytes += range.end - range.start; + state.merge_progress(range.clone()); + } + let _ = match e { + fast_down::Event::Pulling(id) => tx.send(Event::Pulling(id)), + fast_down::Event::PullError(id, e) => { + tx.send(Event::PullError(id, anyhow::anyhow!(e))) + } + fast_down::Event::PullTimeout(id) => tx.send(Event::PullTimeout(id)), + fast_down::Event::PullProgress(id, range) => { + tx.send(Event::PullProgress(id, range)) + } + fast_down::Event::Pushing(id, range) => tx.send(Event::Pushing(id, range)), + fast_down::Event::PushError(id, range, e) => { + tx.send(Event::PushError(id, range, anyhow::anyhow!(e))) + } + fast_down::Event::PushProgress(range) => tx.send(Event::PushProgress(range)), + fast_down::Event::Flushing => tx.send(Event::Flushing), + fast_down::Event::FlushError(e) => { + tx.send(Event::FlushError(anyhow::anyhow!(e))) } + fast_down::Event::Finished(id) => tx.send(Event::Finished(id)), + }; + if store_events >= STATE_STORE_EVENTS || store_bytes >= STATE_STORE_BYTES { + let _ = state.store().await; + store_events = 0; + store_bytes = 0; } + } - if let Err(e) = res.join().await { - let _ = tx2.send(Event::JoinError(e)); - } - // Force a final store so the `.fd` always reflects the truth - // (also covers the cancelled-before-finish case). - let _ = state_ref.store().await; - }) - .force_send() - .await; + if let Err(e) = res.join().await { + let _ = tx.send(Event::JoinError(e)); + } // The `.fd` state file captures the current download progress (url, // config, size, etag) so a later resume can validate the remote file // identity. Called unconditionally even when cancelled before the // transfer finished. - state.update(|_| {}); let _ = state.store().await; // If the download was cancelled, do NOT rename the `.part` and do NOT // remove the `.fd` state file: keep both so a later `download()`/ @@ -480,30 +408,218 @@ impl DownloadHandle { if cancel_token.is_cancelled() { return; } - // In unique mode, reserve a fresh unique destination right before - // rename: the final file may have been created by someone else while - // we were downloading. `gen_unique_path` atomically creates an empty - // placeholder (create_new) which the rename below then replaces, - // closing the TOCTOU gap. - let dest = if unique { - tx_err!(gen_unique_path(&final_path).await, tx, GenPathError) - } else { - final_path.clone() - }; - if let Err(e) = fs::rename(&tmp, &dest).await { - // Best-effort: drop the empty placeholder we just reserved so a - // failed rename doesn't leave an orphan `xxx (1).mp4` behind. - if unique { - let _ = fs::remove_file(&dest).await; + finalize(&tx, unique, &tmp, &cfg, &final_path).await; + return; + } + } +} + +/// Try to resume from an existing `.part`/`.fd` pair, or open a fresh `.part` +/// file. Returns: +/// +/// - [`Acquire::Ready`] with a writable `file` + the effective config + the +/// resume state (if any); +/// - [`Acquire::CollisionRetry`] in unique mode when `create_new` failed +/// (treats the failure as a name collision and asks the caller to retry); +/// - [`Acquire::Abort`] when an unrecoverable error has already been reported +/// through `tx` (e.g. a `resume()` contract violation, or a non-unique open +/// failure), and the caller should stop. +/// +/// Classify the result of probing an existing `.part`/`.fd` pair for resume. +/// +/// Pure: no I/O, no event emission. The two probe results (`open` the `.part`, +/// `load` the `.fd`) map onto one of three outcomes so the resume *contract* +/// lives in a single, unit-testable place. The outcomes are +/// [`ResumeProbe::Valid`] (file + state present and still match the remote +/// file; resume from it), [`ResumeProbe::GiveUp`] (an explicit resume was +/// requested but the pair is unusable; report and stop), and +/// [`ResumeProbe::Discard`] (stale or missing state on a plain download; drop +/// the partial files and open fresh). +fn classify_resume( + open_res: std::io::Result, + load_res: Result, + tmp_exists: bool, + info: &UrlInfo, +) -> ResumeProbe { + match (open_res, load_res) { + (Ok(file), Ok(state)) if state.validate(info) => ResumeProbe::Valid { file, state }, + (Ok(_), Ok(_)) if tmp_exists => ResumeProbe::GiveUp(ResumeError::FileChanged), + _ if tmp_exists => ResumeProbe::GiveUp(ResumeError::NoStateFile), + _ => ResumeProbe::Discard, + } +} + +#[allow(clippy::too_many_arguments)] +async fn try_acquire_target( + tx: &Tx, + can_resume: bool, + tmp_exists: bool, + unique: bool, + info: &UrlInfo, + partial_config: &PartialConfig, + config: &Config, + tmp: &Path, + cfg: &Path, +) -> Acquire { + // ---- 1. Try to resume from an existing `.part`/`.fd` pair ---- + if can_resume { + let opener = open_existing(); + let (open_res, load_res) = tokio::join!(opener.open(tmp), DownloadState::load(cfg)); + match classify_resume(open_res, load_res, tmp_exists, info) { + ResumeProbe::Valid { file, state } => { + let mut pc = partial_config.clone(); + if let Some(c) = &state.config { + pc.inherit_from(c); + } + let parsed = pc.clone(); + let mut effective = pc.build(); + // Fold the saved progress into the engine's "already downloaded" + // set so it only fetches the remaining bytes. This is the other + // half of reconstructing `effective` from the resume state; the + // `config` half above comes from `state.config`. + if let Some(progress) = state.progress.clone() { + for p in progress { + effective.downloaded_chunk.merge_progress(p); + } } - let _ = tx.send(Event::RenameFailed(e)); + return Acquire::Ready { + file, + effective, + parsed, + resume_state: Some(state), + }; + } + ResumeProbe::GiveUp(err) => { + // Explicit resume target but the `.part`/`.fd` pair is unusable + // (stale or missing): report and stop rather than silently + // re-downloading (resume contract). + let _ = tx.send(Event::ResumeError(err)); + return Acquire::Abort; + } + ResumeProbe::Discard => { + // Stale or missing state on a plain download: drop the partial + // files and fall through to a fresh open below. + let _ = fs::remove_file(tmp).await; + let _ = fs::remove_file(cfg).await; + } + } + } + + // ---- 2. Fresh open (also the fall-through after discarding a stale state) ---- + let f = if unique { + open_create_new().open(tmp).await.ok() + } else { + match open_create().open(tmp).await { + Ok(f) => Some(f), + Err(e) => { + let _ = tx.send(Event::BuildPusherError(e)); + return Acquire::Abort; + } + } + }; + f.map_or_else( + || Acquire::CollisionRetry, + |file| Acquire::Ready { + file, + effective: config.clone(), + parsed: partial_config.clone(), + resume_state: None, + }, + ) +} + +/// Build the puller + pusher inside a `force_send` + `run_until_cancelled` +/// future (not provably `Send`; see `run` doc). Yields `None` on a build error +/// (the error event is already sent through `tx`) or on cancel-before-transfer; +/// the caller is responsible for persisting state and stopping. +async fn build_pipeline( + url: &Url, + effective: &Config, + info: &UrlInfo, + file: File, + resp: Response, + cancel_token: CancellationToken, + tx: &Tx, +) -> Option<(FastDownPuller, BoxPusher)> { + let ct = cancel_token; + let resp = Some(Arc::new(Mutex::new(Some(resp)))); + let built = ct + .run_until_cancelled(async move { + let puller = FastDownPuller::new(FastDownPullerOptions { + url: url.clone(), + headers: build_header(&effective.headers).into(), + proxy: effective.proxy.as_deref(), + accept_invalid_certs: effective.accept_invalid_certs, + accept_invalid_hostnames: effective.accept_invalid_hostnames, + cookie_store: effective.cookie_store, + file_id: info.file_id.clone(), + resp, + available_ips: effective.local_address.clone().into(), + max_redirects: effective.max_redirects, + }) + .map_err(Event::BuildClientError)?; + let pusher = if cfg!(target_pointer_width = "64") + && info.fast_download + && effective.write_method == WriteMethod::Mmap + { + MmapFilePusher::new(&file, info.size, effective.sync_all) + .await + .map(BoxPusher::new) + } else { + CacheFilePusher::new( + file, + info.size, + effective.sync_all, + effective.cache_high_watermark, + effective.cache_low_watermark, + effective.write_buffer_size, + ) + .await + .map(BoxPusher::new) + } + .map_err(Event::BuildPusherError)?; + Ok::<_, Event>((puller, pusher)) + }) + .await; + match built { + Some(Ok(b)) => Some(b), + Some(Err(e)) => { + let _ = tx.send(e); + None + } + None => None, + } +} + +/// Rename the finished `.part` into place and drop the `.fd` state file. +/// +/// In unique mode, a fresh unique destination is reserved right before rename +/// via `gen_unique_path` (atomic `create_new`), closing the TOCTOU gap where +/// the final file could have been created by someone else during the download. +/// On any failure the relevant error event is sent through `tx`. +async fn finalize(tx: &Tx, unique: bool, tmp: &Path, cfg: &Path, final_path: &Path) { + let dest = if unique { + match gen_unique_path(final_path).await { + Ok(p) => p, + Err(e) => { + let _ = tx.send(Event::GenPathError(e)); return; } - let _ = tx.send(Event::Renamed(dest)); - // Success: the download is complete and renamed, so the state file is - // no longer needed. Best-effort cleanup only (kept on cancel). - let _ = fs::remove_file(&cfg).await; - return; } + } else { + final_path.to_path_buf() + }; + if let Err(e) = fs::rename(tmp, &dest).await { + // Best-effort: drop the empty placeholder we just reserved so a failed + // rename doesn't leave an orphan `xxx (1).mp4` behind. + if unique { + let _ = fs::remove_file(&dest).await; + } + let _ = tx.send(Event::RenameFailed(e)); + return; } + let _ = tx.send(Event::Renamed(dest)); + // Success: the download is complete and renamed, so the state file is no + // longer needed. Best-effort cleanup only (kept on cancel). + let _ = fs::remove_file(cfg).await; } diff --git a/crates/fast-down-api/src/core/prefetch.rs b/crates/fast-down-api/src/core/prefetch.rs index 0ee3378..5452023 100644 --- a/crates/fast-down-api/src/core/prefetch.rs +++ b/crates/fast-down-api/src/core/prefetch.rs @@ -12,7 +12,10 @@ pub async fn prefetch( let mut retry_count = 0; loop { match client.prefetch(url.clone()).await { - Ok(t) => break Some(t), + Ok(t) => { + let _ = tx.send(Event::Prefetch(t.0.clone())); + break Some(t); + } Err((e, t)) => { let _ = tx.send(Event::PrefetchError(e)); retry_count += 1; diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs index 11b9aab..dab413e 100644 --- a/crates/fast-down-api/src/core/state.rs +++ b/crates/fast-down-api/src/core/state.rs @@ -46,7 +46,7 @@ impl DownloadState { progress: Some(Vec::new()), size: Some(url_info.size), }, - is_dirty: false, + is_dirty: true, config_path: config_path.to_path_buf(), } } diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs index 6ee199c..f10428b 100644 --- a/crates/fast-down-api/src/event.rs +++ b/crates/fast-down-api/src/event.rs @@ -23,6 +23,7 @@ pub enum ResumeError { #[allow(clippy::large_enum_variant)] pub enum Event { + Prefetch(UrlInfo), PrefetchError(ReqwestResponseError), GenPathError(std::io::Error), BuildClientError(reqwest::Error), @@ -37,7 +38,6 @@ pub enum Event { Start { tmp_path: PathBuf, config_path: PathBuf, - url_info: UrlInfo, parsed_config: PartialConfig, }, /// Emitted when a download resumes from a previously-saved state, before From 0178df7b62836b7e6aa235389cdf8fc8c4de8601 Mon Sep 17 00:00:00 2001 From: share121 Date: Mon, 27 Jul 2026 12:27:03 +0800 Subject: [PATCH 11/26] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/src/config.rs | 26 +- .../src/core/download/acquire.rs | 179 ++++++++++ .../src/core/download/finalize.rs | 37 ++ .../src/core/{download.rs => download/mod.rs} | 334 ++---------------- .../src/core/download/pipeline.rs | 75 ++++ crates/fast-down-api/src/core/mod.rs | 3 +- crates/fast-down-api/src/core/state.rs | 14 +- crates/fast-down-api/src/event.rs | 9 +- crates/fast-down-api/tests/resume.rs | 5 +- 9 files changed, 366 insertions(+), 316 deletions(-) create mode 100644 crates/fast-down-api/src/core/download/acquire.rs create mode 100644 crates/fast-down-api/src/core/download/finalize.rs rename crates/fast-down-api/src/core/{download.rs => download/mod.rs} (53%) create mode 100644 crates/fast-down-api/src/core/download/pipeline.rs diff --git a/crates/fast-down-api/src/config.rs b/crates/fast-down-api/src/config.rs index bfe750c..43d6ce8 100644 --- a/crates/fast-down-api/src/config.rs +++ b/crates/fast-down-api/src/config.rs @@ -1,5 +1,5 @@ -use fast_down::{ProgressEntry, Proxy}; -use inherit_config::InheritConfig; +use fast_down::{Merge, ProgressEntry, Proxy}; +use inherit_config::{ConfigLayer, InheritConfig}; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, net::IpAddr, path::PathBuf, time::Duration}; @@ -168,3 +168,25 @@ pub struct Config { /// 是否覆盖已存在的文件，推荐值: `false` pub overwrite: bool, } + +impl PartialConfig { + /// Build the effective [`Config`], folding previously-saved resume progress + /// into `downloaded_chunk` so the engine only fetches the remaining bytes. + /// + /// This is the *single* place where resume progress enters the resolved + /// config. Callers must go through it instead of `build()` followed by a + /// separate `merge_progress`, so `effective` stays a pure function of the + /// partial config + the saved progress — no post-`build` mutation, and the + /// config layer (`self`) and the resolved `Config` can never drift on + /// progress because the layer simply does not carry it. + #[must_use] + pub fn build_seeded(&self, progress: Option<&[ProgressEntry]>) -> Config { + let mut c = self.clone().build(); + if let Some(p) = progress { + for e in p { + c.downloaded_chunk.merge_progress(e.clone()); + } + } + c + } +} diff --git a/crates/fast-down-api/src/core/download/acquire.rs b/crates/fast-down-api/src/core/download/acquire.rs new file mode 100644 index 0000000..df45f74 --- /dev/null +++ b/crates/fast-down-api/src/core/download/acquire.rs @@ -0,0 +1,179 @@ +use crate::{Config, DownloadState, Event, PartialConfig, ResumeError, Tx}; +use fast_down::UrlInfo; +use inherit_config::ConfigLayer; +use std::path::Path; +use tokio::fs::{self, File, OpenOptions}; + +/// Outcome of trying to acquire a writable `.part` file + the matching resume state. +#[allow(clippy::large_enum_variant)] +pub(super) enum Acquire { + /// A file is ready to write, optionally carrying a previously-saved resume state. + /// + /// The large fields are boxed so the variant stays small (`large_enum_variant`). + Ready { + file: File, + effective: Config, + parsed: PartialConfig, + resume_state: Option, + }, + /// Unique-name collision: the caller should regenerate the stem and retry. + CollisionRetry, + /// An unrecoverable error was already reported via `tx`; the caller should stop. + Abort, +} + +/// Outcome of probing an existing `.part`/`.fd` pair for resume eligibility. +/// +/// The classification is pure (see [`classify_resume`]); this enum only names +/// the three possible results so the resume *contract* lives in one place. +#[allow(clippy::large_enum_variant)] +enum ResumeProbe { + /// File + state both present and the state still matches the remote file. + Valid { file: File, state: DownloadState }, + /// An explicit resume was requested but the pair is unusable (stale or + /// missing); the caller must report and stop. + GiveUp(ResumeError), + /// Stale or missing state on a plain download; the caller drops the partial + /// files and opens fresh. + Discard, +} + +/// Borrow the shared `OpenOptions` presets used to open the `.part` file. +fn open_existing() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(false); + o +} +fn open_create() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(true); + o +} +fn open_create_new() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create_new(true); + o +} + +/// Classify the result of probing an existing `.part`/`.fd` pair for resume. +/// +/// Pure: no I/O, no event emission. The two probe results (`open` the `.part`, +/// `load` the `.fd`) map onto one of three outcomes so the resume *contract* +/// lives in a single, unit-testable place. The outcomes are +/// [`ResumeProbe::Valid`] (file + state present and still match the remote +/// file; resume from it), [`ResumeProbe::GiveUp`] (an explicit resume was +/// requested but the pair is unusable; report and stop), and +/// [`ResumeProbe::Discard`] (stale or missing state on a plain download; drop +/// the partial files and open fresh). +fn classify_resume( + open_res: std::io::Result, + load_res: Result, + explicit_resume: bool, + info: &UrlInfo, +) -> ResumeProbe { + match (open_res, load_res) { + (Ok(file), Ok(state)) if state.validate(info) => ResumeProbe::Valid { file, state }, + // Only an *explicit* resume (`resume()`) treats a stale/missing state as + // a hard error (the resume contract). A plain `download()` that happens + // to find a leftover `.part`/`.fd` discards them and re-downloads + // silently — so `explicit_resume` (true only for `resume()`) gates the + // `GiveUp` arms and everything else falls through to `Discard`. + (Ok(_), Ok(state)) if explicit_resume => ResumeProbe::GiveUp(ResumeError::FileChanged { + local_file_id: state.file_id(), + local_file_size: state.size.unwrap_or(0), + remote_file_id: info.file_id.clone(), + remote_file_size: info.size, + }), + (Ok(_), Err(_)) if explicit_resume => ResumeProbe::GiveUp(ResumeError::NoStateFile), + _ => ResumeProbe::Discard, + } +} + +/// Try to resume from an existing `.part`/`.fd` pair, or open a fresh `.part` +/// file. Returns: +/// +/// - [`Acquire::Ready`] with a writable `file` + the effective config + the +/// resume state (if any); +/// - [`Acquire::CollisionRetry`] in unique mode when `create_new` failed +/// (treats the failure as a name collision and asks the caller to retry); +/// - [`Acquire::Abort`] when an unrecoverable error has already been reported +/// through `tx` (e.g. a `resume()` contract violation, or a non-unique open +/// failure), and the caller should stop. +#[allow(clippy::too_many_arguments)] +pub(super) async fn try_acquire_target( + tx: &Tx, + can_resume: bool, + explicit_resume: bool, + unique: bool, + info: &UrlInfo, + partial_config: &PartialConfig, + tmp: &Path, + cfg: &Path, +) -> Acquire { + // ---- 1. Try to resume from an existing `.part`/`.fd` pair ---- + if can_resume { + let opener = open_existing(); + let (open_res, load_res) = tokio::join!(opener.open(tmp), DownloadState::load(cfg)); + match classify_resume(open_res, load_res, explicit_resume, info) { + ResumeProbe::Valid { file, state } => { + let mut pc = partial_config.clone(); + if let Some(c) = &state.config { + pc.inherit_from(c); + } + // Reconstruct `effective` from the resume state in one step: the + // `config` half comes from `inherit_from(state.config)` above, and + // the saved-progress half is folded into `downloaded_chunk` inside + // `build_seeded`. So `effective` is a pure function of `pc` + the + // saved progress (no post-build mutation), and `parsed`/`effective` + // stay in sync by construction — `pc` carries no progress at all. + let effective = pc.build_seeded(state.progress.as_deref()); + let parsed = pc; + return Acquire::Ready { + file, + effective, + parsed, + resume_state: Some(state), + }; + } + ResumeProbe::GiveUp(err) => { + // Explicit resume target but the `.part`/`.fd` pair is unusable + // (stale or missing): report and stop rather than silently + // re-downloading (resume contract). + let _ = tx.send(Event::ResumeError(err)); + return Acquire::Abort; + } + ResumeProbe::Discard => { + // Stale or missing state on a plain download: drop the partial + // files and fall through to a fresh open below. + let _ = fs::remove_file(tmp).await; + let _ = fs::remove_file(cfg).await; + } + } + } + + // ---- 2. Fresh open (also the fall-through after discarding a stale state) ---- + let f = if unique { + open_create_new().open(tmp).await.ok() + } else { + match open_create().open(tmp).await { + Ok(f) => Some(f), + Err(e) => { + let _ = tx.send(Event::BuildPusherError(e)); + return Acquire::Abort; + } + } + }; + f.map_or_else( + || Acquire::CollisionRetry, + |file| { + let pc = partial_config.clone(); + let effective = pc.build_seeded(None); + Acquire::Ready { + file, + effective, + parsed: pc, + resume_state: None, + } + }, + ) +} diff --git a/crates/fast-down-api/src/core/download/finalize.rs b/crates/fast-down-api/src/core/download/finalize.rs new file mode 100644 index 0000000..4b7a4f8 --- /dev/null +++ b/crates/fast-down-api/src/core/download/finalize.rs @@ -0,0 +1,37 @@ +use crate::{Event, Tx}; +use path_helper::tokio::gen_unique_path; +use std::path::Path; +use tokio::fs; + +/// Rename the finished `.part` into place and drop the `.fd` state file. +/// +/// In unique mode, a fresh unique destination is reserved right before rename +/// via `gen_unique_path` (atomic `create_new`), closing the TOCTOU gap where +/// the final file could have been created by someone else during the download. +/// On any failure the relevant error event is sent through `tx`. +pub(super) async fn finalize(tx: &Tx, unique: bool, tmp: &Path, cfg: &Path, final_path: &Path) { + let dest = if unique { + match gen_unique_path(final_path).await { + Ok(p) => p, + Err(e) => { + let _ = tx.send(Event::GenPathError(e)); + return; + } + } + } else { + final_path.to_path_buf() + }; + if let Err(e) = fs::rename(tmp, &dest).await { + // Best-effort: drop the empty placeholder we just reserved so a failed + // rename doesn't leave an orphan `xxx (1).mp4` behind. + if unique { + let _ = fs::remove_file(&dest).await; + } + let _ = tx.send(Event::RenameFailed(e)); + return; + } + let _ = tx.send(Event::Renamed(dest)); + // Success: the download is complete and renamed, so the state file is no + // longer needed. Best-effort cleanup only (kept on cancel). + let _ = fs::remove_file(cfg).await; +} diff --git a/crates/fast-down-api/src/core/download.rs b/crates/fast-down-api/src/core/download/mod.rs similarity index 53% rename from crates/fast-down-api/src/core/download.rs rename to crates/fast-down-api/src/core/download/mod.rs index 3916826..b9dae02 100644 --- a/crates/fast-down-api/src/core/download.rs +++ b/crates/fast-down-api/src/core/download/mod.rs @@ -1,29 +1,17 @@ use crate::{ - Config, DownloadState, Event, PartialConfig, ResumeError, Tx, WriteMethod, - prefetch::prefetch, - tx_err, + Config, DownloadState, Event, PartialConfig, ResumeError, Tx, prefetch, tx_err, utils::{ForceSendExt, build_header, gen_path}, }; use fast_down::{ - BoxPusher, Merge, UrlInfo, - fast_puller::{FastDownPuller, FastDownPullerOptions, build_client}, - file::{CacheFilePusher, MmapFilePusher}, - handle::SharedHandle, - invert, - multi::download_multi, - reqwest::SmartRedirectClient, - single::download_single, + Total, fast_puller::build_client, handle::SharedHandle, invert, multi::download_multi, + reqwest::SmartRedirectClient, single::download_single, }; use inherit_config::ConfigLayer; -use parking_lot::Mutex; -use path_helper::{IterStemExt, tokio::gen_unique_path}; -use reqwest::Response; +use path_helper::IterStemExt; use std::path::{Path, PathBuf}; use std::sync::Arc; -use tokio::{ - fs::{self, File, OpenOptions}, - task::JoinError, -}; +use tokio::fs; +use tokio::task::JoinError; use tokio_util::sync::CancellationToken; use url::Url; @@ -32,56 +20,13 @@ const STATE_STORE_BYTES: u64 = 16 * 1024 * 1024; /// Persist the download state after at least this many `PushProgress` events. const STATE_STORE_EVENTS: usize = 512; -/// Outcome of trying to acquire a writable `.part` file + the matching resume state. -#[allow(clippy::large_enum_variant)] -enum Acquire { - /// A file is ready to write, optionally carrying a previously-saved resume state. - /// - /// The large fields are boxed so the variant stays small (`large_enum_variant`). - Ready { - file: File, - effective: Config, - parsed: PartialConfig, - resume_state: Option, - }, - /// Unique-name collision: the caller should regenerate the stem and retry. - CollisionRetry, - /// An unrecoverable error was already reported via `tx`; the caller should stop. - Abort, -} +mod acquire; +mod finalize; +mod pipeline; -/// Outcome of probing an existing `.part`/`.fd` pair for resume eligibility. -/// -/// The classification is pure (see [`classify_resume`]); this enum only names -/// the three possible results so the resume *contract* lives in one place. -#[allow(clippy::large_enum_variant)] -enum ResumeProbe { - /// File + state both present and the state still matches the remote file. - Valid { file: File, state: DownloadState }, - /// An explicit resume was requested but the pair is unusable (stale or - /// missing); the caller must report and stop. - GiveUp(ResumeError), - /// Stale or missing state on a plain download; the caller drops the partial - /// files and opens fresh. - Discard, -} - -/// Borrow the shared `OpenOptions` presets used to open the `.part` file. -fn open_existing() -> OpenOptions { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create(false); - o -} -fn open_create() -> OpenOptions { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create(true); - o -} -fn open_create_new() -> OpenOptions { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create_new(true); - o -} +use acquire::{Acquire, try_acquire_target}; +use finalize::finalize; +use pipeline::build_pipeline; pub struct DownloadHandle { handle: SharedHandle<()>, @@ -220,20 +165,13 @@ impl DownloadHandle { let Some((info, resp)) = prefetch(&url, &config, &client, &tx).await else { return; }; - - // Resolve whether we are resuming from an explicit temp file (resume()) - // or starting a fresh download (download()). Never a synchronous - // `Path::exists()`, which would block the executor on a slow mount. - let (tmp_path, tmp_exists) = match tmp_path { - Some(path) => { - if fs::try_exists(&path).await.unwrap_or(false) { - config.resume = true; - (Some(path), true) - } else { - (None, false) - } - } - None => (None, false), + let tmp_path = if let Some(path) = tmp_path + && fs::try_exists(&path).await.unwrap_or(false) + { + config.resume = true; + Some(path) + } else { + None }; // Resolve the destination path + whether we run in unique-name mode. @@ -244,11 +182,15 @@ impl DownloadHandle { (p, !config.overwrite) }; - if tmp_exists && !info.fast_download { + if tmp_path.is_some() && !info.fast_download { let _ = tx.send(Event::ResumeError(ResumeError::NotResumable)); return; } let can_resume = config.resume && info.fast_download; + // `explicit_resume` is true only when this is an explicit `resume()` call + // (a `tmp_path` was supplied and the `.part` exists); a plain `download()` + // is `false` and must silently fall back instead of erroring. + let explicit_resume = tmp_path.is_some(); // `resp` is consumed exactly once, by `build_pipeline` on the iteration // that actually starts a transfer (collision retries `continue` before it). @@ -257,18 +199,9 @@ impl DownloadHandle { let cfg = final_path.with_added_extension("fd"); // ---- 1. Resume from / open the `.part` file (handles unique collisions) ---- - let acquired = try_acquire_target( - &tx, - can_resume, - tmp_exists, - unique, - &info, - &partial_config, - &config, - &tmp, - &cfg, - ) - .await; + let acquired = + try_acquire_target(&tx, can_resume, explicit_resume, unique, &info, &partial_config, &tmp, &cfg) + .await; let (file, effective, parsed, resume_state) = match acquired { Acquire::CollisionRetry => continue, Acquire::Abort => return, @@ -363,7 +296,7 @@ impl DownloadHandle { while let Ok(e) = res.event_chain.recv().await { if let fast_down::Event::PushProgress(range) = &e { store_events += 1; - store_bytes += range.end - range.start; + store_bytes += range.total(); state.merge_progress(range.clone()); } let _ = match e { @@ -402,6 +335,7 @@ impl DownloadHandle { // identity. Called unconditionally even when cancelled before the // transfer finished. let _ = state.store().await; + // If the download was cancelled, do NOT rename the `.part` and do NOT // remove the `.fd` state file: keep both so a later `download()`/ // `resume()` can continue from where it left off (design doc §8). @@ -413,213 +347,3 @@ impl DownloadHandle { } } } - -/// Try to resume from an existing `.part`/`.fd` pair, or open a fresh `.part` -/// file. Returns: -/// -/// - [`Acquire::Ready`] with a writable `file` + the effective config + the -/// resume state (if any); -/// - [`Acquire::CollisionRetry`] in unique mode when `create_new` failed -/// (treats the failure as a name collision and asks the caller to retry); -/// - [`Acquire::Abort`] when an unrecoverable error has already been reported -/// through `tx` (e.g. a `resume()` contract violation, or a non-unique open -/// failure), and the caller should stop. -/// -/// Classify the result of probing an existing `.part`/`.fd` pair for resume. -/// -/// Pure: no I/O, no event emission. The two probe results (`open` the `.part`, -/// `load` the `.fd`) map onto one of three outcomes so the resume *contract* -/// lives in a single, unit-testable place. The outcomes are -/// [`ResumeProbe::Valid`] (file + state present and still match the remote -/// file; resume from it), [`ResumeProbe::GiveUp`] (an explicit resume was -/// requested but the pair is unusable; report and stop), and -/// [`ResumeProbe::Discard`] (stale or missing state on a plain download; drop -/// the partial files and open fresh). -fn classify_resume( - open_res: std::io::Result, - load_res: Result, - tmp_exists: bool, - info: &UrlInfo, -) -> ResumeProbe { - match (open_res, load_res) { - (Ok(file), Ok(state)) if state.validate(info) => ResumeProbe::Valid { file, state }, - (Ok(_), Ok(_)) if tmp_exists => ResumeProbe::GiveUp(ResumeError::FileChanged), - _ if tmp_exists => ResumeProbe::GiveUp(ResumeError::NoStateFile), - _ => ResumeProbe::Discard, - } -} - -#[allow(clippy::too_many_arguments)] -async fn try_acquire_target( - tx: &Tx, - can_resume: bool, - tmp_exists: bool, - unique: bool, - info: &UrlInfo, - partial_config: &PartialConfig, - config: &Config, - tmp: &Path, - cfg: &Path, -) -> Acquire { - // ---- 1. Try to resume from an existing `.part`/`.fd` pair ---- - if can_resume { - let opener = open_existing(); - let (open_res, load_res) = tokio::join!(opener.open(tmp), DownloadState::load(cfg)); - match classify_resume(open_res, load_res, tmp_exists, info) { - ResumeProbe::Valid { file, state } => { - let mut pc = partial_config.clone(); - if let Some(c) = &state.config { - pc.inherit_from(c); - } - let parsed = pc.clone(); - let mut effective = pc.build(); - // Fold the saved progress into the engine's "already downloaded" - // set so it only fetches the remaining bytes. This is the other - // half of reconstructing `effective` from the resume state; the - // `config` half above comes from `state.config`. - if let Some(progress) = state.progress.clone() { - for p in progress { - effective.downloaded_chunk.merge_progress(p); - } - } - return Acquire::Ready { - file, - effective, - parsed, - resume_state: Some(state), - }; - } - ResumeProbe::GiveUp(err) => { - // Explicit resume target but the `.part`/`.fd` pair is unusable - // (stale or missing): report and stop rather than silently - // re-downloading (resume contract). - let _ = tx.send(Event::ResumeError(err)); - return Acquire::Abort; - } - ResumeProbe::Discard => { - // Stale or missing state on a plain download: drop the partial - // files and fall through to a fresh open below. - let _ = fs::remove_file(tmp).await; - let _ = fs::remove_file(cfg).await; - } - } - } - - // ---- 2. Fresh open (also the fall-through after discarding a stale state) ---- - let f = if unique { - open_create_new().open(tmp).await.ok() - } else { - match open_create().open(tmp).await { - Ok(f) => Some(f), - Err(e) => { - let _ = tx.send(Event::BuildPusherError(e)); - return Acquire::Abort; - } - } - }; - f.map_or_else( - || Acquire::CollisionRetry, - |file| Acquire::Ready { - file, - effective: config.clone(), - parsed: partial_config.clone(), - resume_state: None, - }, - ) -} - -/// Build the puller + pusher inside a `force_send` + `run_until_cancelled` -/// future (not provably `Send`; see `run` doc). Yields `None` on a build error -/// (the error event is already sent through `tx`) or on cancel-before-transfer; -/// the caller is responsible for persisting state and stopping. -async fn build_pipeline( - url: &Url, - effective: &Config, - info: &UrlInfo, - file: File, - resp: Response, - cancel_token: CancellationToken, - tx: &Tx, -) -> Option<(FastDownPuller, BoxPusher)> { - let ct = cancel_token; - let resp = Some(Arc::new(Mutex::new(Some(resp)))); - let built = ct - .run_until_cancelled(async move { - let puller = FastDownPuller::new(FastDownPullerOptions { - url: url.clone(), - headers: build_header(&effective.headers).into(), - proxy: effective.proxy.as_deref(), - accept_invalid_certs: effective.accept_invalid_certs, - accept_invalid_hostnames: effective.accept_invalid_hostnames, - cookie_store: effective.cookie_store, - file_id: info.file_id.clone(), - resp, - available_ips: effective.local_address.clone().into(), - max_redirects: effective.max_redirects, - }) - .map_err(Event::BuildClientError)?; - let pusher = if cfg!(target_pointer_width = "64") - && info.fast_download - && effective.write_method == WriteMethod::Mmap - { - MmapFilePusher::new(&file, info.size, effective.sync_all) - .await - .map(BoxPusher::new) - } else { - CacheFilePusher::new( - file, - info.size, - effective.sync_all, - effective.cache_high_watermark, - effective.cache_low_watermark, - effective.write_buffer_size, - ) - .await - .map(BoxPusher::new) - } - .map_err(Event::BuildPusherError)?; - Ok::<_, Event>((puller, pusher)) - }) - .await; - match built { - Some(Ok(b)) => Some(b), - Some(Err(e)) => { - let _ = tx.send(e); - None - } - None => None, - } -} - -/// Rename the finished `.part` into place and drop the `.fd` state file. -/// -/// In unique mode, a fresh unique destination is reserved right before rename -/// via `gen_unique_path` (atomic `create_new`), closing the TOCTOU gap where -/// the final file could have been created by someone else during the download. -/// On any failure the relevant error event is sent through `tx`. -async fn finalize(tx: &Tx, unique: bool, tmp: &Path, cfg: &Path, final_path: &Path) { - let dest = if unique { - match gen_unique_path(final_path).await { - Ok(p) => p, - Err(e) => { - let _ = tx.send(Event::GenPathError(e)); - return; - } - } - } else { - final_path.to_path_buf() - }; - if let Err(e) = fs::rename(tmp, &dest).await { - // Best-effort: drop the empty placeholder we just reserved so a failed - // rename doesn't leave an orphan `xxx (1).mp4` behind. - if unique { - let _ = fs::remove_file(&dest).await; - } - let _ = tx.send(Event::RenameFailed(e)); - return; - } - let _ = tx.send(Event::Renamed(dest)); - // Success: the download is complete and renamed, so the state file is no - // longer needed. Best-effort cleanup only (kept on cancel). - let _ = fs::remove_file(cfg).await; -} diff --git a/crates/fast-down-api/src/core/download/pipeline.rs b/crates/fast-down-api/src/core/download/pipeline.rs new file mode 100644 index 0000000..94ebf62 --- /dev/null +++ b/crates/fast-down-api/src/core/download/pipeline.rs @@ -0,0 +1,75 @@ +use crate::{Config, Event, Tx, WriteMethod, utils::build_header}; +use fast_down::{ + BoxPusher, UrlInfo, + fast_puller::{FastDownPuller, FastDownPullerOptions}, + file::{CacheFilePusher, MmapFilePusher}, +}; +use parking_lot::Mutex; +use reqwest::Response; +use std::sync::Arc; +use tokio::fs::File; +use tokio_util::sync::CancellationToken; +use url::Url; + +/// Build the puller + pusher inside a `force_send` + `run_until_cancelled` +/// future (not provably `Send`; see `run` doc). Yields `None` on a build error +/// (the error event is already sent through `tx`) or on cancel-before-transfer; +/// the caller is responsible for persisting state and stopping. +pub(super) async fn build_pipeline( + url: &Url, + effective: &Config, + info: &UrlInfo, + file: File, + resp: Response, + cancel_token: CancellationToken, + tx: &Tx, +) -> Option<(FastDownPuller, BoxPusher)> { + let ct = cancel_token; + let resp = Some(Arc::new(Mutex::new(Some(resp)))); + let built = ct + .run_until_cancelled(async move { + let puller = FastDownPuller::new(FastDownPullerOptions { + url: url.clone(), + headers: build_header(&effective.headers).into(), + proxy: effective.proxy.as_deref(), + accept_invalid_certs: effective.accept_invalid_certs, + accept_invalid_hostnames: effective.accept_invalid_hostnames, + cookie_store: effective.cookie_store, + file_id: info.file_id.clone(), + resp, + available_ips: effective.local_address.clone().into(), + max_redirects: effective.max_redirects, + }) + .map_err(Event::BuildClientError)?; + let pusher = if cfg!(target_pointer_width = "64") + && info.fast_download + && effective.write_method == WriteMethod::Mmap + { + MmapFilePusher::new(&file, info.size, effective.sync_all) + .await + .map(BoxPusher::new) + } else { + CacheFilePusher::new( + file, + info.size, + effective.sync_all, + effective.cache_high_watermark, + effective.cache_low_watermark, + effective.write_buffer_size, + ) + .await + .map(BoxPusher::new) + } + .map_err(Event::BuildPusherError)?; + Ok::<_, Event>((puller, pusher)) + }) + .await; + match built { + Some(Ok(b)) => Some(b), + Some(Err(e)) => { + let _ = tx.send(e); + None + } + None => None, + } +} diff --git a/crates/fast-down-api/src/core/mod.rs b/crates/fast-down-api/src/core/mod.rs index 4529374..625df38 100644 --- a/crates/fast-down-api/src/core/mod.rs +++ b/crates/fast-down-api/src/core/mod.rs @@ -1,6 +1,7 @@ mod download; -pub mod prefetch; +mod prefetch; mod state; pub use download::*; +pub use prefetch::*; pub use state::*; diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs index dab413e..0a11af6 100644 --- a/crates/fast-down-api/src/core/state.rs +++ b/crates/fast-down-api/src/core/state.rs @@ -100,11 +100,7 @@ impl DownloadState { if !info.fast_download { return false; } - let saved = FileId { - etag: self.etag.clone().flatten(), - last_modified: self.last_modified.clone().flatten(), - }; - if saved != info.file_id { + if self.file_id() != info.file_id { return false; } true @@ -132,6 +128,14 @@ impl DownloadState { .as_ref() .map_or(0, |v| v.iter().map(|r| r.end - r.start).sum()) } + + #[must_use] + pub fn file_id(&self) -> FileId { + FileId { + etag: self.etag.clone().flatten(), + last_modified: self.last_modified.clone().flatten(), + } + } } impl Deref for DownloadState { diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs index f10428b..ea9f4b0 100644 --- a/crates/fast-down-api/src/event.rs +++ b/crates/fast-down-api/src/event.rs @@ -1,5 +1,5 @@ use crate::PartialConfig; -use fast_down::{ProgressEntry, UrlInfo, WorkerId, reqwest::ReqwestResponseError}; +use fast_down::{FileId, ProgressEntry, UrlInfo, WorkerId, reqwest::ReqwestResponseError}; use std::{path::PathBuf, sync::Arc}; use thiserror::Error; @@ -15,7 +15,12 @@ pub enum ResumeError { NoStateFile, /// The remote file changed (size / etag / last-modified mismatch), so resuming would corrupt the output. #[error("remote file changed, cannot resume")] - FileChanged, + FileChanged { + local_file_id: FileId, + local_file_size: u64, + remote_file_id: FileId, + remote_file_size: u64, + }, /// The server does not support resumable (range) downloads. #[error("server does not support resumable download")] NotResumable, diff --git a/crates/fast-down-api/tests/resume.rs b/crates/fast-down-api/tests/resume.rs index 50a5825..b7d1de1 100644 --- a/crates/fast-down-api/tests/resume.rs +++ b/crates/fast-down-api/tests/resume.rs @@ -423,7 +423,10 @@ async fn test_file_changed_resume_reports_error() { _ => None, }) .expect("expected Event::ResumeError"); - assert_eq!(err, ResumeError::FileChanged); + assert!( + matches!(err, ResumeError::FileChanged { .. }), + "expected Event::ResumeError(ResumeError::FileChanged), got {err:?}" + ); assert!( !events.iter().any(|e| matches!(e, Event::Renamed(_))), From 3ed6c6ec68f6603db3ec72097b5098a9906c8cae Mon Sep 17 00:00:00 2001 From: share121 Date: Mon, 27 Jul 2026 14:00:33 +0800 Subject: [PATCH 12/26] =?UTF-8?q?feat(fast-pull):=20SharedHandle=20?= =?UTF-8?q?=E6=94=AF=E6=8C=81=20clone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-pull/src/core/handle.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fast-pull/src/core/handle.rs b/crates/fast-pull/src/core/handle.rs index 4cbc452..d7a2d54 100644 --- a/crates/fast-pull/src/core/handle.rs +++ b/crates/fast-pull/src/core/handle.rs @@ -9,7 +9,7 @@ use tokio::{ /// Unlike a raw [`JoinHandle`], [`SharedHandle`] can be cloned and awaited /// concurrently without consuming the result. The first awaiter gets the result, /// subsequent awaiters will see the same cached result. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct SharedHandle { rx: watch::Receiver>>>, } From b4b19cbd783f7edda32721674f2ec6625d955c17 Mon Sep 17 00:00:00 2001 From: share121 Date: Mon, 27 Jul 2026 14:01:13 +0800 Subject: [PATCH 13/26] =?UTF-8?q?refactor(fast-down-api):=20=E7=AE=80?= =?UTF-8?q?=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/src/core/state.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs index 0a11af6..9185163 100644 --- a/crates/fast-down-api/src/core/state.rs +++ b/crates/fast-down-api/src/core/state.rs @@ -94,16 +94,7 @@ impl DownloadState { /// server to support range requests. #[must_use] pub fn validate(&self, info: &UrlInfo) -> bool { - if self.size != Some(info.size) { - return false; - } - if !info.fast_download { - return false; - } - if self.file_id() != info.file_id { - return false; - } - true + info.fast_download && self.size == Some(info.size) && self.file_id() == info.file_id } /// Merge a freshly-written byte range into the recorded progress. From f2edb4a04124e223bb7ee0a8b5d822b949d9b520 Mon Sep 17 00:00:00 2001 From: share121 Date: Mon, 27 Jul 2026 14:02:07 +0800 Subject: [PATCH 14/26] =?UTF-8?q?refactor(fast-down-api):=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E5=8F=82=E6=95=B0=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- .../src/core/download/pipeline.rs | 48 ++++++++----------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/crates/fast-down-api/src/core/download/pipeline.rs b/crates/fast-down-api/src/core/download/pipeline.rs index 94ebf62..c750020 100644 --- a/crates/fast-down-api/src/core/download/pipeline.rs +++ b/crates/fast-down-api/src/core/download/pipeline.rs @@ -1,4 +1,4 @@ -use crate::{Config, Event, Tx, WriteMethod, utils::build_header}; +use crate::{Config, Ctx, Event, WriteMethod, utils::build_header}; use fast_down::{ BoxPusher, UrlInfo, fast_puller::{FastDownPuller, FastDownPullerOptions}, @@ -8,54 +8,48 @@ use parking_lot::Mutex; use reqwest::Response; use std::sync::Arc; use tokio::fs::File; -use tokio_util::sync::CancellationToken; use url::Url; -/// Build the puller + pusher inside a `force_send` + `run_until_cancelled` -/// future (not provably `Send`; see `run` doc). Yields `None` on a build error -/// (the error event is already sent through `tx`) or on cancel-before-transfer; -/// the caller is responsible for persisting state and stopping. -pub(super) async fn build_pipeline( +pub async fn build_pipeline( url: &Url, - effective: &Config, + config: &Config, info: &UrlInfo, - file: File, resp: Response, - cancel_token: CancellationToken, - tx: &Tx, + file: File, + ctx: &Ctx, ) -> Option<(FastDownPuller, BoxPusher)> { - let ct = cancel_token; let resp = Some(Arc::new(Mutex::new(Some(resp)))); - let built = ct + let built = ctx + .token .run_until_cancelled(async move { let puller = FastDownPuller::new(FastDownPullerOptions { url: url.clone(), - headers: build_header(&effective.headers).into(), - proxy: effective.proxy.as_deref(), - accept_invalid_certs: effective.accept_invalid_certs, - accept_invalid_hostnames: effective.accept_invalid_hostnames, - cookie_store: effective.cookie_store, + headers: build_header(&config.headers).into(), + proxy: config.proxy.as_deref(), + accept_invalid_certs: config.accept_invalid_certs, + accept_invalid_hostnames: config.accept_invalid_hostnames, + cookie_store: config.cookie_store, file_id: info.file_id.clone(), resp, - available_ips: effective.local_address.clone().into(), - max_redirects: effective.max_redirects, + available_ips: config.local_address.clone().into(), + max_redirects: config.max_redirects, }) .map_err(Event::BuildClientError)?; let pusher = if cfg!(target_pointer_width = "64") && info.fast_download - && effective.write_method == WriteMethod::Mmap + && config.write_method == WriteMethod::Mmap { - MmapFilePusher::new(&file, info.size, effective.sync_all) + MmapFilePusher::new(&file, info.size, config.sync_all) .await .map(BoxPusher::new) } else { CacheFilePusher::new( file, info.size, - effective.sync_all, - effective.cache_high_watermark, - effective.cache_low_watermark, - effective.write_buffer_size, + config.sync_all, + config.cache_high_watermark, + config.cache_low_watermark, + config.write_buffer_size, ) .await .map(BoxPusher::new) @@ -67,7 +61,7 @@ pub(super) async fn build_pipeline( match built { Some(Ok(b)) => Some(b), Some(Err(e)) => { - let _ = tx.send(e); + let _ = ctx.tx.send(e); None } None => None, From 7a7f839fef65a8ce341447f92748f9eb96c543ef Mon Sep 17 00:00:00 2001 From: share121 Date: Mon, 27 Jul 2026 14:43:34 +0800 Subject: [PATCH 15/26] =?UTF-8?q?feat(fast-pull):=20=E5=AE=8C=E5=96=84?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-pull/src/core/mod.rs | 47 +++++++++++++++++++---------- crates/fast-pull/src/core/multi.rs | 4 +-- crates/fast-pull/src/core/single.rs | 6 ++-- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/crates/fast-pull/src/core/mod.rs b/crates/fast-pull/src/core/mod.rs index 291d29d..cc2cbaa 100644 --- a/crates/fast-pull/src/core/mod.rs +++ b/crates/fast-pull/src/core/mod.rs @@ -3,7 +3,6 @@ use core::sync::atomic::{AtomicBool, Ordering}; use crossfire::{MAsyncRx, mpmc}; use fast_steal::{Executor, Handle, TaskQueue}; use std::fmt; -use std::ops::Deref; use std::sync::{Arc, OnceLock, Weak}; use std::thread::Thread; use tokio::task::{AbortHandle, JoinError, JoinHandle}; @@ -21,13 +20,13 @@ pub mod single; /// cancellation happens, giving `DownloadResult` `Arc`-style "last owner gone → /// release" semantics: the download keeps running as long as any handle is /// alive, and is cancelled only when the final one is dropped. -pub struct DownloadResultInner +struct DownloadResultInner where E: Executor + Send + Sync, PullError: Send + Unpin + 'static, PushError: Send + Unpin + 'static, { - pub event_chain: MAsyncRx>>, + event_chain: MAsyncRx>>, handle: SharedHandle<()>, abort_handles: Option>, task_queue: Option<(Weak, TaskQueue)>, @@ -163,19 +162,6 @@ where } } -impl Deref for DownloadResult -where - E: Executor + Send + Sync, - PullError: Send + Unpin + 'static, - PushError: Send + Unpin + 'static, -{ - type Target = DownloadResultInner; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - impl DownloadResult where E: Executor + Send + Sync, @@ -201,4 +187,33 @@ where }), } } + + #[must_use] + pub fn event_chain(&self) -> &MAsyncRx>> { + &self.inner.event_chain + } + + /// # Errors + /// Returns `Arc` if the writer thread exits unexpectedly + pub async fn join(&self) -> Result<(), Arc> { + self.inner.join().await + } + + /// Cancel all workers immediately. + /// + /// Safe to call multiple times and safe to call while other clones of the + /// owning [`DownloadResult`] are still alive. The implicit drop-based + /// cancellation (on the last clone) becomes a no-op once this has run. + pub fn abort(&self) { + self.inner.abort(); + } + + pub fn set_threads(&self, threads: usize, min_chunk_size: u64) { + self.inner.set_threads(threads, min_chunk_size); + } + + #[must_use] + pub fn is_aborted(&self) -> bool { + self.inner.is_aborted() + } } diff --git a/crates/fast-pull/src/core/multi.rs b/crates/fast-pull/src/core/multi.rs index 4d77d19..2510017 100644 --- a/crates/fast-pull/src/core/multi.rs +++ b/crates/fast-pull/src/core/multi.rs @@ -302,7 +302,7 @@ mod tests { let mut pull_progress: Vec = Vec::new(); let mut push_progress: Vec = Vec::new(); let mut pull_ids = [false; 32]; - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { match e { Event::PullProgress(id, p) => { pull_ids[id] = true; @@ -447,7 +447,7 @@ mod tests { // Abort as soon as the push driver starts processing (first `Pushing`). let mut aborted = false; - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { if matches!(e, Event::Pushing(_, _)) { result.abort(); assert!(result.is_aborted()); diff --git a/crates/fast-pull/src/core/single.rs b/crates/fast-pull/src/core/single.rs index de27264..da3b16b 100644 --- a/crates/fast-pull/src/core/single.rs +++ b/crates/fast-pull/src/core/single.rs @@ -155,7 +155,7 @@ mod tests { let mut pull_progress: Vec = Vec::new(); let mut push_progress: Vec = Vec::new(); - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { match e { Event::PullProgress(_, p) => pull_progress.merge_progress(p), Event::PushProgress(p) => push_progress.merge_progress(p), @@ -284,7 +284,7 @@ mod tests { // event). This lands reliably before completion because the puller is // slow. let mut aborted = false; - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { if matches!(e, Event::Pushing(_, _)) { result.abort(); assert!(result.is_aborted()); @@ -339,7 +339,7 @@ mod tests { ); let mut aborted = false; - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { if matches!(e, Event::Pushing(_, _)) { result.abort(); assert!(result.is_aborted()); From 6e76b2e5726425b226121db2b4a20593435cf758 Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 00:22:58 +0800 Subject: [PATCH 16/26] =?UTF-8?q?fix:(fast-down):=20=E4=BF=AE=E5=A4=8D=20u?= =?UTF-8?q?rl=20=E5=81=9A=E6=96=87=E4=BB=B6=E5=90=8D=E6=97=B6=EF=BC=8C?= =?UTF-8?q?=E8=AF=AF=E6=8A=8A=20.com=20=E5=BD=93=E6=88=90=E6=89=A9?= =?UTF-8?q?=E5=B1=95=E5=90=8D=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down/src/http/prefetch.rs | 4 ++-- crates/fast-down/src/reqwest/mod.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/fast-down/src/http/prefetch.rs b/crates/fast-down/src/http/prefetch.rs index c83e1e4..6eac3f5 100644 --- a/crates/fast-down/src/http/prefetch.rs +++ b/crates/fast-down/src/http/prefetch.rs @@ -50,8 +50,8 @@ fn get_filename(headers: &impl HttpHeaders, url: &Url) -> String { }) .filter(|s| !s.trim().is_empty()) }) - .or_else(|| url.host_str().map(ToString::to_string)) - .unwrap_or_else(|| url.to_string()) + .or_else(|| url.host_str().map(|s| s.replace('.', "_"))) + .unwrap_or_else(|| url.to_string().replace('.', "_")) } async fn prefetch(client: &Client, url: Url) -> PrefetchResult { diff --git a/crates/fast-down/src/reqwest/mod.rs b/crates/fast-down/src/reqwest/mod.rs index 009bec8..4bbcf8c 100644 --- a/crates/fast-down/src/reqwest/mod.rs +++ b/crates/fast-down/src/reqwest/mod.rs @@ -486,7 +486,7 @@ mod tests { // `PushProgress` now flows on the same `event_chain` as the engine events // (the sink's listener emits it the moment data is actually written), so a // single drain collects both pull and push progress. - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { match e { Event::PullProgress(_, p) => pull_progress.merge_progress(p), Event::PushProgress(p) => push_progress.merge_progress(p), @@ -541,7 +541,7 @@ mod tests { // `PushProgress` now flows on the same `event_chain` as the engine events // (the sink's listener emits it the moment data is actually written), so a // single drain collects both pull and push progress. - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { match e { Event::PullProgress(_, p) => pull_progress.merge_progress(p), Event::PushProgress(p) => push_progress.merge_progress(p), From a9da073ec71c8419b5dad086318a256eb4118111 Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 14:09:52 +0800 Subject: [PATCH 17/26] =?UTF-8?q?refactor(fast-down-api):=20=E5=85=A8?= =?UTF-8?q?=E9=9D=A2=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/src/config.rs | 24 +- .../src/core/download/acquire.rs | 179 ------- .../src/core/download/finalize.rs | 37 -- crates/fast-down-api/src/core/download/mod.rs | 480 +++++++----------- .../src/core/download/overwrite.rs | 166 ++++++ .../src/core/download/pipeline.rs | 21 +- crates/fast-down-api/src/core/prefetch.rs | 21 +- crates/fast-down-api/src/core/state.rs | 105 +++- crates/fast-down-api/src/event.rs | 31 +- crates/fast-down-api/src/utils/tx_err.rs | 9 + .../{tests => tests.bak}/resume.rs | 0 11 files changed, 473 insertions(+), 600 deletions(-) delete mode 100644 crates/fast-down-api/src/core/download/acquire.rs delete mode 100644 crates/fast-down-api/src/core/download/finalize.rs create mode 100644 crates/fast-down-api/src/core/download/overwrite.rs rename crates/fast-down-api/{tests => tests.bak}/resume.rs (100%) diff --git a/crates/fast-down-api/src/config.rs b/crates/fast-down-api/src/config.rs index 43d6ce8..eab6183 100644 --- a/crates/fast-down-api/src/config.rs +++ b/crates/fast-down-api/src/config.rs @@ -1,5 +1,5 @@ use fast_down::{Merge, ProgressEntry, Proxy}; -use inherit_config::{ConfigLayer, InheritConfig}; +use inherit_config::InheritConfig; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, net::IpAddr, path::PathBuf, time::Duration}; @@ -170,23 +170,9 @@ pub struct Config { } impl PartialConfig { - /// Build the effective [`Config`], folding previously-saved resume progress - /// into `downloaded_chunk` so the engine only fetches the remaining bytes. - /// - /// This is the *single* place where resume progress enters the resolved - /// config. Callers must go through it instead of `build()` followed by a - /// separate `merge_progress`, so `effective` stays a pure function of the - /// partial config + the saved progress — no post-`build` mutation, and the - /// config layer (`self`) and the resolved `Config` can never drift on - /// progress because the layer simply does not carry it. - #[must_use] - pub fn build_seeded(&self, progress: Option<&[ProgressEntry]>) -> Config { - let mut c = self.clone().build(); - if let Some(p) = progress { - for e in p { - c.downloaded_chunk.merge_progress(e.clone()); - } - } - c + pub fn merge_progress(&mut self, progress: ProgressEntry) { + self.downloaded_chunk + .get_or_insert_default() + .merge_progress(progress); } } diff --git a/crates/fast-down-api/src/core/download/acquire.rs b/crates/fast-down-api/src/core/download/acquire.rs deleted file mode 100644 index df45f74..0000000 --- a/crates/fast-down-api/src/core/download/acquire.rs +++ /dev/null @@ -1,179 +0,0 @@ -use crate::{Config, DownloadState, Event, PartialConfig, ResumeError, Tx}; -use fast_down::UrlInfo; -use inherit_config::ConfigLayer; -use std::path::Path; -use tokio::fs::{self, File, OpenOptions}; - -/// Outcome of trying to acquire a writable `.part` file + the matching resume state. -#[allow(clippy::large_enum_variant)] -pub(super) enum Acquire { - /// A file is ready to write, optionally carrying a previously-saved resume state. - /// - /// The large fields are boxed so the variant stays small (`large_enum_variant`). - Ready { - file: File, - effective: Config, - parsed: PartialConfig, - resume_state: Option, - }, - /// Unique-name collision: the caller should regenerate the stem and retry. - CollisionRetry, - /// An unrecoverable error was already reported via `tx`; the caller should stop. - Abort, -} - -/// Outcome of probing an existing `.part`/`.fd` pair for resume eligibility. -/// -/// The classification is pure (see [`classify_resume`]); this enum only names -/// the three possible results so the resume *contract* lives in one place. -#[allow(clippy::large_enum_variant)] -enum ResumeProbe { - /// File + state both present and the state still matches the remote file. - Valid { file: File, state: DownloadState }, - /// An explicit resume was requested but the pair is unusable (stale or - /// missing); the caller must report and stop. - GiveUp(ResumeError), - /// Stale or missing state on a plain download; the caller drops the partial - /// files and opens fresh. - Discard, -} - -/// Borrow the shared `OpenOptions` presets used to open the `.part` file. -fn open_existing() -> OpenOptions { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create(false); - o -} -fn open_create() -> OpenOptions { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create(true); - o -} -fn open_create_new() -> OpenOptions { - let mut o = OpenOptions::new(); - o.read(true).write(true).truncate(false).create_new(true); - o -} - -/// Classify the result of probing an existing `.part`/`.fd` pair for resume. -/// -/// Pure: no I/O, no event emission. The two probe results (`open` the `.part`, -/// `load` the `.fd`) map onto one of three outcomes so the resume *contract* -/// lives in a single, unit-testable place. The outcomes are -/// [`ResumeProbe::Valid`] (file + state present and still match the remote -/// file; resume from it), [`ResumeProbe::GiveUp`] (an explicit resume was -/// requested but the pair is unusable; report and stop), and -/// [`ResumeProbe::Discard`] (stale or missing state on a plain download; drop -/// the partial files and open fresh). -fn classify_resume( - open_res: std::io::Result, - load_res: Result, - explicit_resume: bool, - info: &UrlInfo, -) -> ResumeProbe { - match (open_res, load_res) { - (Ok(file), Ok(state)) if state.validate(info) => ResumeProbe::Valid { file, state }, - // Only an *explicit* resume (`resume()`) treats a stale/missing state as - // a hard error (the resume contract). A plain `download()` that happens - // to find a leftover `.part`/`.fd` discards them and re-downloads - // silently — so `explicit_resume` (true only for `resume()`) gates the - // `GiveUp` arms and everything else falls through to `Discard`. - (Ok(_), Ok(state)) if explicit_resume => ResumeProbe::GiveUp(ResumeError::FileChanged { - local_file_id: state.file_id(), - local_file_size: state.size.unwrap_or(0), - remote_file_id: info.file_id.clone(), - remote_file_size: info.size, - }), - (Ok(_), Err(_)) if explicit_resume => ResumeProbe::GiveUp(ResumeError::NoStateFile), - _ => ResumeProbe::Discard, - } -} - -/// Try to resume from an existing `.part`/`.fd` pair, or open a fresh `.part` -/// file. Returns: -/// -/// - [`Acquire::Ready`] with a writable `file` + the effective config + the -/// resume state (if any); -/// - [`Acquire::CollisionRetry`] in unique mode when `create_new` failed -/// (treats the failure as a name collision and asks the caller to retry); -/// - [`Acquire::Abort`] when an unrecoverable error has already been reported -/// through `tx` (e.g. a `resume()` contract violation, or a non-unique open -/// failure), and the caller should stop. -#[allow(clippy::too_many_arguments)] -pub(super) async fn try_acquire_target( - tx: &Tx, - can_resume: bool, - explicit_resume: bool, - unique: bool, - info: &UrlInfo, - partial_config: &PartialConfig, - tmp: &Path, - cfg: &Path, -) -> Acquire { - // ---- 1. Try to resume from an existing `.part`/`.fd` pair ---- - if can_resume { - let opener = open_existing(); - let (open_res, load_res) = tokio::join!(opener.open(tmp), DownloadState::load(cfg)); - match classify_resume(open_res, load_res, explicit_resume, info) { - ResumeProbe::Valid { file, state } => { - let mut pc = partial_config.clone(); - if let Some(c) = &state.config { - pc.inherit_from(c); - } - // Reconstruct `effective` from the resume state in one step: the - // `config` half comes from `inherit_from(state.config)` above, and - // the saved-progress half is folded into `downloaded_chunk` inside - // `build_seeded`. So `effective` is a pure function of `pc` + the - // saved progress (no post-build mutation), and `parsed`/`effective` - // stay in sync by construction — `pc` carries no progress at all. - let effective = pc.build_seeded(state.progress.as_deref()); - let parsed = pc; - return Acquire::Ready { - file, - effective, - parsed, - resume_state: Some(state), - }; - } - ResumeProbe::GiveUp(err) => { - // Explicit resume target but the `.part`/`.fd` pair is unusable - // (stale or missing): report and stop rather than silently - // re-downloading (resume contract). - let _ = tx.send(Event::ResumeError(err)); - return Acquire::Abort; - } - ResumeProbe::Discard => { - // Stale or missing state on a plain download: drop the partial - // files and fall through to a fresh open below. - let _ = fs::remove_file(tmp).await; - let _ = fs::remove_file(cfg).await; - } - } - } - - // ---- 2. Fresh open (also the fall-through after discarding a stale state) ---- - let f = if unique { - open_create_new().open(tmp).await.ok() - } else { - match open_create().open(tmp).await { - Ok(f) => Some(f), - Err(e) => { - let _ = tx.send(Event::BuildPusherError(e)); - return Acquire::Abort; - } - } - }; - f.map_or_else( - || Acquire::CollisionRetry, - |file| { - let pc = partial_config.clone(); - let effective = pc.build_seeded(None); - Acquire::Ready { - file, - effective, - parsed: pc, - resume_state: None, - } - }, - ) -} diff --git a/crates/fast-down-api/src/core/download/finalize.rs b/crates/fast-down-api/src/core/download/finalize.rs deleted file mode 100644 index 4b7a4f8..0000000 --- a/crates/fast-down-api/src/core/download/finalize.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::{Event, Tx}; -use path_helper::tokio::gen_unique_path; -use std::path::Path; -use tokio::fs; - -/// Rename the finished `.part` into place and drop the `.fd` state file. -/// -/// In unique mode, a fresh unique destination is reserved right before rename -/// via `gen_unique_path` (atomic `create_new`), closing the TOCTOU gap where -/// the final file could have been created by someone else during the download. -/// On any failure the relevant error event is sent through `tx`. -pub(super) async fn finalize(tx: &Tx, unique: bool, tmp: &Path, cfg: &Path, final_path: &Path) { - let dest = if unique { - match gen_unique_path(final_path).await { - Ok(p) => p, - Err(e) => { - let _ = tx.send(Event::GenPathError(e)); - return; - } - } - } else { - final_path.to_path_buf() - }; - if let Err(e) = fs::rename(tmp, &dest).await { - // Best-effort: drop the empty placeholder we just reserved so a failed - // rename doesn't leave an orphan `xxx (1).mp4` behind. - if unique { - let _ = fs::remove_file(&dest).await; - } - let _ = tx.send(Event::RenameFailed(e)); - return; - } - let _ = tx.send(Event::Renamed(dest)); - // Success: the download is complete and renamed, so the state file is no - // longer needed. Best-effort cleanup only (kept on cancel). - let _ = fs::remove_file(cfg).await; -} diff --git a/crates/fast-down-api/src/core/download/mod.rs b/crates/fast-down-api/src/core/download/mod.rs index b9dae02..5437651 100644 --- a/crates/fast-down-api/src/core/download/mod.rs +++ b/crates/fast-down-api/src/core/download/mod.rs @@ -1,349 +1,227 @@ -use crate::{ - Config, DownloadState, Event, PartialConfig, ResumeError, Tx, prefetch, tx_err, - utils::{ForceSendExt, build_header, gen_path}, -}; -use fast_down::{ - Total, fast_puller::build_client, handle::SharedHandle, invert, multi::download_multi, - reqwest::SmartRedirectClient, single::download_single, -}; +use crate::core::download::overwrite::OverwriteOption; +use crate::utils::ForceSendExt; +use crate::{DownloadState, Event, StateError}; +use crate::{PartialConfig, Tx, prefetch, tx_err, utils::gen_path}; +use fast_down::handle::SharedHandle; use inherit_config::ConfigLayer; +use overwrite::overwrite; use path_helper::IterStemExt; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Arc; -use tokio::fs; +use tokio::fs::{self, OpenOptions}; use tokio::task::JoinError; use tokio_util::sync::CancellationToken; use url::Url; -/// Persist the download state after at least this many freshly-written bytes. -const STATE_STORE_BYTES: u64 = 16 * 1024 * 1024; -/// Persist the download state after at least this many `PushProgress` events. -const STATE_STORE_EVENTS: usize = 512; - -mod acquire; -mod finalize; +mod overwrite; mod pipeline; -use acquire::{Acquire, try_acquire_target}; -use finalize::finalize; -use pipeline::build_pipeline; +fn open_existing() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(false); + o +} +fn open_create() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(true); + o +} +fn open_create_new() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create_new(true); + o +} +#[derive(Debug, Clone)] pub struct DownloadHandle { handle: SharedHandle<()>, } impl DownloadHandle { - /// Wait for the download to complete. - /// - /// # Panics - /// Panics if the background download task exits unexpectedly. + /// Returns the join of this [`DownloadHandle`]. /// /// # Errors - /// Returns `Arc` if the download task itself panics or is - /// cancelled. + /// + /// This function will return an error if download thread panic pub async fn join(&self) -> Result<(), Arc> { self.handle.join().await } - /// # Errors + #[must_use] pub fn download( url: Url, partial_config: PartialConfig, tx: Tx, - cancel_token: CancellationToken, - ) -> anyhow::Result { - Self::spawn(url, partial_config, tx, cancel_token, None) + token: CancellationToken, + ) -> Self { + let handle = tokio::spawn( + async move { + let token2 = token.clone(); + let opt = token + .run_until_cancelled( + async move { download(url, partial_config, tx, token2).await }, + ) + .await + .flatten(); + if let Some(opt) = opt { + overwrite(opt).await; + } + } + .force_send(), + ); + Self { + handle: SharedHandle::new(handle), + } } - /// Explicitly resume an interrupted download from the given temporary file. - /// - /// `tmp_path` is the `.part` file left behind by a previous (cancelled or - /// crashed) download. - /// - /// - When `tmp_path` **exists**, the download continues from it - /// (`force_resume = true`, and the `resume` config flag is forced on). - /// - When `tmp_path` does **not** exist, the call falls back to a fresh - /// [`DownloadHandle::download`] (`force_resume = false`) and starts writing - /// at `tmp_path` from scratch. - /// - /// Note: even when `tmp_path` exists, if the `.fd` state file is missing or - /// the remote file changed, an [`Event::ResumeError`] is still emitted — - /// this is the `resume` contract (unlike `download`, it never silently - /// falls back to a full re-download). - /// - /// # Errors pub fn resume( tmp_path: impl AsRef, url: Url, partial_config: PartialConfig, tx: Tx, - cancel_token: CancellationToken, - ) -> anyhow::Result { - // Hand the path down verbatim; `run` probes its existence asynchronously - // via `tokio::fs::metadata` (never a synchronous `Path::exists()`, which - // would block the calling executor thread on a slow/network mount). - Self::spawn( - url, - partial_config, - tx, - cancel_token, - Some(tmp_path.as_ref().to_path_buf()), - ) - } - - /// Shared entry point for [`DownloadHandle::download`] and - /// [`DownloadHandle::resume`]. Builds the HTTP client (errors here propagate - /// to the caller via `?`), then spawns the background task that runs the - /// entire download pipeline in [`Self::run`]. - /// - /// # Errors - fn spawn( - url: Url, - partial_config: PartialConfig, - tx: Tx, - cancel_token: CancellationToken, - tmp_path: Option, - ) -> anyhow::Result { - let config = partial_config.clone().build(); - let client = build_client( - build_header(&config.headers), - config.proxy.as_deref(), - config.accept_invalid_certs, - config.accept_invalid_hostnames, - config.cookie_store, - config.local_address.first().copied(), - config.max_redirects, - )?; + token: CancellationToken, + ) -> Self { + let tmp_path = tmp_path.as_ref().to_path_buf(); let handle = tokio::spawn( - Self::run( - url, - config, - partial_config, - client, - tx, - cancel_token, - tmp_path, - ) + async move { + let token2 = token.clone(); + let opt = Box::pin(token.run_until_cancelled(async move { + resume(&tmp_path, url, partial_config, tx, token2).await + })) + .await + .flatten(); + if let Some(opt) = opt { + overwrite(opt).await; + } + } .force_send(), ); - let handle = SharedHandle::new(handle); - Ok(Self { handle }) + Self { + handle: SharedHandle::new(handle), + } } +} - /// The entire download pipeline, consolidated into a single function: - /// - /// 1. prefetch + resolve the destination path, - /// 2. per-iteration: try to resume from an existing `.part`/`.fd` pair (or - /// start fresh), then build the puller/pusher and run the transfer - /// (multi or single), - /// 3. forward engine events, persist the resume state with debounce, - /// 4. on cancel keep `.part`+`.fd`; otherwise rename into place and drop - /// the state file. - /// - /// In `overwrite` mode (`unique = false`) a single create attempt is made; - /// in `unique` mode (`unique = true`) a fresh `create_new` is retried with a - /// regenerated stem (`xxx (1).mp4`, …) until a free path is found. - /// - /// The whole `run` future is wrapped in `force_send` at the `spawn` site - /// because the futures it drives are not provably `Send`: they move the - /// file handle / response into the engine builders, and the engine's - /// event-channel receiver is `!Send`. The future still runs on this task, - /// so the `unsafe impl Send` assertions are sound. The transfer event loop - /// is kept *outside* `run_until_cancelled` (only the puller/pusher build - /// uses it) so a mid-download cancel is handled by the loop's final state - /// store rather than being silently dropped. - #[allow(clippy::too_many_lines)] - async fn run( - url: Url, - mut config: Config, - partial_config: PartialConfig, - client: SmartRedirectClient, - tx: Tx, - cancel_token: CancellationToken, - tmp_path: Option, - ) { - let Some((info, resp)) = prefetch(&url, &config, &client, &tx).await else { - return; - }; - let tmp_path = if let Some(path) = tmp_path - && fs::try_exists(&path).await.unwrap_or(false) +async fn download( + url: Url, + partial_config: PartialConfig, + tx: Tx, + token: CancellationToken, +) -> Option { + let config = partial_config.clone().build(); + let (info, resp) = prefetch(&url, &config, &tx).await?; + let can_resume = config.resume && info.fast_download; + + let origin_path = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError, None); + + if config.overwrite { + let cfg_path = origin_path.with_added_extension("fd"); + let tmp_path = origin_path.with_added_extension("part"); + + let state = if can_resume + && let Ok(mut s) = DownloadState::load(&cfg_path).await + && s.validate(&info).is_ok() + && fs::try_exists(&tmp_path).await.unwrap_or(false) { - config.resume = true; - Some(path) - } else { - None - }; - - // Resolve the destination path + whether we run in unique-name mode. - let (origin_final, unique) = if let Some(path) = &tmp_path { - (path.with_extension(""), false) + s.merge_config(&partial_config); + let _ = tx.send(Event::Resumed { + config_path: cfg_path, + progress: s.get_progress(), + size: info.size, + }); + s } else { - let p = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError); - (p, !config.overwrite) + tx_err!( + open_create().open(tmp_path).await, + tx, + BuildPusherError, + None + ); + DownloadState::new(&url, &info, &partial_config, &cfg_path) }; + return Some(OverwriteOption { + state, + final_path: origin_path, + info, + resp, + tx, + token, + }); + } - if tmp_path.is_some() && !info.fast_download { - let _ = tx.send(Event::ResumeError(ResumeError::NotResumable)); - return; - } - let can_resume = config.resume && info.fast_download; - // `explicit_resume` is true only when this is an explicit `resume()` call - // (a `tmp_path` was supplied and the `.part` exists); a plain `download()` - // is `false` and must silently fall back instead of erroring. - let explicit_resume = tmp_path.is_some(); - - // `resp` is consumed exactly once, by `build_pipeline` on the iteration - // that actually starts a transfer (collision retries `continue` before it). - for final_path in origin_final.iter_stem() { - let tmp = final_path.with_added_extension("part"); - let cfg = final_path.with_added_extension("fd"); - - // ---- 1. Resume from / open the `.part` file (handles unique collisions) ---- - let acquired = - try_acquire_target(&tx, can_resume, explicit_resume, unique, &info, &partial_config, &tmp, &cfg) - .await; - let (file, effective, parsed, resume_state) = match acquired { - Acquire::CollisionRetry => continue, - Acquire::Abort => return, - Acquire::Ready { - file, - effective, - parsed, - resume_state, - } => (file, effective, parsed, resume_state), - }; - - // ===== attempt: build state, emit events, transfer, persist, rename ===== - let is_resume = resume_state.is_some(); - let mut state = - resume_state.unwrap_or_else(|| DownloadState::new(&url, &info, &parsed, &cfg)); + for base_path in origin_path.iter_stem() { + let tmp_path = base_path.with_added_extension("part"); + let cfg_path = base_path.with_added_extension("fd"); - // Announce a resumed download. The progress ranges were already folded - // into `effective.downloaded_chunk` inside `try_acquire_target`, so here - // we only notify the consumer. - if is_resume { - let _ = tx.send(Event::Resumed { - config_path: cfg.clone(), - progress: state.progress.clone().unwrap_or_default(), - size: info.size, - }); - } - let _ = tx.send(Event::Start { - tmp_path: tmp.clone(), - config_path: cfg.clone(), - parsed_config: parsed, + let state = if can_resume + && let Ok(mut s) = DownloadState::load(&cfg_path).await + && s.validate(&info).is_ok() + && fs::try_exists(&tmp_path).await.unwrap_or(false) + { + s.merge_config(&partial_config); + let _ = tx.send(Event::Resumed { + config_path: cfg_path, + progress: s.get_progress(), + size: info.size, }); + s + } else if open_create_new().open(&tmp_path).await.is_ok() { + DownloadState::new(&url, &info, &partial_config, &cfg_path) + } else { + continue; + }; + return Some(OverwriteOption { + state, + final_path: origin_path, + info, + resp, + tx, + token, + }); + } + unreachable!() +} - // ---- 3a. Build the puller + pusher (force_send; not provably Send) ---- - // The build itself is fast and non-blocking, so a cancel landing here - // is rare and simply ends the task without a transfer. - let Some((puller, pusher)) = build_pipeline( - &url, - &effective, - &info, - file, - resp, - cancel_token.clone(), - &tx, - ) - .await - else { - // Build failed (errored or cancelled before transfer): persist the - // (possibly empty) state so a later resume still finds a coherent .fd. - let _ = state.store().await; - return; - }; - - // ---- 3b. Run the transfer + forward engine events with debounced store ---- - // Multi-threaded when the server supports range requests, otherwise a - // single sequential stream. Kept outside `run_until_cancelled` (see - // `run` doc) so a mid-download cancel is handled by this loop's final - // state store rather than being silently dropped. - let res = if info.fast_download { - download_multi( - puller, - pusher, - fast_down::multi::DownloadOptions { - download_chunks: invert( - effective.downloaded_chunk.into_iter(), - info.size, - effective.chunk_window, - ), - concurrent: effective.threads, - retry_gap: effective.retry_gap, - pull_timeout: effective.pull_timeout, - push_queue_cap: effective.write_queue_cap, - min_chunk_size: effective.min_chunk_size, - max_speculative: effective.max_speculative, - }, - ) - } else { - download_single( - puller, - pusher, - fast_down::single::DownloadOptions { - retry_gap: effective.retry_gap, - push_queue_cap: effective.write_queue_cap, - }, - ) - }; - - // All events (including `PushProgress`) flow through the single - // `event_chain`. `PushProgress` also updates `state` with debounced - // store so the `.fd` always reflects the truth. - let mut store_events: usize = 0; - let mut store_bytes: u64 = 0; - while let Ok(e) = res.event_chain.recv().await { - if let fast_down::Event::PushProgress(range) = &e { - store_events += 1; - store_bytes += range.total(); - state.merge_progress(range.clone()); - } - let _ = match e { - fast_down::Event::Pulling(id) => tx.send(Event::Pulling(id)), - fast_down::Event::PullError(id, e) => { - tx.send(Event::PullError(id, anyhow::anyhow!(e))) - } - fast_down::Event::PullTimeout(id) => tx.send(Event::PullTimeout(id)), - fast_down::Event::PullProgress(id, range) => { - tx.send(Event::PullProgress(id, range)) - } - fast_down::Event::Pushing(id, range) => tx.send(Event::Pushing(id, range)), - fast_down::Event::PushError(id, range, e) => { - tx.send(Event::PushError(id, range, anyhow::anyhow!(e))) - } - fast_down::Event::PushProgress(range) => tx.send(Event::PushProgress(range)), - fast_down::Event::Flushing => tx.send(Event::Flushing), - fast_down::Event::FlushError(e) => { - tx.send(Event::FlushError(anyhow::anyhow!(e))) - } - fast_down::Event::Finished(id) => tx.send(Event::Finished(id)), - }; - if store_events >= STATE_STORE_EVENTS || store_bytes >= STATE_STORE_BYTES { - let _ = state.store().await; - store_events = 0; - store_bytes = 0; - } - } - - if let Err(e) = res.join().await { - let _ = tx.send(Event::JoinError(e)); - } +async fn resume( + tmp_path: &Path, + url: Url, + mut partial_config: PartialConfig, + tx: Tx, + token: CancellationToken, +) -> Option { + partial_config.overwrite = Some(false); + let tmp_exits = fs::try_exists(tmp_path).await.unwrap_or(false); + if !tmp_exits { + partial_config.resume = Some(false); + return download(url, partial_config, tx, token).await; + } - // The `.fd` state file captures the current download progress (url, - // config, size, etag) so a later resume can validate the remote file - // identity. Called unconditionally even when cancelled before the - // transfer finished. - let _ = state.store().await; + let cfg_path = tmp_path.with_extension("fd"); + let mut state = tx_err!(DownloadState::load(&cfg_path).await, tx, ResumeError, None); - // If the download was cancelled, do NOT rename the `.part` and do NOT - // remove the `.fd` state file: keep both so a later `download()`/ - // `resume()` can continue from where it left off (design doc §8). - if cancel_token.is_cancelled() { - return; - } - finalize(&tx, unique, &tmp, &cfg, &final_path).await; - return; - } + partial_config.resume = Some(true); + let config = partial_config.clone().build(); + let (info, resp) = prefetch(&url, &config, &tx).await?; + if !info.fast_download { + let _ = tx.send(Event::ResumeError(StateError::NotResumable(info, resp))); + return None; } + tx_err!(state.validate(&info), tx, ResumeError, None); + + state.merge_config(&partial_config); + let _ = tx.send(Event::Resumed { + config_path: cfg_path, + progress: state.get_progress(), + size: info.size, + }); + + let final_path = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError, None); + Some(OverwriteOption { + state, + final_path, + info, + resp, + tx, + token, + }) } diff --git a/crates/fast-down-api/src/core/download/overwrite.rs b/crates/fast-down-api/src/core/download/overwrite.rs new file mode 100644 index 0000000..a0f95f7 --- /dev/null +++ b/crates/fast-down-api/src/core/download/overwrite.rs @@ -0,0 +1,166 @@ +use crate::{ + DownloadState, Event, PartialConfig, Tx, core::download::pipeline::build_pipeline, tx_err, +}; +use fast_down::{ + AnyError, DownloadResult, UrlInfo, + fast_puller::FastDownPuller, + http::HttpError, + invert, + multi::{TokioExecutor, download_multi}, + reqwest::SmartRedirectClient, + single::download_single, +}; +use inherit_config::ConfigLayer; +use path_helper::tokio::gen_unique_path; +use reqwest::Response; +use std::{ + path::PathBuf, + time::{Duration, Instant}, +}; +use tokio::fs; +use tokio_util::sync::CancellationToken; + +pub struct OverwriteOption { + pub state: DownloadState, + pub final_path: PathBuf, + pub info: UrlInfo, + pub resp: Response, + pub tx: Tx, + pub token: CancellationToken, +} + +#[allow(clippy::too_many_lines)] +pub async fn overwrite(option: OverwriteOption) { + let OverwriteOption { + mut state, + final_path, + info, + resp, + tx, + token, + } = option; + tx_err!(state.store().await, tx, StateSaveError); + + let inner = state.clone().build(); + let tmp_path = state.tmp_path(); + let url = &inner.url; + let config = &inner.config; + + let pipeline = build_pipeline(url, config, &info, resp, &tmp_path, &tx, &token).await; + let Some((puller, pusher)) = pipeline else { + return; + }; + + let _ = tx.send(Event::Start { + tmp_path: tmp_path.clone(), + config_path: state.config_path.clone(), + parsed_config: state.config.clone().unwrap_or_default(), + }); + + let res = if info.fast_download { + download_multi( + puller, + pusher, + fast_down::multi::DownloadOptions { + download_chunks: invert( + config.downloaded_chunk.iter().cloned(), + info.size, + config.chunk_window, + ), + concurrent: config.threads, + retry_gap: config.retry_gap, + pull_timeout: config.pull_timeout, + push_queue_cap: config.write_queue_cap, + min_chunk_size: config.min_chunk_size, + max_speculative: config.max_speculative, + }, + ) + } else { + download_single( + puller, + pusher, + fast_down::single::DownloadOptions { + retry_gap: config.retry_gap, + push_queue_cap: config.write_queue_cap, + }, + ) + }; + let _guard = abort_ctrl(&token, &res); + + let mut store_time = Instant::now(); + while let Ok(e) = res.event_chain().recv().await { + if let fast_down::Event::PushProgress(range) = &e { + state.merge_progress(range.clone()); + } + let _ = match e { + fast_down::Event::Pulling(id) => tx.send(Event::Pulling(id)), + fast_down::Event::PullError(id, e) => tx.send(Event::PullError(id, anyhow::anyhow!(e))), + fast_down::Event::PullTimeout(id) => tx.send(Event::PullTimeout(id)), + fast_down::Event::PullProgress(id, range) => tx.send(Event::PullProgress(id, range)), + fast_down::Event::Pushing(id, range) => tx.send(Event::Pushing(id, range)), + fast_down::Event::PushError(id, range, e) => { + tx.send(Event::PushError(id, range, anyhow::anyhow!(e))) + } + fast_down::Event::PushProgress(range) => tx.send(Event::PushProgress(range)), + fast_down::Event::Flushing => tx.send(Event::Flushing), + fast_down::Event::FlushError(e) => tx.send(Event::FlushError(anyhow::anyhow!(e))), + fast_down::Event::Finished(id) => tx.send(Event::Finished(id)), + }; + if store_time.elapsed() > Duration::from_secs(1) { + if let Err(e) = state.store().await { + let _ = tx.send(Event::StateSaveError(e)); + } + store_time = Instant::now(); + } + } + + if let Err(e) = res.join().await { + let _ = tx.send(Event::JoinError(e)); + } + let download_complete = info.size == 0 + || matches!(&state.config, Some(PartialConfig { downloaded_chunk: Some(x), .. }) if x.len() == 1 && x[0] == (0..info.size)); + if token.is_cancelled() || !download_complete { + if let Err(e) = state.store().await { + let _ = tx.send(Event::StateSaveError(e)); + } + return; + } + + let final_path = if config.overwrite { + final_path + } else { + tx_err!(gen_unique_path(final_path).await, tx, GenPathError) + }; + if let Err(e) = fs::rename(tmp_path, &final_path).await { + if !config.overwrite { + let _ = fs::remove_file(&final_path).await; + } + let _ = tx.send(Event::RenameFailed(e)); + return; + } + let _ = fs::remove_file(&state.config_path).await; + let _ = tx.send(Event::Renamed(final_path)); +} + +struct Guard(tokio::task::JoinHandle<()>); +impl Drop for Guard { + fn drop(&mut self) { + self.0.abort(); + } +} + +fn abort_ctrl(token: &CancellationToken, res: &MyDownloadResult) -> Guard { + let token = token.clone(); + let res = res.clone(); + let handle = tokio::spawn(async move { + token.cancelled().await; + res.abort(); + }); + Guard(handle) +} + +type MyDownloadResult = DownloadResult< + TokioExecutor>, + HttpError, + Box, +>; diff --git a/crates/fast-down-api/src/core/download/pipeline.rs b/crates/fast-down-api/src/core/download/pipeline.rs index c750020..94a2197 100644 --- a/crates/fast-down-api/src/core/download/pipeline.rs +++ b/crates/fast-down-api/src/core/download/pipeline.rs @@ -1,4 +1,4 @@ -use crate::{Config, Ctx, Event, WriteMethod, utils::build_header}; +use crate::{Config, Event, Tx, WriteMethod, core::download::open_existing, utils::build_header}; use fast_down::{ BoxPusher, UrlInfo, fast_puller::{FastDownPuller, FastDownPullerOptions}, @@ -6,8 +6,8 @@ use fast_down::{ }; use parking_lot::Mutex; use reqwest::Response; -use std::sync::Arc; -use tokio::fs::File; +use std::{path::Path, sync::Arc}; +use tokio_util::sync::CancellationToken; use url::Url; pub async fn build_pipeline( @@ -15,12 +15,12 @@ pub async fn build_pipeline( config: &Config, info: &UrlInfo, resp: Response, - file: File, - ctx: &Ctx, + path: &Path, + tx: &Tx, + token: &CancellationToken, ) -> Option<(FastDownPuller, BoxPusher)> { let resp = Some(Arc::new(Mutex::new(Some(resp)))); - let built = ctx - .token + let built = token .run_until_cancelled(async move { let puller = FastDownPuller::new(FastDownPullerOptions { url: url.clone(), @@ -35,6 +35,11 @@ pub async fn build_pipeline( max_redirects: config.max_redirects, }) .map_err(Event::BuildClientError)?; + + let file = open_existing() + .open(path) + .await + .map_err(Event::BuildPusherError)?; let pusher = if cfg!(target_pointer_width = "64") && info.fast_download && config.write_method == WriteMethod::Mmap @@ -61,7 +66,7 @@ pub async fn build_pipeline( match built { Some(Ok(b)) => Some(b), Some(Err(e)) => { - let _ = ctx.tx.send(e); + let _ = tx.send(e); None } None => None, diff --git a/crates/fast-down-api/src/core/prefetch.rs b/crates/fast-down-api/src/core/prefetch.rs index 5452023..1dad30d 100644 --- a/crates/fast-down-api/src/core/prefetch.rs +++ b/crates/fast-down-api/src/core/prefetch.rs @@ -1,14 +1,19 @@ -use crate::{Config, Event, Tx}; -use fast_down::{UrlInfo, http::Prefetch, reqwest::SmartRedirectClient}; +use crate::{Config, Event, Tx, tx_err, utils::build_header}; +use fast_down::{UrlInfo, fast_puller::build_client, http::Prefetch}; use reqwest::Response; use url::Url; -pub async fn prefetch( - url: &Url, - config: &Config, - client: &SmartRedirectClient, - tx: &Tx, -) -> Option<(UrlInfo, Response)> { +pub async fn prefetch(url: &Url, config: &Config, tx: &Tx) -> Option<(UrlInfo, Response)> { + let client = build_client( + build_header(&config.headers), + config.proxy.as_deref(), + config.accept_invalid_certs, + config.accept_invalid_hostnames, + config.cookie_store, + config.local_address.first().copied(), + config.max_redirects, + ); + let client = tx_err!(client, tx, BuildClientError, None); let mut retry_count = 0; loop { match client.prefetch(url.clone()).await { diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs index 9185163..cab2c48 100644 --- a/crates/fast-down-api/src/core/state.rs +++ b/crates/fast-down-api/src/core/state.rs @@ -1,7 +1,8 @@ use crate::{Config, PartialConfig}; -use fast_down::{FileId, Merge, ProgressEntry, UrlInfo}; +use fast_down::{FileId, ProgressEntry, UrlInfo}; use inherit_config::{ConfigLayer, InheritConfig}; use path_helper::tokio::safe_replace; +use reqwest::Response; use std::{ ops::Deref, path::{Path, PathBuf}, @@ -10,6 +11,38 @@ use std::{ use tokio::fs; use url::Url; +/// Errors that can occur when an explicit [`DownloadHandle::resume`](crate::DownloadHandle::resume) +/// cannot continue an interrupted download. +/// +/// These are surfaced through [`Event::ResumeError`] so the caller is notified +/// instead of silently falling back to a full re-download. +#[derive(Debug, thiserror::Error)] +#[allow(clippy::large_enum_variant)] +pub enum StateError { + /// No `.fd` state file exists for this download, so there is nothing to resume from. + #[error("no .fd state file to resume from")] + Open(std::io::Error), + /// No `.fd` state file exists for this download, so there is nothing to resume from. + #[error("no .fd state file to resume from")] + Save(std::io::Error), + /// No `.fd` state file exists for this download, so there is nothing to resume from. + #[error("no .fd state file to resume from")] + Decode(#[from] toml::de::Error), + #[error("no .fd state file to resume from")] + Encode(#[from] toml::ser::Error), + /// The remote file changed (size / etag / last-modified mismatch), so resuming would corrupt the output. + #[error("remote file changed, cannot resume")] + FileChanged { + local_file_id: FileId, + local_file_size: u64, + remote_file_id: FileId, + remote_file_size: u64, + }, + /// The server does not support resumable (range) downloads. + #[error("server does not support resumable download")] + NotResumable(UrlInfo, Response), +} + #[derive(Debug, Clone, InheritConfig)] pub struct DownloadStateInner { #[config(default = Url::parse("about:blank").unwrap())] @@ -18,7 +51,6 @@ pub struct DownloadStateInner { pub last_modified: Option>, #[config(nest)] pub config: Config, - pub progress: Vec, /// Total file size recorded at save time, compared against the server /// `UrlInfo.size` during resume validation. /// @@ -31,7 +63,7 @@ pub struct DownloadStateInner { pub struct DownloadState { inner: PartialDownloadStateInner, is_dirty: bool, - config_path: PathBuf, + pub config_path: PathBuf, } impl DownloadState { @@ -43,7 +75,6 @@ impl DownloadState { etag: Some(url_info.file_id.etag.clone()), last_modified: Some(url_info.file_id.last_modified.clone()), config: Some(config.clone()), - progress: Some(Vec::new()), size: Some(url_info.size), }, is_dirty: true, @@ -55,8 +86,8 @@ impl DownloadState { /// /// # Errors /// Returns an error if the file cannot be read or deserialized. - pub async fn load(config_path: &Path) -> anyhow::Result { - let inner = fs::read(&config_path).await?; + pub async fn load(config_path: &Path) -> Result { + let inner = fs::read(&config_path).await.map_err(StateError::Open)?; let inner: PartialDownloadStateInner = toml::from_slice(&inner)?; Ok(Self { inner, @@ -69,12 +100,14 @@ impl DownloadState { /// /// # Errors /// Returns an error if serializing or writing the state fails. - pub async fn store(&mut self) -> anyhow::Result<()> { + pub async fn store(&mut self) -> Result<(), StateError> { if self.is_dirty { self.inner .simplify_from(&PartialDownloadStateInner::default()); let inner = toml::to_string_pretty(&self.inner)?; - safe_replace(&self.config_path, inner.as_bytes()).await?; + safe_replace(&self.config_path, inner.as_bytes()) + .await + .map_err(StateError::Save)?; self.is_dirty = false; } Ok(()) @@ -92,9 +125,31 @@ impl DownloadState { /// sides are missing identity headers) the `FileId` (`etag` + /// `last_modified`) to be equal. Resumable downloads also require the /// server to support range requests. + /// + /// # Errors + #[allow(clippy::result_large_err)] + pub fn validate(&self, info: &UrlInfo) -> Result<(), StateError> { + let local_file_id = self.file_id(); + let local_file_size = self.size.unwrap_or(0); + let is_same = local_file_size == info.size && local_file_id == info.file_id; + if is_same { + Ok(()) + } else { + Err(StateError::FileChanged { + local_file_id, + local_file_size, + remote_file_id: info.file_id.clone(), + remote_file_size: info.size, + }) + } + } + #[must_use] - pub fn validate(&self, info: &UrlInfo) -> bool { - info.fast_download && self.size == Some(info.size) && self.file_id() == info.file_id + pub fn get_progress(&self) -> Vec { + self.config + .as_ref() + .and_then(|c| c.downloaded_chunk.clone()) + .unwrap_or_default() } /// Merge a freshly-written byte range into the recorded progress. @@ -103,21 +158,24 @@ impl DownloadState { /// are merged, de-duplicated and normalized. Marks the state dirty. pub fn merge_progress(&mut self, range: ProgressEntry) { self.inner - .progress - .get_or_insert_with(Vec::new) + .config + .get_or_insert_default() .merge_progress(range); self.is_dirty = true; } - /// Total number of bytes already downloaded, derived from `progress`. - /// - /// This value is intentionally not persisted; it is recomputed on demand - /// from the recorded ranges. - #[must_use] - pub fn downloaded_bytes(&self) -> u64 { - self.progress - .as_ref() - .map_or(0, |v| v.iter().map(|r| r.end - r.start).sum()) + pub fn merge_config(&mut self, partial_config: &PartialConfig) { + if let Some(config) = &self.config { + let mut pc = partial_config.clone(); + if let Some(downloaded_chunk) = &config.downloaded_chunk { + for i in downloaded_chunk { + pc.merge_progress(i.clone()); + } + } + pc.inherit_from(config); + self.inner.config = Some(pc); + self.is_dirty = true; + } } #[must_use] @@ -127,6 +185,11 @@ impl DownloadState { last_modified: self.last_modified.clone().flatten(), } } + + #[must_use] + pub fn tmp_path(&self) -> PathBuf { + self.config_path.with_extension("part") + } } impl Deref for DownloadState { diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs index ea9f4b0..c093411 100644 --- a/crates/fast-down-api/src/event.rs +++ b/crates/fast-down-api/src/event.rs @@ -1,36 +1,13 @@ -use crate::PartialConfig; -use fast_down::{FileId, ProgressEntry, UrlInfo, WorkerId, reqwest::ReqwestResponseError}; +use crate::{PartialConfig, StateError}; +use fast_down::{ProgressEntry, UrlInfo, WorkerId, reqwest::ReqwestResponseError}; use std::{path::PathBuf, sync::Arc}; -use thiserror::Error; - -/// Errors that can occur when an explicit [`DownloadHandle::resume`](crate::DownloadHandle::resume) -/// cannot continue an interrupted download. -/// -/// These are surfaced through [`Event::ResumeError`] so the caller is notified -/// instead of silently falling back to a full re-download. -#[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum ResumeError { - /// No `.fd` state file exists for this download, so there is nothing to resume from. - #[error("no .fd state file to resume from")] - NoStateFile, - /// The remote file changed (size / etag / last-modified mismatch), so resuming would corrupt the output. - #[error("remote file changed, cannot resume")] - FileChanged { - local_file_id: FileId, - local_file_size: u64, - remote_file_id: FileId, - remote_file_size: u64, - }, - /// The server does not support resumable (range) downloads. - #[error("server does not support resumable download")] - NotResumable, -} #[allow(clippy::large_enum_variant)] pub enum Event { Prefetch(UrlInfo), PrefetchError(ReqwestResponseError), GenPathError(std::io::Error), + StateSaveError(StateError), BuildClientError(reqwest::Error), BuildPusherError(std::io::Error), JoinError(Arc), @@ -56,7 +33,7 @@ pub enum Event { /// Emitted when an explicit `resume()` call cannot continue the download. /// Unlike `download()` (which silently falls back to a full re-download), /// `resume()` reports the failure so the caller can decide what to do. - ResumeError(ResumeError), + ResumeError(StateError), Pulling(WorkerId), PullError(WorkerId, anyhow::Error), diff --git a/crates/fast-down-api/src/utils/tx_err.rs b/crates/fast-down-api/src/utils/tx_err.rs index f5530ab..3db9f0e 100644 --- a/crates/fast-down-api/src/utils/tx_err.rs +++ b/crates/fast-down-api/src/utils/tx_err.rs @@ -10,4 +10,13 @@ macro_rules! tx_err { } } }; + ($x: expr, $tx: expr, $event: ident, $ret: expr) => { + match $x { + Ok(r) => r, + Err(e) => { + let _ = $tx.send(Event::$event(e)); + return $ret; + } + } + }; } diff --git a/crates/fast-down-api/tests/resume.rs b/crates/fast-down-api/tests.bak/resume.rs similarity index 100% rename from crates/fast-down-api/tests/resume.rs rename to crates/fast-down-api/tests.bak/resume.rs From 12e7990389d3a5aa5775631225963d7adf6d6a67 Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 14:49:09 +0800 Subject: [PATCH 18/26] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20clippy=20?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/src/core/state.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs index cab2c48..6fcda73 100644 --- a/crates/fast-down-api/src/core/state.rs +++ b/crates/fast-down-api/src/core/state.rs @@ -86,6 +86,7 @@ impl DownloadState { /// /// # Errors /// Returns an error if the file cannot be read or deserialized. + #[allow(clippy::result_large_err)] pub async fn load(config_path: &Path) -> Result { let inner = fs::read(&config_path).await.map_err(StateError::Open)?; let inner: PartialDownloadStateInner = toml::from_slice(&inner)?; @@ -100,6 +101,7 @@ impl DownloadState { /// /// # Errors /// Returns an error if serializing or writing the state fails. + #[allow(clippy::result_large_err)] pub async fn store(&mut self) -> Result<(), StateError> { if self.is_dirty { self.inner From af9fb01e13e96655b1cb3cd28b480879f5a1ece6 Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 17:40:47 +0800 Subject: [PATCH 19/26] =?UTF-8?q?docs:=20=E4=BC=98=E5=8C=96=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- README.md | 2 +- crates/fast-down-api/README.md | 112 +++++++++++++++++- crates/fast-down-api/src/config.rs | 7 ++ crates/fast-down-api/src/core/download/mod.rs | 32 +++++ .../src/core/download/overwrite.rs | 37 ++++++ .../src/core/download/pipeline.rs | 22 ++++ crates/fast-down-api/src/core/state.rs | 88 ++++++++++++-- crates/fast-down-api/src/event.rs | 59 +++++++++ crates/fast-down-api/src/lib.rs | 7 +- crates/fast-down/README.md | 98 ++++++++++++--- .../fast-down/src/http/content_disposition.rs | 7 ++ crates/fast-down/src/http/manual_redirect.rs | 9 ++ crates/fast-down/src/http/mod.rs | 24 +++- crates/fast-down/src/http/prefetch.rs | 7 ++ crates/fast-down/src/http/puller.rs | 16 +++ crates/fast-down/src/proxy.rs | 12 ++ crates/fast-down/src/reqwest/mod.rs | 26 ++++ crates/fast-down/src/url_info.rs | 19 +++ crates/fast-down/src/utils/fast_puller.rs | 22 ++++ crates/fast-down/src/utils/getifaddrs.rs | 7 ++ crates/fast-down/src/utils/mod.rs | 8 ++ crates/fast-pull/README.md | 98 +++++++++++++-- crates/fast-pull/src/base/event.rs | 5 +- crates/fast-pull/src/base/invert.rs | 7 ++ crates/fast-pull/src/base/merge.rs | 4 + crates/fast-pull/src/base/mod.rs | 8 ++ crates/fast-pull/src/base/progress.rs | 3 + crates/fast-pull/src/base/puller.rs | 13 ++ crates/fast-pull/src/base/pusher.rs | 13 ++ crates/fast-pull/src/cache/buf_writer.rs | 3 + crates/fast-pull/src/cache/direct.rs | 6 + crates/fast-pull/src/cache/merge.rs | 6 + crates/fast-pull/src/cache/mod.rs | 10 ++ crates/fast-pull/src/cache/seq.rs | 6 + crates/fast-pull/src/core/handle.rs | 2 + crates/fast-pull/src/core/mock.rs | 3 + crates/fast-pull/src/core/mod.rs | 32 ++++- crates/fast-pull/src/core/multi.rs | 2 + crates/fast-pull/src/core/single.rs | 2 + crates/fast-pull/src/file/cache_std.rs | 2 + crates/fast-pull/src/file/mmap.rs | 2 + crates/fast-pull/src/file/mod.rs | 7 ++ crates/fast-pull/src/file/std.rs | 2 + crates/fast-pull/src/mem/mod.rs | 2 + crates/fast-pull/src/mem/pusher.rs | 2 + crates/fast-steal/src/executor.rs | 5 + crates/fast-steal/src/lib.rs | 13 ++ crates/fast-steal/src/task.rs | 48 +++++++- crates/fast-steal/src/task_queue.rs | 28 +++++ 49 files changed, 905 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 05a277b..c1b8cda 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ - fast-steal: [![Latest version](https://img.shields.io/crates/v/fast-steal.svg)](https://crates.io/crates/fast-steal) [![Documentation](https://docs.rs/fast-steal/badge.svg)](https://docs.rs/fast-steal) - fast-pull: [![Latest version](https://img.shields.io/crates/v/fast-pull.svg)](https://crates.io/crates/fast-pull) [![Documentation](https://docs.rs/fast-pull/badge.svg)](https://docs.rs/fast-pull) - fast-down: [![Latest version](https://img.shields.io/crates/v/fast-down.svg)](https://crates.io/crates/fast-down) [![Documentation](https://docs.rs/fast-down/badge.svg)](https://docs.rs/fast-down) -- fast-down-ffi: [![Latest version](https://img.shields.io/crates/v/fast-down-ffi.svg)](https://crates.io/crates/fast-down-ffi) [![Documentation](https://docs.rs/fast-down-ffi/badge.svg)](https://docs.rs/fast-down-ffi) +- fast-down-api: [![Latest version](https://img.shields.io/crates/v/fast-down-api.svg)](https://crates.io/crates/fast-down-api) [![Documentation](https://docs.rs/fast-down-api/badge.svg)](https://docs.rs/fast-down-api) **[Official Website (Simplified Chinese)](https://fd.s121.top/)** diff --git a/crates/fast-down-api/README.md b/crates/fast-down-api/README.md index 43eed85..dbd6212 100644 --- a/crates/fast-down-api/README.md +++ b/crates/fast-down-api/README.md @@ -6,4 +6,114 @@ [![Documentation](https://docs.rs/fast-down-api/badge.svg)](https://docs.rs/fast-down-api) [![License](https://img.shields.io/crates/l/fast-down-api.svg)](https://github.com/fast-down/core/blob/main/LICENSE) -This library provides a convenient and easy-to-use wrapper around fast-down +A convenient, high-level wrapper around [`fast-down`](https://github.com/fast-down/fast-down) +that turns the pull/push engine into a few lines of async code: spawn a download, +drain progress events, resume after interruption, and cancel cooperatively. + +- **Concurrent, resumable downloads** powered by the `fast-down` engine (work-stealing, range requests). +- **Two entry points**: `download` (auto-resume when possible) and `resume` (hard error if it can't continue). +- **Event stream**: a single channel carries prefetch, per-worker progress, rename, and error events. +- **Cooperative cancellation**: cancelling mid-flight preserves the `.part` / `.fd` files so you can resume later. +- **Configurable**: threads, chunk size, write method (`Mmap` / `Std`), proxies, headers, retries, and more via `PartialConfig`. + +## Quick start + +```rust,ignore +use fast_down_api::{create_cancellation_token, create_channel, DownloadHandle, Event, PartialConfig}; +use std::path::PathBuf; +use url::Url; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // 1. Channel for progress / lifecycle events, plus a cancellation token. + let (tx, rx) = create_channel(); + let token = create_cancellation_token(); + + // 2. Configure the download. Every field is optional; unset fields + // fall back to Config::default(). (save_dir is required at runtime.) + let config = PartialConfig { + save_dir: Some(PathBuf::from("./downloads")), + overwrite: Some(true), + threads: Some(16), + ..Default::default() + }; + + // 3. Start the download. This spawns a detached task and returns at once. + let url = Url::parse("https://example.com/large-file.bin")?; + let handle = DownloadHandle::download(url, config, tx, token.clone()); + + // 4. Drain events until the task finishes or is cancelled. + while let Ok(event) = rx.recv().await { + match event { + Event::PushProgress(p) => println!("wrote range: {p:?}"), + Event::Renamed(path) => { + println!("done -> {path:?}"); + break; + } + Event::RenameFailed(e) => { + eprintln!("rename failed: {e}"); + break; + } + Event::ResumeError(e) => eprintln!("resume error: {e}"), + _ => {} + } + } + + // 5. Optionally await the spawned task (returns Err if it panicked). + handle.join().await?; + Ok(()) +} +``` + +### Resuming an interrupted download + +`resume` targets the existing `.part` file. If the download cannot be continued +(no `.fd` state, no range support, or the remote file changed) it emits +`Event::ResumeError` and **does not** silently restart — unlike `download`, +which auto-resumes when it can and otherwise falls back to a fresh download. + +```rust,ignore +let handle = DownloadHandle::resume( + "./downloads/large-file.bin.part", // the .part file from a previous run + url, + config, + tx, + token, +); +``` + +### Cancelling cooperatively + +```rust,ignore +token.cancel(); // stops fetching, keeps .part / .fd so you can resume later +``` + +## API overview + +| Item | Purpose | +| --- | --- | +| [`DownloadHandle::download`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.download) | Start a download; auto-resume when a valid `.fd` + `.part` exist, else fresh. | +| [`DownloadHandle::resume`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.resume) | Resume a specific `.part` file; hard-error (`Event::ResumeError`) if it can't. | +| [`DownloadHandle::join`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.join) | Await the detached task; errors if the worker panicked. | +| [`create_channel`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_channel.html) | Create the `(Tx, Rx)` event channel. | +| [`create_cancellation_token`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_cancellation_token.html) | Create a `CancellationToken` for cooperative cancellation. | +| [`Event`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.Event.html) | The event enum delivered over the channel. | +| [`PartialConfig`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.PartialConfig.html) | Layered, optional configuration for a download. | +| [`StateError`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.StateError.html) | Errors surfaced via `Event::ResumeError`. | + +## How resume works + +During a download the engine periodically persists a `.fd` state file next to +the `.part` partial file. That state records the byte ranges already written +(`downloaded_chunk`) and the remote file identity (`etag` / `last_modified` / +size). On the next run: + +1. `download` loads the `.fd`, validates it still matches the remote (size + identity), and — if the `.part` file is present — emits `Event::Resumed` and continues from the recorded offset. +2. If validation fails (remote changed) or there is no `.part`, it starts fresh. +3. `resume` applies the same checks but, instead of falling back, reports `Event::ResumeError`. + +Cancellation leaves both files in place, so a later `resume` (or `download`) can pick up exactly where it stopped. + +## License + +MIT — see [LICENSE](https://github.com/fast-down/core/blob/main/LICENSE). diff --git a/crates/fast-down-api/src/config.rs b/crates/fast-down-api/src/config.rs index eab6183..0b16450 100644 --- a/crates/fast-down-api/src/config.rs +++ b/crates/fast-down-api/src/config.rs @@ -170,6 +170,13 @@ pub struct Config { } impl PartialConfig { + /// Merge a freshly-written byte range into this partial config's progress. + /// + /// The range is folded into `downloaded_chunk` (created if absent), keeping + /// it normalized and de-duplicated. This is the in-memory counterpart of + /// [`DownloadState::merge_progress`](crate::DownloadState::merge_progress): + /// callers use it to record progress before handing the config to + /// [`DownloadHandle::resume`](crate::DownloadHandle::resume). pub fn merge_progress(&mut self, progress: ProgressEntry) { self.downloaded_chunk .get_or_insert_default() diff --git a/crates/fast-down-api/src/core/download/mod.rs b/crates/fast-down-api/src/core/download/mod.rs index 5437651..1fecfbb 100644 --- a/crates/fast-down-api/src/core/download/mod.rs +++ b/crates/fast-down-api/src/core/download/mod.rs @@ -32,6 +32,13 @@ fn open_create_new() -> OpenOptions { o } +/// A handle to a spawned download task. +/// +/// [`download`](Self::download) and [`resume`](Self::resume) spawn a detached +/// background task and return a `DownloadHandle`. Progress and lifecycle events +/// are delivered through the [`Tx`](crate::Tx) channel you pass in; call +/// [`join`](Self::join) to await the task's completion (it errors if the task +/// panicked). #[derive(Debug, Clone)] pub struct DownloadHandle { handle: SharedHandle<()>, @@ -47,6 +54,20 @@ impl DownloadHandle { self.handle.join().await } + /// Start a download, resuming automatically when possible. + /// + /// Spawns a detached task. The task first `prefetch`es metadata, then: + /// - if `config.resume` is enabled, a valid `.fd` state file and its `.part` + /// exist, and the remote file still matches ([`crate::Event::Resumed`]), it + /// continues from the recorded offset; + /// - otherwise it starts a fresh download. + /// + /// Unlike [`resume`](Self::resume), a failure to resume here is **not** an + /// error: the task silently falls back to a full re-download and emits the + /// normal event stream. Pass a + /// [`CancellationToken`](crate::create_cancellation_token) to cancel; on + /// cancellation the `.part`/`.fd` files are preserved so a later + /// [`resume`](Self::resume) can continue. #[must_use] pub fn download( url: Url, @@ -74,6 +95,17 @@ impl DownloadHandle { } } + /// Resume a previously interrupted download from its `.part` file. + /// + /// Spawns a detached task pinned to `tmp_path`. If the download cannot be + /// continued — the `.fd` state file is missing, the server does not support + /// range requests, or the remote file changed — the task emits + /// [`Event::ResumeError`](crate::Event::ResumeError) and returns **without** + /// falling back to a full re-download. This is the "hard error" counterpart + /// to [`download`](Self::download). + /// + /// If `tmp_path` itself does not exist, the call falls back to + /// [`download`](Self::download) (a fresh download under a unique name). pub fn resume( tmp_path: impl AsRef, url: Url, diff --git a/crates/fast-down-api/src/core/download/overwrite.rs b/crates/fast-down-api/src/core/download/overwrite.rs index a0f95f7..ee87030 100644 --- a/crates/fast-down-api/src/core/download/overwrite.rs +++ b/crates/fast-down-api/src/core/download/overwrite.rs @@ -1,3 +1,14 @@ +//! Full-pipeline download driver: persists state, runs the engine, then renames +//! the `.part` file into place. +//! +//! [`overwrite`] is the shared core of both [`crate::DownloadHandle::download`] +//! and [`crate::DownloadHandle::resume`]. It takes a fully-prepared +//! [`OverwriteOption`] (state + paths + prefetch result + channel + token), +//! builds the pull/push pipeline, drives [`fast_down::multi::download_multi`] or +//! [`fast_down::single::download_single`], forwards engine events to the public +//! [`crate::Event`] stream, periodically saves progress, and on success renames +//! the `.part` file to its final destination (or a unique variant when +//! `overwrite` is disabled). use crate::{ DownloadState, Event, PartialConfig, Tx, core::download::pipeline::build_pipeline, tx_err, }; @@ -20,15 +31,41 @@ use std::{ use tokio::fs; use tokio_util::sync::CancellationToken; +/// Fully-prepared inputs for [`overwrite`]. +/// +/// Callers build this once prefetch has succeeded and the [`DownloadState`] is +/// ready; [`overwrite`] then owns it and runs the download to completion. pub struct OverwriteOption { + /// The download state backing resume (progress, identity, `.fd` path). pub state: DownloadState, + /// Intended final destination of the file (before unique-path adjustment). pub final_path: PathBuf, + /// Prefetch metadata for the remote file (size, identity, range support). pub info: UrlInfo, + /// The prefetch HTTP response, reused to seed the first range request. pub resp: Response, + /// Channel to forward public [`crate::Event`]s to the consumer. pub tx: Tx, + /// Cancellation token; cancelling stops fetching and leaves `.part`/`.fd`. pub token: CancellationToken, } +/// Run a complete download: persist state, drive the engine, rename into place. +/// +/// This is the shared core of [`crate::DownloadHandle::download`] and +/// [`crate::DownloadHandle::resume`]. It: +/// 1. Saves the `.fd` state up front. +/// 2. Builds the pull/push pipeline for the `.part` file. +/// 3. Emits [`crate::Event::Start`] and runs `download_multi` (fast downloads) +/// or `download_single` (single-stream) according to `info.fast_download`. +/// 4. Forwards every engine event to the public [`crate::Event`] channel and +/// merges `PushProgress` ranges into the state. +/// 5. Periodically (≈1s) re-saves the `.fd` so progress survives interruption. +/// 6. On success, renames `.part` to `final_path` (or a unique variant when +/// `overwrite` is disabled) and emits [`crate::Event::Renamed`]. +/// +/// If the token is cancelled or the download did not complete, the `.part` and +/// `.fd` files are left in place so a later resume can continue. #[allow(clippy::too_many_lines)] pub async fn overwrite(option: OverwriteOption) { let OverwriteOption { diff --git a/crates/fast-down-api/src/core/download/pipeline.rs b/crates/fast-down-api/src/core/download/pipeline.rs index 94a2197..b132b59 100644 --- a/crates/fast-down-api/src/core/download/pipeline.rs +++ b/crates/fast-down-api/src/core/download/pipeline.rs @@ -1,3 +1,9 @@ +//! Builds the pull/push pipeline shared by `download` and `resume`. +//! +//! [`build_pipeline`] constructs a [`FastDownPuller`] (network side) and a +//! [`BoxPusher`] (file side) for the `.part` file, choosing the memory-mapped +//! writer on 64-bit targets when the server supports fast (resumable) downloads +//! and `Mmap` writing is configured, and the buffered/cache writer otherwise. use crate::{Config, Event, Tx, WriteMethod, core::download::open_existing, utils::build_header}; use fast_down::{ BoxPusher, UrlInfo, @@ -10,6 +16,22 @@ use std::{path::Path, sync::Arc}; use tokio_util::sync::CancellationToken; use url::Url; +/// Construct the (puller, pusher) pipeline for a `.part` file. +/// +/// Returns `None` (after forwarding the failure as a public +/// [`crate::Event`]) if the HTTP client or the output file cannot be created, +/// or if `token` is cancelled before construction finishes. +/// +/// * `url` / `config` drive the puller (headers, proxy, cert handling, range +/// identity, local bind address, redirect limit). +/// * `info` supplies the file identity used for range validation and selects +/// the writer: on 64-bit targets a resumable `info.fast_download` download +/// with [`WriteMethod::Mmap`] uses [`MmapFilePusher`]; otherwise +/// [`CacheFilePusher`] (buffered + out-of-order reordering). +/// * `resp` is the prefetch response, reused to seed the first range request +/// without an extra round-trip. +/// * `path` is the `.part` file; `tx` receives error events; `token` makes +/// construction cancellable. pub async fn build_pipeline( url: &Url, config: &Config, diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs index 6fcda73..3f24dd2 100644 --- a/crates/fast-down-api/src/core/state.rs +++ b/crates/fast-down-api/src/core/state.rs @@ -14,24 +14,39 @@ use url::Url; /// Errors that can occur when an explicit [`DownloadHandle::resume`](crate::DownloadHandle::resume) /// cannot continue an interrupted download. /// -/// These are surfaced through [`Event::ResumeError`] so the caller is notified +/// These are surfaced through [`crate::Event::ResumeError`] so the caller is notified /// instead of silently falling back to a full re-download. #[derive(Debug, thiserror::Error)] #[allow(clippy::large_enum_variant)] pub enum StateError { - /// No `.fd` state file exists for this download, so there is nothing to resume from. - #[error("no .fd state file to resume from")] + /// Failed to open or read the `.fd` state file from disk. + /// + /// This usually means the file is missing or unreadable (permission error, + /// removed by another process, etc.). + #[error("failed to open .fd state file: {0}")] Open(std::io::Error), - /// No `.fd` state file exists for this download, so there is nothing to resume from. - #[error("no .fd state file to resume from")] + /// Failed to persist the `.fd` state file to disk. + /// + /// The state was computed but could not be written (disk full, permission + /// error, etc.). + #[error("failed to save .fd state file: {0}")] Save(std::io::Error), - /// No `.fd` state file exists for this download, so there is nothing to resume from. - #[error("no .fd state file to resume from")] + /// Failed to deserialize the `.fd` state file. + /// + /// The file was read but is not valid TOML, or its schema no longer matches + /// the current [`DownloadStateInner`] definition. + #[error("failed to decode .fd state file: {0}")] Decode(#[from] toml::de::Error), - #[error("no .fd state file to resume from")] + /// Failed to serialize the download state into the `.fd` state file. + /// + /// The state could not be encoded as TOML (e.g. a field holds a value that + /// has no valid TOML representation). + #[error("failed to encode .fd state file: {0}")] Encode(#[from] toml::ser::Error), /// The remote file changed (size / etag / last-modified mismatch), so resuming would corrupt the output. - #[error("remote file changed, cannot resume")] + #[error( + "remote file changed, cannot resume\n local: size={local_file_size}, id={local_file_id:?}\n remote: size={remote_file_size}, id={remote_file_id:?}" + )] FileChanged { local_file_id: FileId, local_file_size: u64, @@ -39,10 +54,20 @@ pub enum StateError { remote_file_size: u64, }, /// The server does not support resumable (range) downloads. - #[error("server does not support resumable download")] + #[error( + "server does not support resumable download\n url_info: {:?}\n url: {}\n status: {}\n headers: {:?}", + .0, .1.url(), .1.status(), .1.headers() + )] NotResumable(UrlInfo, Response), } +/// Full (resolved) download state that is serialized into the `.fd` file. +/// +/// `DownloadStateInner` is the non-partial form of the persisted state: every +/// field is present. It is encoded to TOML by [`DownloadState::store`] and read +/// back as the generated partial [`PartialDownloadStateInner`] on resume. The +/// `size` field is recorded for resume validation; older `.fd` files omit it and +/// read back as `None` in the partial form. #[derive(Debug, Clone, InheritConfig)] pub struct DownloadStateInner { #[config(default = Url::parse("about:blank").unwrap())] @@ -59,6 +84,18 @@ pub struct DownloadStateInner { pub size: u64, } +/// On-disk state for an in-progress download, backing resume support. +/// +/// `DownloadState` pairs a [`PartialDownloadStateInner`] (the persisted config, +/// whose fields may be absent if the `.fd` file predates them) with the path of +/// the `.fd` file and a dirty flag. It is the bridge between a saved `.fd` file +/// and a fresh [`crate::PartialConfig`] handed to +/// [`crate::DownloadHandle::resume`]: [`DownloadState::merge_config`] folds the +/// loaded progress into the new request so a resumed download continues from the +/// correct byte offset instead of restarting from zero. +/// +/// `DownloadState` derefs to [`PartialDownloadStateInner`], so the saved `url`, +/// `etag`, `last_modified`, `config` and `size` are reachable directly. #[derive(Debug)] pub struct DownloadState { inner: PartialDownloadStateInner, @@ -67,6 +104,12 @@ pub struct DownloadState { } impl DownloadState { + /// Build a fresh, dirty download state from the initial prefetch metadata. + /// + /// `url` and `url_info` come from the prefetch step, `config` is the + /// caller-provided [`PartialConfig`], and `config_path` is where the `.fd` + /// file will be written. The returned state is marked dirty so the first + /// [`DownloadState::store`] persists it. #[must_use] pub fn new(url: &Url, url_info: &UrlInfo, config: &PartialConfig, config_path: &Path) -> Self { Self { @@ -115,6 +158,10 @@ impl DownloadState { Ok(()) } + /// Apply a mutation to the inner partial state and mark it dirty. + /// + /// Any change made through `cb` (e.g. updating `etag`/`last_modified` or the + /// nested `config`) schedules the next [`DownloadState::store`]. pub fn update(&mut self, cb: impl FnOnce(&mut PartialDownloadStateInner)) { cb(&mut self.inner); self.is_dirty = true; @@ -146,6 +193,10 @@ impl DownloadState { } } + /// Return the list of byte ranges already written to the `.part` file. + /// + /// This is the authoritative on-disk progress. It is empty for a brand new + /// download and grows as [`DownloadState::merge_progress`] is called. #[must_use] pub fn get_progress(&self) -> Vec { self.config @@ -166,6 +217,15 @@ impl DownloadState { self.is_dirty = true; } + /// Fold a freshly-built [`PartialConfig`] into this loaded state for resume. + /// + /// This is the key bridge that preserves download progress across a resume: + /// the ranges already recorded in `self.config.downloaded_chunk` are merged + /// into `partial_config` first, so that even if `partial_config` starts from + /// an empty progress list the resumed download keeps the already-downloaded + /// bytes. `partial_config` is then layered on top via + /// `inherit_from`, and the merged result replaces the stored + /// config. Marks the state dirty. pub fn merge_config(&mut self, partial_config: &PartialConfig) { if let Some(config) = &self.config { let mut pc = partial_config.clone(); @@ -180,6 +240,11 @@ impl DownloadState { } } + /// Reconstruct the identity of the file this state was saved for. + /// + /// Returns a [`FileId`] from the stored `etag` / `last_modified`. Missing + /// headers collapse to `None`, which makes [`DownloadState::validate`] treat + /// "no identity on either side" as a match. #[must_use] pub fn file_id(&self) -> FileId { FileId { @@ -188,6 +253,9 @@ impl DownloadState { } } + /// Path of the partial (`.part`) output file paired with this state. + /// + /// Derived from `config_path` by swapping the extension to `.part`. #[must_use] pub fn tmp_path(&self) -> PathBuf { self.config_path.with_extension("part") diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs index c093411..48d59a0 100644 --- a/crates/fast-down-api/src/event.rs +++ b/crates/fast-down-api/src/event.rs @@ -2,21 +2,60 @@ use crate::{PartialConfig, StateError}; use fast_down::{ProgressEntry, UrlInfo, WorkerId, reqwest::ReqwestResponseError}; use std::{path::PathBuf, sync::Arc}; +/// Events emitted by a download run, consumed through the crossfire channel +/// returned by [`crate::create_channel`]. +/// +/// Events cover the full lifecycle: prefetch ([`Event::Prefetch`]), pipeline +/// setup ([`Event::Start`]), per-worker fetch/write progress +/// ([`Event::Pulling`] … [`Event::Finished`]), resume ([`Event::Resumed`], +/// [`Event::ResumeError`]), and completion ([`Event::Renamed`]). Error variants +/// (`*Error`) report failures without aborting the stream, so a consumer can +/// decide whether to retry, cancel, or surface them in a UI. #[allow(clippy::large_enum_variant)] pub enum Event { + /// Emitted after the prefetch step resolves the remote file's metadata. + /// + /// Carries the [`UrlInfo`] (size, identity headers, range support) so a + /// caller can inspect the remote before or while the download proceeds. Prefetch(UrlInfo), + /// The prefetch request failed (server unreachable, non-2xx response, etc.). + /// + /// Fatal for the download: without [`UrlInfo`] the engine cannot plan ranges. PrefetchError(ReqwestResponseError), + /// Failed to compute the output / `.part` / `.fd` paths. + /// + /// For example the target directory is not writable or the file name is + /// invalid on this platform. GenPathError(std::io::Error), + /// Persisting the `.fd` state file failed (see [`StateError`]). + /// + /// Non-fatal for the current run, but the download can no longer be resumed + /// reliably if it is interrupted afterwards. StateSaveError(StateError), + /// Building the HTTP client failed (TLS / backend initialization, etc.). BuildClientError(reqwest::Error), + /// Creating the output sink — opening the `.part` file — failed. BuildPusherError(std::io::Error), + /// A spawned download worker task panicked or was cancelled. + /// + /// The engine joins its worker tasks; if a join handle reports an error it is + /// surfaced here rather than silently dropping the panic. JoinError(Arc), + /// The final rename of the `.part` file to its destination failed. + /// + /// The success counterpart is [`Event::Renamed`]. The bytes are already on + /// disk under the `.part` name, so they can still be resumed or retried. RenameFailed(std::io::Error), /// Emitted after the `.part` file is successfully renamed to its final /// destination. Carries the actual landing path, which in unique mode may /// differ from the originally-planned name (e.g. `xxx (1).mp4`) when the /// target got occupied during the download. Renamed(PathBuf), + /// Emitted once the pipeline is set up and writing is about to begin. + /// + /// Carries the `.part` path, the `.fd` state-file path, and the resolved + /// [`PartialConfig`] actually used for this run (after merging any resumed + /// progress and applying defaults). Start { tmp_path: PathBuf, config_path: PathBuf, @@ -35,14 +74,34 @@ pub enum Event { /// `resume()` reports the failure so the caller can decide what to do. ResumeError(StateError), + /// Worker `id` started fetching its assigned byte range from the network. Pulling(WorkerId), + /// Worker `id` failed to fetch its assigned range (network / decode error). PullError(WorkerId, anyhow::Error), + /// Worker `id`'s fetch exceeded its time budget and was aborted. PullTimeout(WorkerId), + /// Worker `id` pulled some bytes into memory. + /// + /// `ProgressEntry` describes the contiguous range that just arrived from the + /// network but is not yet written to disk. PullProgress(WorkerId, ProgressEntry), + /// Worker `id` is handing a pulled range to the sink (writing it to `.part`). + /// + /// `ProgressEntry` is the range being written. Pushing(WorkerId, ProgressEntry), + /// Worker `id` failed to write `ProgressEntry` to the sink. PushError(WorkerId, ProgressEntry, anyhow::Error), + /// A range was written to the sink. + /// + /// `ProgressEntry` is the range persisted (at least to the OS page cache) by + /// this write. This drives the progress bar and the resume bookkeeping; it is + /// intentionally emitted before `fsync`, so it reflects "written", not + /// "durably flushed". PushProgress(ProgressEntry), + /// The sink is being flushed and synced to the `.part` file. Flushing, + /// Flushing / syncing the sink failed. FlushError(anyhow::Error), + /// Worker `id` completed its assigned range and exited. Finished(WorkerId), } diff --git a/crates/fast-down-api/src/lib.rs b/crates/fast-down-api/src/lib.rs index 3f3d4a1..cb07433 100644 --- a/crates/fast-down-api/src/lib.rs +++ b/crates/fast-down-api/src/lib.rs @@ -28,8 +28,11 @@ pub fn create_channel() -> (Tx, Rx) { /// Create a new cancellation token for use with download tasks. /// -/// Pass the token to any `DownloadTask` method (`start`, `start_with_pusher`, -/// or `start_in_memory`) to cancel the download at any time. +/// Pass the token to [`DownloadHandle::download`] or [`DownloadHandle::resume`] +/// to cancel the download at any time. Cancellation is cooperative: the running +/// task stops fetching, leaves the `.part`/`.fd` files in place, and returns +/// without renaming — so a later [`DownloadHandle::resume`] call can continue +/// from where it stopped. #[must_use] pub fn create_cancellation_token() -> CancellationToken { CancellationToken::new() diff --git a/crates/fast-down/README.md b/crates/fast-down/README.md index db44313..04b196a 100644 --- a/crates/fast-down/README.md +++ b/crates/fast-down/README.md @@ -6,20 +6,90 @@ [![Documentation](https://docs.rs/fast-down/badge.svg)](https://docs.rs/fast-down) [![License](https://img.shields.io/crates/l/fast-down.svg)](https://github.com/fast-down/core/blob/main/LICENSE) -`fast-down` **Fastest** concurrent downloader! +`fast-down` is a fast, concurrent file downloader library built on top of the +[`fast_pull`](https://crates.io/crates/fast-pull) pull/push engine, with first-class +HTTP support. **[Official Website (Simplified Chinese)](https://fd.s121.top/)** -## Features - -1. **⚡️ Fastest Download**\ - We created [fast-steal](https://github.com/fast-down/fast-steal) With optimized Work Stealing, **1.43 x faster** than - NDM. -2. **🔄 File consistency**\ - Switching Wi-Fi, Turn Off Wi-Fi, Switch proxies. **We guarantee the consistency**. -3. **⛓️‍💥 Resuming Downloads**\ - You can **interrupt** at any time, and **resume downloading** after. -4. **⛓️‍💥 Incremental Downloads**\ - 1000 more lines server logs? Don't worry, we **only download new lines**. -5. **💰 Free and open-source**\ - The code stays free and open-source. Thanks to [share121](https://github.com/share121), [Cyan](https://github.com/CyanChanges) and other fast-down contributors. +## What this crate provides + +`fast-down` re-exports the entire [`fast_pull`](https://crates.io/crates/fast-pull) +pull/push engine (the `download_single` / `download_multi` entry points, the `Puller` +and `Pusher` traits, and the file/memory pushers) and adds the following on top of it: + +1. **HTTP / reqwest puller** — `FastDownPuller` (with `FastDownPullerOptions`) is the + default `Puller` for HTTP(S) sources. It builds on a `SmartRedirectClient` that + follows redirects _manually_ so it can honor the `Referrer-Policy` header and strip + resource-specific headers (`Origin` / `Authorization` / `Cookie`) on cross-origin + hops, per RFC 9110 §15.4. +2. **URL info resolution** — `UrlInfo` and `FileId` capture a resource's size, suggested + filename, content type, range support, and a stable identity derived from the + `ETag` / `Last-Modified` headers, which powers incremental and resumable downloads. +3. **Proxy support** — the `Proxy` enum selects no proxy, the system proxy, or a custom + proxy URL for outgoing requests. +4. **Task handles** — the re-exported `SharedHandle` task-handle type lets you await + and abort an in-flight download from multiple owners. + +Supporting building blocks (behind feature flags) include the backend-agnostic +`http` module (`HttpClient` traits, `HttpPuller`, `Prefetch`, `ContentDisposition`, +`HttpError`) and the `reqwest` module (`SmartRedirectClient`, +`ManualRedirectRequestBuilder`). + +## Example + +Download a file concurrently to disk. This requires the `reqwest` feature +(which also enables `http` and `fast-puller`) and a network connection, so the +block is marked `ignore` to keep doctests hermetic: + +```rust,ignore +use std::sync::Arc; +use std::time::Duration; +use url::Url; + +use fast_down::{FastDownPuller, FastDownPullerOptions, FileId, Proxy}; +use fast_pull::file::StdFilePusher; +use fast_pull::multi::DownloadOptions; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let url = Url::parse("https://example.com/large.bin")?; + let file = tokio::fs::File::create("large.bin").await?; + // Pre-size the file; pass `true` to fsync on flush. + let pusher = StdFilePusher::new(file, /* size */ 0, /* sync_all */ false).await?; + + let puller = FastDownPuller::new(FastDownPullerOptions { + url, + headers: Arc::new(Default::default()), + proxy: Proxy::System, + accept_invalid_certs: false, + accept_invalid_hostnames: false, + cookie_store: false, + file_id: FileId::default(), + resp: None, + available_ips: Arc::from(Vec::::new()), + max_redirects: 10, + })?; + + let result = fast_pull::download_multi( + puller, + pusher, + DownloadOptions { + download_chunks: vec![0..u64::MAX].into_iter(), + concurrent: 8, + retry_gap: Duration::from_secs(1), + pull_timeout: Duration::from_secs(30), + push_queue_cap: 1024, + min_chunk_size: 1 << 20, + max_speculative: 3, + }, + ); + + result.join().await?; + Ok(()) +} +``` + +For a sequential, single-threaded download, swap `download_multi` for +`fast_pull::download_single` and use `fast_pull::single::DownloadOptions` +(which only has `retry_gap` and `push_queue_cap`). diff --git a/crates/fast-down/src/http/content_disposition.rs b/crates/fast-down/src/http/content_disposition.rs index 95715bc..c6ac1ee 100644 --- a/crates/fast-down/src/http/content_disposition.rs +++ b/crates/fast-down/src/http/content_disposition.rs @@ -1,3 +1,10 @@ +//! Parser for the HTTP `Content-Disposition` header. +//! +//! Used during prefetch to derive a suggested filename for a downloaded +//! resource. [`ContentDisposition::parse`] handles both the quoted `filename` +//! form and the RFC 5987 `filename*` (UTF-8, percent-encoded) form, with the +//! latter taking precedence when both are present. + use std::{iter::Peekable, str::Chars}; /// Parsed `Content-Disposition` header, extracting the `filename` parameter. diff --git a/crates/fast-down/src/http/manual_redirect.rs b/crates/fast-down/src/http/manual_redirect.rs index 86f2445..7d49a91 100644 --- a/crates/fast-down/src/http/manual_redirect.rs +++ b/crates/fast-down/src/http/manual_redirect.rs @@ -1,3 +1,11 @@ +//! Manual redirect handling that respects the `Referrer-Policy` header. +//! +//! When `SmartRedirectClient` follows redirects itself (rather +//! than letting `reqwest` do it), it must compute the correct `Referer` header +//! for each hop. [`compute_referer`] implements the W3C Referrer Policy +//! algorithm (with RFC 9110 §7.4 defaults), and [`ReferrerPolicy`] parses the +//! policy tokens sent by servers. + use url::Url; /// Serialize a URL for use as a `Referer` header value, @@ -51,6 +59,7 @@ impl ReferrerPolicy { } } +/// Returns `true` when following `from` -> `to` would downgrade from HTTPS to HTTP. fn is_downgrade(from: &Url, to: &Url) -> bool { from.scheme() == "https" && to.scheme() == "http" } diff --git a/crates/fast-down/src/http/mod.rs b/crates/fast-down/src/http/mod.rs index 3fd28e1..c1daf77 100644 --- a/crates/fast-down/src/http/mod.rs +++ b/crates/fast-down/src/http/mod.rs @@ -1,3 +1,21 @@ +//! A backend-agnostic HTTP download layer. +//! +//! This module defines a set of small traits — [`HttpClient`], +//! [`HttpRequestBuilder`], [`HttpResponse`], and [`HttpHeaders`] — that abstract +//! over any HTTP client implementation (for example the `reqwest` wrapper in +//! the `reqwest` module). On top of them it provides: +//! +//! * [`HttpPuller`]: a [`fast_pull::Puller`] that streams bytes over HTTP, with +//! range support and file-identity (resumability) checks. +//! * [`Prefetch`]: resolves a [`crate::UrlInfo`] for a URL via a prefetch request. +//! * [`ContentDisposition`]: parses the `Content-Disposition` header for filenames. +//! * [`manual_redirect`]: RFC 9110-aware `Referer` computation for redirect following. +//! * [`HttpError`]: the error type produced by this layer. +//! +//! Most users do not use these types directly; instead they use +//! `FastDownPuller` (from the `fast-puller` feature), which wraps +//! [`HttpPuller`] with a smart-redirecting `reqwest` client. + mod content_disposition; pub mod manual_redirect; mod prefetch; @@ -61,13 +79,13 @@ pub type GetHeaderError = as HttpHeaders>::GetHeaderE /// detecting mismatched file identity, and irrecoverable failures. #[derive(thiserror::Error)] pub enum HttpError { - #[error("HTTP request failed")] + #[error("HTTP request failed: {0:?}")] Request(GetRequestError), - #[error("HTTP chunk read failed")] + #[error("HTTP chunk read failed: {0:?}\n response: {1:?}")] Chunk(GetChunkError, GetResponse), #[error("irrecoverable pull error")] Irrecoverable, - #[error("body mismatch: expected file {0:?}, got different content")] + #[error("body mismatch: expected file {0:?}, got different content\n response: {1:?}")] MismatchedBody(FileId, GetResponse), } diff --git a/crates/fast-down/src/http/prefetch.rs b/crates/fast-down/src/http/prefetch.rs index 6eac3f5..42dbf3c 100644 --- a/crates/fast-down/src/http/prefetch.rs +++ b/crates/fast-down/src/http/prefetch.rs @@ -1,3 +1,10 @@ +//! Prefetch metadata for a downloadable URL over HTTP. +//! +//! [`Prefetch::prefetch`] issues the initial GET (and a range probe) through a +//! [`crate::http::HttpClient`], then assembles a [`crate::UrlInfo`] describing +//! the resource: its size, suggested filename, content type, range support, and +//! the [`crate::FileId`] used for resumable downloads. + use crate::{ UrlInfo, http::{ diff --git a/crates/fast-down/src/http/puller.rs b/crates/fast-down/src/http/puller.rs index 7925415..2658d2a 100644 --- a/crates/fast-down/src/http/puller.rs +++ b/crates/fast-down/src/http/puller.rs @@ -1,3 +1,11 @@ +//! An HTTP implementation of the [`fast_pull::Puller`] trait. +//! +//! [`HttpPuller`] builds range requests through the generic [`crate::http::HttpClient`] +//! trait and streams the response body back as a [`fast_pull::PullStream`]. It +//! verifies the server's `ETag` / `Last-Modified` headers against the expected +//! [`crate::FileId`] so that a changed file is reported as [`crate::http::HttpError::MismatchedBody`] +//! rather than silently corrupting an incremental download. + use crate::http::{ FileId, GetRequestError, GetResponse, HttpClient, HttpError, HttpHeaders, HttpRequestBuilder, HttpResponse, @@ -39,6 +47,14 @@ impl Clone for HttpPuller { } } impl HttpPuller { + /// Create a new [`HttpPuller`]. + /// + /// * `url` — the resource to download. + /// * `client` — the HTTP client used to issue requests. + /// * `resp` — an optional already-open response to reuse for the first + /// (full-file) request, typically the one produced by a prefetch. + /// * `file_id` — the expected [`crate::FileId`], compared against the + /// server's headers to detect a changed resource. pub const fn new( url: Arc, client: Client, diff --git a/crates/fast-down/src/proxy.rs b/crates/fast-down/src/proxy.rs index 2261d97..1a96fa9 100644 --- a/crates/fast-down/src/proxy.rs +++ b/crates/fast-down/src/proxy.rs @@ -1,3 +1,9 @@ +//! Proxy selection for outgoing HTTP requests. +//! +//! This module defines the [`Proxy`] enum, which tells a `FastDownPuller` +//! how to route its connections: no proxy, the system proxy (honoring +//! platform/Environment settings), or a caller-supplied custom proxy URL. + use std::ops::Deref; /// Proxy configuration for outgoing HTTP requests. @@ -14,6 +20,8 @@ pub enum Proxy { } impl Proxy { + /// Transform the inner custom value, leaving [`Proxy::No`] and + /// [`Proxy::System`] unchanged. pub fn map(self, f: impl FnOnce(T) -> U) -> Proxy { match self { Self::No => Proxy::No, @@ -22,6 +30,8 @@ impl Proxy { } } + /// Borrow the inner custom value as a reference, leaving [`Proxy::No`] and + /// [`Proxy::System`] unchanged. Requires `T: Deref`. pub fn as_deref(&self) -> Proxy<&T::Target> where T: Deref, @@ -33,6 +43,8 @@ impl Proxy { } } + /// Borrow the inner custom value as a shared reference, leaving + /// [`Proxy::No`] and [`Proxy::System`] unchanged. pub const fn as_ref(&self) -> Proxy<&T> { match self { Self::No => Proxy::No, diff --git a/crates/fast-down/src/reqwest/mod.rs b/crates/fast-down/src/reqwest/mod.rs index 4bbcf8c..0a27e2c 100644 --- a/crates/fast-down/src/reqwest/mod.rs +++ b/crates/fast-down/src/reqwest/mod.rs @@ -1,5 +1,19 @@ #![cfg(not(target_family = "wasm"))] +//! A `reqwest`-based implementation of the [`crate::http`] HTTP traits with +//! smart redirect handling. +//! +//! This module adapts `reqwest` to the backend-agnostic [`crate::http::HttpClient`] +//! trait family and, more importantly, provides [`SmartRedirectClient`]: a +//! `reqwest::Client` wrapper that follows redirects **manually** so it can honor +//! the `Referrer-Policy` header and strip resource-specific headers +//! (`Origin` / `Authorization` / `Cookie`) on cross-origin hops, per RFC 9110 +//! §15.4. The corresponding request builder is [`ManualRedirectRequestBuilder`]. +//! +//! Most users do not construct these types directly; instead they build a +//! `FastDownPuller` via `build_client`, which creates a +//! correctly-configured [`SmartRedirectClient`]. + use crate::http::{ HttpClient, HttpHeaders, HttpRequestBuilder, HttpResponse, manual_redirect::{ReferrerPolicy, compute_referer}, @@ -118,6 +132,18 @@ pub struct SmartRedirectClient { } impl SmartRedirectClient { + /// Build a [`SmartRedirectClient`] from an already-constructed `reqwest::Client`. + /// + /// * `client` — the underlying client. **Must** be built with + /// `redirect(reqwest::redirect::Policy::none())`, otherwise the manual + /// redirect logic here conflicts with reqwest's own auto-follow. + /// * `initial_referer` — the `Referer` sent on the first request. + /// * `referrer_policy` — the policy applied when no `Referrer-Policy` + /// header is present on a response; per-hop headers override it. + /// * `origin` / `authorization` / `cookie` — resource-specific headers + /// injected only on the first hop and stripped on redirect (RFC 9110 §15.4). + /// * `max_redirects` — the maximum number of redirects to follow before + /// failing with a `StatusCode` error. #[must_use] pub const fn new( client: Client, diff --git a/crates/fast-down/src/url_info.rs b/crates/fast-down/src/url_info.rs index d9cb02b..a2648ca 100644 --- a/crates/fast-down/src/url_info.rs +++ b/crates/fast-down/src/url_info.rs @@ -1,3 +1,11 @@ +//! Resolution of downloadable-resource metadata from an initial HTTP request. +//! +//! After a `FastDownPuller` performs a prefetch, it produces a +//! [`UrlInfo`] describing the resource (size, name, content type, range support) +//! together with a [`FileId`] (derived from the `ETag` / `Last-Modified` +//! headers) used to detect when a previously-downloaded file is still valid for +//! incremental/resumable downloads. + use std::sync::Arc; use url::Url; @@ -8,6 +16,7 @@ use url::Url; #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Debug, Clone, PartialEq, Eq)] pub struct UrlInfo { + /// Total size of the resource in bytes, taken from the `Content-Length` header (0 if absent). pub size: u64, /// Raw filename returned by the server. Sanitize invalid characters before using it safely. #[cfg_attr( @@ -19,10 +28,15 @@ pub struct UrlInfo { doc = "Enable the `sanitize-filename` feature to use the `filename()` method for sanitization." )] pub raw_name: String, + /// Whether the server supports HTTP range requests (used to split the download into concurrent chunks). pub supports_range: bool, + /// Whether the resource can be downloaded with the optimized fast (multi-chunk) path. pub fast_download: bool, + /// The URL the response was actually served from, after any redirects. pub final_url: Url, + /// Stable identity of the file, used to validate incremental/resumable downloads. pub file_id: FileId, + /// The `Content-Type` header value, if the server provided one. pub content_type: Option, } @@ -37,14 +51,19 @@ impl UrlInfo { /// File identity used for incremental and resumable downloads. /// /// Combines the `ETag` and `Last-Modified` headers into a stable identifier. +/// Two downloads of the same logical file share a [`FileId`] only if neither +/// header changed, which lets the engine resume instead of re-downloading. #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct FileId { + /// The `ETag` header value, if present. pub etag: Option>, + /// The `Last-Modified` header value, if present. pub last_modified: Option>, } impl FileId { + /// Build a [`FileId`] from borrowed header values, copying them into `Arc`. pub fn new(etag: Option<&str>, last_modified: Option<&str>) -> Self { Self { etag: etag.map(Arc::from), diff --git a/crates/fast-down/src/utils/fast_puller.rs b/crates/fast-down/src/utils/fast_puller.rs index 7795502..3f6bc04 100644 --- a/crates/fast-down/src/utils/fast_puller.rs +++ b/crates/fast-down/src/utils/fast_puller.rs @@ -1,3 +1,13 @@ +//! The default HTTP [`fast_pull::Puller`] for this crate. +//! +//! [`FastDownPuller`] ties together the [`crate::http::HttpPuller`] engine and a +//! `SmartRedirectClient`, adding proxy support, optional +//! multi-interface IP rotation, and file-identity-based resumability. Construct +//! one from [`FastDownPullerOptions`] (typically via [`build_client`] to wire up +//! the underlying reqwest client), then pass it to `fast_pull::download_multi` +//! or `fast_pull::download_single` alongside a `Pusher` such as +//! `fast_pull::file::StdFilePusher` (requires the `file` feature of `fast-pull`). + use crate::Proxy; use crate::{ FileId, ProgressEntry, PullResult, PullStream, @@ -81,19 +91,31 @@ pub struct FastDownPuller { turn: Arc, max_redirects: usize, } +// Field-level docs live on [`FastDownPullerOptions`], the public construction +// surface; the runtime struct mirrors those fields. /// Options for constructing a [`FastDownPuller`]. #[derive(Debug)] pub struct FastDownPullerOptions<'a> { + /// The URL to download. pub url: Url, + /// Extra request headers sent on every request. pub headers: Arc, + /// Proxy selection (no / system / custom URL). pub proxy: Proxy<&'a str>, + /// Accept invalid TLS certificates (requires the `reqwest-tls` feature). pub accept_invalid_certs: bool, + /// Accept invalid TLS hostnames (requires the `reqwest-tls` feature). pub accept_invalid_hostnames: bool, + /// Enable a cookie store (requires the `cookie-store` feature). pub cookie_store: bool, + /// The expected [`FileId`], used to detect a changed resource and resume safely. pub file_id: FileId, + /// An already-open response to reuse for the first request (e.g. from a prefetch). pub resp: Option>>>, + /// Candidate local source IPs for outbound connections; rotated across clones. pub available_ips: Arc<[std::net::IpAddr]>, + /// Maximum number of redirects to follow before failing. pub max_redirects: usize, } diff --git a/crates/fast-down/src/utils/getifaddrs.rs b/crates/fast-down/src/utils/getifaddrs.rs index 493c3e5..ee39ac5 100644 --- a/crates/fast-down/src/utils/getifaddrs.rs +++ b/crates/fast-down/src/utils/getifaddrs.rs @@ -1,3 +1,10 @@ +//! Enumeration of usable local network interfaces. +//! +//! [`get_available_local_ips`] returns the non-loopback, non-virtual, +//! non-link-local IP addresses of the host. This powers the multi-interface +//! IP-rotation feature of `FastDownPuller`, where each clone can bind +//! to a different local address. + use std::net::IpAddr; /// # Errors diff --git a/crates/fast-down/src/utils/mod.rs b/crates/fast-down/src/utils/mod.rs index f0a4792..7fd8ce2 100644 --- a/crates/fast-down/src/utils/mod.rs +++ b/crates/fast-down/src/utils/mod.rs @@ -1,3 +1,11 @@ +//! Internal helpers backing the higher-level pullers. +//! +//! * [`fast_puller`] (feature `fast-puller`): the `FastDownPuller` +//! type and its `FastDownPullerOptions`, plus `build_client` +//! which constructs a correctly-configured `SmartRedirectClient`. +//! * [`getifaddrs`] (feature `getifaddrs`): enumerates the machine's non-virtual +//! local IP addresses for multi-interface download setups. + #[cfg(feature = "fast-puller")] #[cfg(not(target_family = "wasm"))] pub mod fast_puller; diff --git a/crates/fast-pull/README.md b/crates/fast-pull/README.md index 032e155..f514828 100644 --- a/crates/fast-pull/README.md +++ b/crates/fast-pull/README.md @@ -6,20 +6,94 @@ [![Documentation](https://docs.rs/fast-pull/badge.svg)](https://docs.rs/fast-pull) [![License](https://img.shields.io/crates/l/fast-pull.svg)](https://github.com/fast-down/core/blob/main/LICENSE) -`fast-pull` **Fastest** concurrent pull/push engine for data streaming! +`fast-pull` is a low-level concurrent **pull/push streaming engine** for moving +byte ranges from any source to any sink. **[Official Website (Simplified Chinese)](https://fd.s121.top/)** ## Features -1. **⚡️ Fastest Download**\ - We created [fast-steal](https://github.com/fast-down/fast-steal) With optimized Work Stealing, **1.43 x faster** than - NDM. -2. **🔄 File consistency**\ - Switching Wi-Fi, Turn Off Wi-Fi, Switch proxies. **We guarantee the consistency**. -3. **⛓️‍💥 Resuming Downloads**\ - You can **interrupt** at any time, and **resume downloading** after. -4. **⛓️‍💥 Incremental Downloads**\ - 1000 more lines server logs? Don't worry, we **only download new lines**. -5. **💰 Free and open-source**\ - The code stays free and open-source. Thanks to [share121](https://github.com/share121), [Cyan](https://github.com/CyanChanges) and other fast-down contributors. +1. **⚡️ Concurrent pull/push** + Built on [`fast-steal`](https://github.com/fast-down/fast-steal) with optimized + work-stealing across worker tasks, plus a single-threaded sequential path + (`download_single`). +2. **🔌 `Puller` / `Pusher` abstractions** + A download is just a `Puller` (source) feeding a `Pusher` (sink). Bring your + own, or use the built-ins. `Puller` must be `Clone` so work can be stolen and + retried; `Pusher` reports partial failures so the engine can retry them. +3. **💾 Multiple write paths** (feature-gated) + - `file` — `StdFilePusher` (raw `std::fs::File` random-access writes) and + `MmapFilePusher` (memory-mapped zero-copy writes), plus the ready-made + `CacheFilePusher` stack. + - `mem` — `MemPusher`, an in-memory sink backed by a shared `Vec`. +4. **🧩 Out-of-order & buffered writes** + Cache decorators `CacheDirectPusher`, `CacheMergePusher`, and + `CacheSeqPusher` absorb out-of-order chunks (keyed by `range.start`) and + flush runs once a watermark is reached; `BufWriterPusher` batches contiguous + writes like `std::io::BufWriter`. +5. **📈 Progress & cancellation** + Streaming `Event`s (pull/push progress, errors, completion) are delivered on + `DownloadResult::event_chain`, and a session is cancelled by + `DownloadResult::abort` or simply dropping the last handle clone. +6. **🧪 Testing-friendly** + `MockPuller` + `build_mock_data` give you a deterministic in-memory source for + tests — no network or disk required. + +## Usage + +```rust +use std::sync::{Arc, Mutex}; + +use bytes::Bytes; +use fast_pull::{ + mock::{build_mock_data, MockPuller}, + single::{download_single, DownloadOptions}, + ProgressEntry, Pusher, +}; + +/// A minimal in-memory [`Pusher`] so this example compiles with **no** optional +/// features. In real code, prefer `fast_pull::mem::MemPusher` (feature `mem`) or +/// a file pusher (feature `file`). +#[derive(Clone, Default)] +struct VecPusher { + data: Arc>>, +} + +impl Pusher for VecPusher { + type Error = std::convert::Infallible; + fn push(&mut self, range: &ProgressEntry, content: Bytes) -> Result<(), (Self::Error, Bytes)> { + let mut g = self.data.lock().unwrap(); + if g.len() < range.end as usize { + g.resize(range.end as usize, 0); + } + g[range.start as usize..range.end as usize].copy_from_slice(&content); + Ok(()) + } +} + +#[tokio::main] +async fn main() { + let expected = build_mock_data(1024); + let puller = MockPuller::new(&expected); + let pusher = VecPusher::default(); + let out = pusher.data.clone(); + + let result = download_single( + puller, + pusher, + DownloadOptions { + retry_gap: std::time::Duration::from_secs(1), + push_queue_cap: 16, + }, + ); + result.join().await.unwrap(); + + assert_eq!(&*out.lock().unwrap(), &expected); +} +``` + +## License + +Licensed under the same terms as the rest of the `fast-down` workspace. Thanks to +[share121](https://github.com/share121), [Cyan](https://github.com/CyanChanges) +and other `fast-down` contributors. diff --git a/crates/fast-pull/src/base/event.rs b/crates/fast-pull/src/base/event.rs index 891760d..c7d0f04 100644 --- a/crates/fast-pull/src/base/event.rs +++ b/crates/fast-pull/src/base/event.rs @@ -1,9 +1,12 @@ +//! Download lifecycle events emitted on the +//! [`event_chain`](crate::DownloadResult::event_chain). + use crate::ProgressEntry; /// Numeric identifier assigned to each worker thread/task. pub type WorkerId = usize; -/// Events emitted during a download session, received via [`DownloadResultInner::event_chain`](crate::DownloadResultInner::event_chain). +/// Events emitted during a download session, received via [`DownloadResult::event_chain`](crate::DownloadResult::event_chain). /// /// Each variant records a state change: pulling, pushing, progress, errors, or completion. #[derive(Debug)] diff --git a/crates/fast-pull/src/base/invert.rs b/crates/fast-pull/src/base/invert.rs index 171b554..fe9f860 100644 --- a/crates/fast-pull/src/base/invert.rs +++ b/crates/fast-pull/src/base/invert.rs @@ -1,3 +1,6 @@ +//! Iterator and helper for computing the *gaps* (not-yet-downloaded ranges) +//! from a set of [`ProgressEntry`](crate::ProgressEntry)s. + use crate::ProgressEntry; /// Iterator that yields the *gaps* (non-downloaded ranges) from a list of [`ProgressEntry`]s. @@ -5,9 +8,13 @@ use crate::ProgressEntry; /// Entries shorter than `window` are merged into adjacent gaps to reduce fragmentation. #[derive(Debug)] pub struct InvertIter> { + /// Iterator over the already-downloaded (sorted) ranges. iter: I, + /// End offset of the last range consumed from `iter`. prev_end: u64, + /// Total size of the source. total_size: u64, + /// Merge entries shorter than this into the surrounding gap. window: u64, } diff --git a/crates/fast-pull/src/base/merge.rs b/crates/fast-pull/src/base/merge.rs index 9fe6e61..18edd8b 100644 --- a/crates/fast-pull/src/base/merge.rs +++ b/crates/fast-pull/src/base/merge.rs @@ -1,9 +1,13 @@ +//! Merging of [`ProgressEntry`](crate::ProgressEntry) ranges into a sorted list. + use crate::ProgressEntry; /// Trait for merging a new [`ProgressEntry`] into a sorted list of existing entries. /// /// Used to consolidate downloaded ranges and remove redundant gaps. pub trait Merge { + /// Merge `new` into the existing (sorted) progress list, coalescing overlaps + /// so the list stays sorted and gap-free where ranges touch. fn merge_progress(&mut self, new: ProgressEntry); } diff --git a/crates/fast-pull/src/base/mod.rs b/crates/fast-pull/src/base/mod.rs index a47f46f..8d90063 100644 --- a/crates/fast-pull/src/base/mod.rs +++ b/crates/fast-pull/src/base/mod.rs @@ -1,3 +1,11 @@ +//! Core abstractions shared by every download engine in `fast-pull`. +//! +//! This module defines the two central traits — [`Puller`](crate::Puller) and +//! [`Pusher`](crate::Pusher) — plus the supporting types used to describe +//! progress and events: [`ProgressEntry`](crate::ProgressEntry), +//! [`Event`](crate::Event), [`WorkerId`](crate::WorkerId), and helpers for +//! merging and inverting progress ranges. + mod event; mod invert; mod merge; diff --git a/crates/fast-pull/src/base/progress.rs b/crates/fast-pull/src/base/progress.rs index 5894189..1a88d46 100644 --- a/crates/fast-pull/src/base/progress.rs +++ b/crates/fast-pull/src/base/progress.rs @@ -1,3 +1,5 @@ +//! Progress range type and total-size computation. + use core::ops::Range; /// A byte-range representing downloaded or to-be-downloaded progress. @@ -7,6 +9,7 @@ pub type ProgressEntry = Range; /// Trait for computing the total size from one or more [`ProgressEntry`] values. pub trait Total { + /// Total number of bytes represented by this progress value. fn total(&self) -> u64; } diff --git a/crates/fast-pull/src/base/puller.rs b/crates/fast-pull/src/base/puller.rs index d37b641..524c2f1 100644 --- a/crates/fast-pull/src/base/puller.rs +++ b/crates/fast-pull/src/base/puller.rs @@ -1,3 +1,5 @@ +//! The [`Puller`](crate::Puller) trait: an abstraction over a chunked data source. + use crate::ProgressEntry; use bytes::Bytes; use core::time::Duration; @@ -25,6 +27,12 @@ pub type PullResult = Result)>; /// specific byte range. Cloning is required for retry and work-stealing scenarios. pub trait Puller: Send + Sync + Clone + 'static { type Error: PullerError; + /// Pull a (sub)range of the source as a stream of byte chunks. + /// + /// Passing `None` for `range` requests the entire source. The returned + /// [`PullStream`] yields [`Bytes`] chunks; each error carries an optional + /// retry delay that the engine honors via its retry backoff. Implementors + /// must be `Clone` so workers can be spawned and work can be stolen/retried. fn pull( &mut self, range: Option<&ProgressEntry>, @@ -33,6 +41,11 @@ pub trait Puller: Send + Sync + Clone + 'static { /// Extension trait for pull errors, distinguishing recoverable from irrecoverable failures. pub trait PullerError: std::error::Error + Send + Sync + Unpin + 'static { + /// Whether an error is fatal and must **not** be retried. + /// + /// The default (`false`) means the error is recoverable and the engine will + /// retry after the configured backoff. Return `true` to stop retrying and + /// abort the affected worker. fn is_irrecoverable(&self) -> bool { false } diff --git a/crates/fast-pull/src/base/pusher.rs b/crates/fast-pull/src/base/pusher.rs index 9ad0378..fdcc3a0 100644 --- a/crates/fast-pull/src/base/pusher.rs +++ b/crates/fast-pull/src/base/pusher.rs @@ -1,3 +1,5 @@ +//! The [`Pusher`](crate::Pusher) trait: an abstraction over a chunked data sink. + use crate::ProgressEntry; use bytes::Bytes; @@ -14,8 +16,18 @@ pub type ProgressListener = Box; /// The pusher writes data to its destination and can optionally flush. pub trait Pusher: Send + 'static { type Error: std::error::Error + Send + Sync + Unpin + 'static; + /// Write `content` covering the given `range` to the destination. + /// + /// On success returns `Ok(())`. On failure returns `Err((error, bytes))` + /// where `bytes` is the (possibly partial) payload that was **not** written, + /// so the engine can retry it. Implementors should keep already-written + /// bytes internally on failure rather than dropping them. #[allow(clippy::missing_errors_doc)] fn push(&mut self, range: &ProgressEntry, content: Bytes) -> Result<(), (Self::Error, Bytes)>; + /// Flush any buffered data to the destination. + /// + /// The default implementation is a no-op. File-backed pushers use this to + /// issue `fsync` / `flush` on the underlying file. #[allow(clippy::missing_errors_doc)] fn flush(&mut self) -> Result<(), Self::Error> { Ok(()) @@ -41,6 +53,7 @@ impl std::error::Error for Box {} /// Useful for FFI boundaries or heterogeneous collections of pushers. #[allow(missing_debug_implementations)] pub struct BoxPusher { + /// The boxed, type-erased inner pusher. pub pusher: Box>>, } impl Pusher for BoxPusher { diff --git a/crates/fast-pull/src/cache/buf_writer.rs b/crates/fast-pull/src/cache/buf_writer.rs index 8337fe3..5e32db9 100644 --- a/crates/fast-pull/src/cache/buf_writer.rs +++ b/crates/fast-pull/src/cache/buf_writer.rs @@ -1,3 +1,6 @@ +//! A `std::io::BufWriter`-style contiguous write buffer for +//! [`Pusher`](crate::Pusher)s. + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::{Bytes, BytesMut}; diff --git a/crates/fast-pull/src/cache/direct.rs b/crates/fast-pull/src/cache/direct.rs index 702ceb7..9c287ab 100644 --- a/crates/fast-pull/src/cache/direct.rs +++ b/crates/fast-pull/src/cache/direct.rs @@ -1,3 +1,5 @@ +//! Pusher cache that flushes contiguous runs without byte merging. + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::Bytes; use std::collections::BTreeMap; @@ -18,6 +20,10 @@ pub struct CacheDirectPusher

{ } impl CacheDirectPusher

{ + /// Wrap `inner` with the given `high_watermark` / `low_watermark` (in bytes). + /// + /// Eviction to the inner pusher triggers once the buffered size reaches + /// `high_watermark`, and stops once it falls back to `low_watermark`. pub const fn new(inner: P, high_watermark: usize, low_watermark: usize) -> Self { Self { inner, diff --git a/crates/fast-pull/src/cache/merge.rs b/crates/fast-pull/src/cache/merge.rs index 64d3ea7..52c3a40 100644 --- a/crates/fast-pull/src/cache/merge.rs +++ b/crates/fast-pull/src/cache/merge.rs @@ -1,3 +1,5 @@ +//! Pusher cache that merges each flush run into a single contiguous buffer. + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::{Bytes, BytesMut}; use std::collections::{BTreeMap, btree_map::Entry}; @@ -18,6 +20,10 @@ pub struct CacheMergePusher

{ } impl CacheMergePusher

{ + /// Wrap `inner` with the given `high_watermark` / `low_watermark` (in bytes). + /// + /// Eviction to the inner pusher triggers once the buffered size reaches + /// `high_watermark`, and stops once it falls back to `low_watermark`. pub const fn new(inner: P, high_watermark: usize, low_watermark: usize) -> Self { Self { inner, diff --git a/crates/fast-pull/src/cache/mod.rs b/crates/fast-pull/src/cache/mod.rs index caa539c..5361d7c 100644 --- a/crates/fast-pull/src/cache/mod.rs +++ b/crates/fast-pull/src/cache/mod.rs @@ -1,3 +1,13 @@ +//! Pusher decorators that buffer out-of-order chunks before writing. +//! +//! Each decorator in this module wraps an inner [`Pusher`](crate::Pusher) and +//! absorbs out-of-order writes using a `BTreeMap` keyed by `range.start`, +//! flushing runs to the inner sink once a watermark is reached. Variants differ +//! in how runs are emitted: [`CacheDirectPusher`] flushes each chunk as-is, +//! [`CacheMergePusher`] coalesces each run into one contiguous buffer, and +//! [`CacheSeqPusher`] reorders runs into sequential order. [`BufWriterPusher`] +//! instead mimics `std::io::BufWriter` with a fixed-size contiguous buffer. + mod buf_writer; mod direct; mod merge; diff --git a/crates/fast-pull/src/cache/seq.rs b/crates/fast-pull/src/cache/seq.rs index eccb1b0..2cc71ee 100644 --- a/crates/fast-pull/src/cache/seq.rs +++ b/crates/fast-pull/src/cache/seq.rs @@ -1,3 +1,5 @@ +//! Pusher cache that reorders out-of-order chunks into sequential order. + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::Bytes; use std::collections::BTreeMap; @@ -17,6 +19,10 @@ pub struct CacheSeqPusher

{ } impl CacheSeqPusher

{ + /// Wrap `inner` with the given `high_watermark` / `low_watermark` (in bytes). + /// + /// Eviction to the inner pusher triggers once the buffered size reaches + /// `high_watermark`, and stops once it falls back to `low_watermark`. pub const fn new(inner: P, high_watermark: usize, low_watermark: usize) -> Self { Self { inner, diff --git a/crates/fast-pull/src/core/handle.rs b/crates/fast-pull/src/core/handle.rs index d7a2d54..ff6a8bc 100644 --- a/crates/fast-pull/src/core/handle.rs +++ b/crates/fast-pull/src/core/handle.rs @@ -1,3 +1,5 @@ +//! A shareable, multi-consumer handle to a tokio task. + use std::sync::Arc; use tokio::{ sync::watch, diff --git a/crates/fast-pull/src/core/mock.rs b/crates/fast-pull/src/core/mock.rs index cd9db51..8eb4bb8 100644 --- a/crates/fast-pull/src/core/mock.rs +++ b/crates/fast-pull/src/core/mock.rs @@ -1,3 +1,6 @@ +//! A deterministic in-memory [`Puller`](crate::Puller) for tests, plus a helper +//! to build mock payloads. + use crate::{ProgressEntry, PullResult, PullStream, Puller}; use futures::stream; use std::{sync::Arc, vec::Vec}; diff --git a/crates/fast-pull/src/core/mod.rs b/crates/fast-pull/src/core/mod.rs index cc2cbaa..850dd4d 100644 --- a/crates/fast-pull/src/core/mod.rs +++ b/crates/fast-pull/src/core/mod.rs @@ -1,3 +1,12 @@ +//! Top-level download orchestration: session handle plus single- and +//! multi-threaded entry points. +//! +//! [`download_single`](crate::single::download_single) runs a sequential pull, +//! while [`download_multi`](crate::multi::download_multi) splits the work across +//! concurrent workers with work-stealing. Both return a [`DownloadResult`], a +//! cheaply cloneable handle that keeps the session alive until the last clone is +//! dropped (or [`DownloadResult::abort`] is called). + use crate::{Event, handle::SharedHandle}; use core::sync::atomic::{AtomicBool, Ordering}; use crossfire::{MAsyncRx, mpmc}; @@ -134,11 +143,11 @@ where /// /// Cheaply cloneable shared handle. The underlying download keeps running as /// long as **any** clone is alive, and is cancelled only once the last clone is -/// dropped. An explicit [`DownloadResultInner::abort`] cancels immediately. +/// dropped. An explicit [`abort`](Self::abort) cancels immediately. /// -/// `DownloadResult` derefs to [`DownloadResultInner`], so all session methods -/// (`join`, `abort`, `set_threads`, `is_aborted`) and the `event_chain` field -/// are reachable directly on the handle. +/// `DownloadResult` derefs to `DownloadResultInner`, so all session methods +/// (`join`, `abort`, `set_threads`, `is_aborted`) and the [`event_chain`](Self::event_chain) +/// method are reachable directly on the handle. #[derive(Debug)] pub struct DownloadResult where @@ -168,6 +177,12 @@ where PullError: Send + Unpin + 'static, PushError: Send + Unpin + 'static, { + /// Construct a [`DownloadResult`] from the raw session pieces. + /// + /// This is an internal constructor used by + /// [`download_single`](crate::single::download_single) and + /// [`download_multi`](crate::multi::download_multi); prefer those entry + /// points instead of calling this directly. pub fn new( event_chain: MAsyncRx>>, handle: JoinHandle<()>, @@ -188,6 +203,10 @@ where } } + /// Access the stream of [`Event`]s emitted during the session. + /// + /// The receiver closes once the last clone of this handle is dropped or the + /// session is aborted, so draining it is a natural way to observe progress. #[must_use] pub fn event_chain(&self) -> &MAsyncRx>> { &self.inner.event_chain @@ -208,10 +227,15 @@ where self.inner.abort(); } + /// Adjust the worker thread count and minimum chunk size of a running + /// multi-threaded session. + /// + /// No-op for single-threaded sessions, which have no task queue. pub fn set_threads(&self, threads: usize, min_chunk_size: u64) { self.inner.set_threads(threads, min_chunk_size); } + /// Whether the session has been (or is being) cancelled. #[must_use] pub fn is_aborted(&self) -> bool { self.inner.is_aborted() diff --git a/crates/fast-pull/src/core/multi.rs b/crates/fast-pull/src/core/multi.rs index 2510017..ffb2320 100644 --- a/crates/fast-pull/src/core/multi.rs +++ b/crates/fast-pull/src/core/multi.rs @@ -1,3 +1,5 @@ +//! Multi-threaded concurrent download with work-stealing. + use crate::{DownloadResult, Event, ProgressEntry, Puller, PullerError, Pusher, WorkerId}; use bytes::Bytes; use core::{ diff --git a/crates/fast-pull/src/core/single.rs b/crates/fast-pull/src/core/single.rs index da3b16b..9acb512 100644 --- a/crates/fast-pull/src/core/single.rs +++ b/crates/fast-pull/src/core/single.rs @@ -1,3 +1,5 @@ +//! Single-threaded sequential download. + use crate::{ DownloadResult, Event, ProgressEntry, Puller, PullerError, Pusher, multi::TokioExecutor, }; diff --git a/crates/fast-pull/src/file/cache_std.rs b/crates/fast-pull/src/file/cache_std.rs index 7a97a6e..fb88c21 100644 --- a/crates/fast-pull/src/file/cache_std.rs +++ b/crates/fast-pull/src/file/cache_std.rs @@ -1,3 +1,5 @@ +//! High-level file pusher combining caching, buffering, and standard file I/O. + use crate::{ BufWriterPusher, CacheSeqPusher, ProgressEntry, ProgressListener, Pusher, file::StdFilePusher, }; diff --git a/crates/fast-pull/src/file/mmap.rs b/crates/fast-pull/src/file/mmap.rs index ce7798e..906a22b 100644 --- a/crates/fast-pull/src/file/mmap.rs +++ b/crates/fast-pull/src/file/mmap.rs @@ -1,3 +1,5 @@ +//! Memory-mapped file pusher (feature `file`). + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::Bytes; use memmap2::MmapMut; diff --git a/crates/fast-pull/src/file/mod.rs b/crates/fast-pull/src/file/mod.rs index 650fbdf..85d89e6 100644 --- a/crates/fast-pull/src/file/mod.rs +++ b/crates/fast-pull/src/file/mod.rs @@ -1,3 +1,10 @@ +//! File-backed [`Pusher`](crate::Pusher) implementations (feature `file`). +//! +//! Provides [`StdFilePusher`] (raw `std::fs::File` random-access writes), +//! [`MmapFilePusher`] (memory-mapped zero-copy writes), and [`CacheFilePusher`] +//! (a ready-made `CacheSeqPusher>` stack). +//! Enabled by the `file` feature. + mod cache_std; mod mmap; mod std; diff --git a/crates/fast-pull/src/file/std.rs b/crates/fast-pull/src/file/std.rs index 945af1d..b251b41 100644 --- a/crates/fast-pull/src/file/std.rs +++ b/crates/fast-pull/src/file/std.rs @@ -1,3 +1,5 @@ +//! Raw `std::fs::File` random-access pusher (feature `file`). + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::Bytes; use std::{ diff --git a/crates/fast-pull/src/mem/mod.rs b/crates/fast-pull/src/mem/mod.rs index 0ca5edd..c8437d4 100644 --- a/crates/fast-pull/src/mem/mod.rs +++ b/crates/fast-pull/src/mem/mod.rs @@ -1,3 +1,5 @@ +//! In-memory [`Pusher`](crate::Pusher) implementation (feature `mem`). + mod pusher; pub use pusher::*; diff --git a/crates/fast-pull/src/mem/pusher.rs b/crates/fast-pull/src/mem/pusher.rs index 5ed81ee..ae3b786 100644 --- a/crates/fast-pull/src/mem/pusher.rs +++ b/crates/fast-pull/src/mem/pusher.rs @@ -1,3 +1,5 @@ +//! In-memory pusher backed by a shared `Vec` (feature `mem`). + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::Bytes; use parking_lot::Mutex; diff --git a/crates/fast-steal/src/executor.rs b/crates/fast-steal/src/executor.rs index 29bd4a7..6241a6a 100644 --- a/crates/fast-steal/src/executor.rs +++ b/crates/fast-steal/src/executor.rs @@ -1,3 +1,8 @@ +//! Traits that connect a [`TaskQueue`](crate::TaskQueue) to an async runtime. +//! +//! Implement [`Executor`] to spawn work onto your runtime, and [`Handle`] to give the +//! queue a way to identify and abort the tasks it spawned. + use crate::{Task, TaskQueue}; /// User-defined executor that runs tasks on a [`TaskQueue`]. diff --git a/crates/fast-steal/src/lib.rs b/crates/fast-steal/src/lib.rs index 312a439..08db8a1 100644 --- a/crates/fast-steal/src/lib.rs +++ b/crates/fast-steal/src/lib.rs @@ -1,4 +1,17 @@ #![no_std] +//! `fast-steal` provides a `no_std`-compatible building block for work-stealing +//! schedulers. +//! +//! The crate is built around three public types: +//! - [`Task`] — a lock-free, cancellable unit of work tracking a `start..end` range. +//! - [`TaskQueue`] — a concurrent queue that hands out work and steals sub-ranges +//! from busy workers. +//! - [`Executor`] / [`Handle`] — traits you implement to plug the queue into any +//! async runtime. +//! +//! Only [`alloc`](https://doc.rust-lang.org/alloc/) is required by the core paths; +//! `std` is used exclusively in tests and doctests. For a complete runnable example, +//! see the crate-level documentation (the included README) below. #![doc = include_str!("../README.md")] mod executor; diff --git a/crates/fast-steal/src/task.rs b/crates/fast-steal/src/task.rs index bf47958..f2a24e5 100644 --- a/crates/fast-steal/src/task.rs +++ b/crates/fast-steal/src/task.rs @@ -1,3 +1,10 @@ +//! Cancellable, lock-free units of work. +//! +//! A [`Task`] stores its remaining work as a single atomic `u128` packing the +//! `start` (high 64 bits) and `end` (low 64 bits) bounds, which lets multiple +//! worker threads advance the same task without locking. Because the crate is +//! `no_std`, only `alloc` is required for the reference-counted state. + extern crate alloc; use alloc::sync::{Arc, Weak}; use core::{fmt, ops::Range, sync::atomic::Ordering}; @@ -8,8 +15,15 @@ use portable_atomic::AtomicU128; /// The range is stored as a single atomic `u128`, allowing lock-free reads and /// fine-grained progress updates. Multiple workers can safely steal sub-ranges /// from the same task via [`split_two`](Task::split_two). +/// +/// Two `Task`s are equal iff they point to the same underlying state (see the +/// `PartialEq` impl, which uses `Arc::ptr_eq`). #[derive(Debug, Clone)] pub struct Task { + /// Atomic state packing `start` (high 64 bits) and `end` (low 64 bits). + /// + /// Prefer the safe accessors ([`Task::start`], [`Task::end`], [`Task::get`], + /// [`Task::safe_add_start`]); this field is exposed for advanced use. pub state: Arc, } /// A weak reference to a [`Task`], obtained via [`Task::downgrade`]. @@ -18,18 +32,25 @@ pub struct Task { /// to attempt to obtain a strong [`Task`] reference. #[derive(Debug, Clone)] pub struct WeakTask { + /// Weak reference to the underlying atomic state of the originating [`Task`]. pub state: Weak, } impl WeakTask { + /// Attempts to upgrade to a strong [`Task`]. + /// + /// Returns `None` if all strong references to the underlying state have already + /// been dropped. #[must_use] pub fn upgrade(&self) -> Option { self.state.upgrade().map(|state| Task { state }) } + /// Returns the number of strong [`Task`] references to the underlying state. #[must_use] pub fn strong_count(&self) -> usize { self.state.strong_count() } + /// Returns the number of weak [`WeakTask`] references to the underlying state. #[must_use] pub fn weak_count(&self) -> usize { self.state.weak_count() @@ -71,18 +92,27 @@ impl Task { state: Arc::new(AtomicU128::new(Self::pack(range))), } } + /// Returns the current `start..end` range, loaded atomically with `Acquire` + /// ordering. #[must_use] pub fn get(&self) -> Range { let state = self.state.load(Ordering::Acquire); Self::unpack(state) } + /// Returns the current start of the work range (the high 64 bits of the atomic + /// state). #[must_use] pub fn start(&self) -> u64 { (self.state.load(Ordering::Acquire) >> 64) as u64 } + /// Atomically advances `start` to `min(start + bias, end)`, but only if that + /// makes forward progress; then returns the slice that was claimed. + /// /// # Errors - /// Returns [`RangeError`] when `start` + `bias` <= `old_start` - /// Otherwise returns `old_start..new_start.min(end)` + /// Returns [`RangeError`] when `start + bias` would not exceed the current + /// `start` (no progress, a non-positive bias, or `u64` overflow), or when the + /// task range invariant `start <= end` is violated. On success returns the + /// claimed `old_start..new_start` sub-range. pub fn safe_add_start(&self, start: u64, bias: u64) -> Result, RangeError> { let new_start = start.checked_add(bias).ok_or(RangeError)?; let mut old_state = self.state.load(Ordering::Acquire); @@ -106,6 +136,8 @@ impl Task { } } } + /// Returns the current end of the work range (the low 64 bits of the atomic + /// state). #[must_use] pub fn end(&self) -> u64 { let state = self.state.load(Ordering::Acquire); @@ -113,6 +145,7 @@ impl Task { let end = state as u64; end } + /// Returns `end - start` (saturating), i.e. how much work is left. #[must_use] pub fn remain(&self) -> u64 { let range = self.get(); @@ -144,6 +177,10 @@ impl Task { } } } + /// Atomically claims and returns the entire remaining range `start..end`, + /// emptying this task (sets `start = end`). + /// + /// Returns `None` if the task is already empty. #[must_use] pub fn take(&self) -> Option> { let mut old_state = self.state.load(Ordering::Acquire); @@ -164,21 +201,28 @@ impl Task { } } } + /// Creates a [`WeakTask`] that does not keep the task's state alive. #[must_use] pub fn downgrade(&self) -> WeakTask { WeakTask { state: Arc::downgrade(&self.state), } } + /// Returns the number of strong ([`Task`]) references to this task's state. #[must_use] pub fn strong_count(&self) -> usize { Arc::strong_count(&self.state) } + /// Returns the number of weak ([`WeakTask`]) references to this task's state. #[must_use] pub fn weak_count(&self) -> usize { Arc::weak_count(&self.state) } } +/// Creates a [`Task`] from a `start..end` range. +/// +/// # Panics +/// Panics (via [`Task::new`]) if `range.start > range.end`. impl From> for Task { fn from(value: Range) -> Self { Self::new(value) diff --git a/crates/fast-steal/src/task_queue.rs b/crates/fast-steal/src/task_queue.rs index c4704c7..28a96fb 100644 --- a/crates/fast-steal/src/task_queue.rs +++ b/crates/fast-steal/src/task_queue.rs @@ -1,3 +1,10 @@ +//! A concurrent work-stealing queue. +//! +//! [`TaskQueue`] holds pending and running [`Task`](crate::Task)s and lets worker +//! threads pull fresh work or steal a sub-range from a busy peer via +//! [`steal`](TaskQueue::steal). The number of running workers can be adjusted at +//! runtime with [`set_threads`](TaskQueue::set_threads). + extern crate alloc; use crate::{Executor, Handle, Task, WeakTask}; use alloc::{collections::vec_deque::VecDeque, sync::Arc, vec::Vec}; @@ -26,6 +33,8 @@ struct TaskQueueInner { waiting: VecDeque, } impl TaskQueue { + /// Creates a queue from an iterator of `start..end` ranges, each wrapped in its + /// own [`Task`]. pub fn new(tasks: impl Iterator>) -> Self { let waiting: VecDeque<_> = tasks.map(Task::from).collect(); Self { @@ -35,10 +44,25 @@ impl TaskQueue { })), } } + /// Appends a [`Task`] to the waiting queue so a future + /// [`steal`](TaskQueue::steal) or [`set_threads`](TaskQueue::set_threads) can + /// pick it up. pub fn add(&self, task: Task) { let mut guard = self.inner.lock(); guard.waiting.push_back(task); } + /// Tries to refill `task` with more work for the worker identified by `id`. + /// + /// The caller must pass its own currently-held [`Task`] plus `id` (compared via + /// [`Handle::is_self`](crate::Handle::is_self)). The function first hands out a + /// pending task from the waiting queue; if none is available it steals a half + /// range from the busiest running task via [`Task::split_two`](crate::Task::split_two) + /// (when at least `min_chunk_size * 2` work remains), or, if `max_speculative > 1` + /// and the stolen task has few enough strong references, shares that same task + /// speculatively. + /// + /// Returns `true` if `task` was refilled, or `false` if the worker is not + /// registered or no work could be found. pub fn steal( &self, id: &H::Id, @@ -143,6 +167,8 @@ impl TaskQueue { } Some(()) } + /// Provides mutable access to the handles of all running tasks, e.g. to abort + /// or inspect them. pub fn handles(&self, f: F) -> R where F: FnOnce(&mut dyn Iterator) -> R, @@ -153,6 +179,8 @@ impl TaskQueue { f(&mut iter) } + /// Aborts every running task equal to `task` that does not belong to the + /// worker `id`, reclaiming it back into the waiting queue. pub fn cancel_task(&self, task: &Task, id: &H::Id) { let mut guard = self.inner.lock(); for (weak, handle) in &mut guard.running { From 0c30ac1d08df536dfd503d436d5c447d4146d429 Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 18:06:36 +0800 Subject: [PATCH 20/26] =?UTF-8?q?fix(fast-pull):=20=E4=BF=AE=E5=A4=8D=20me?= =?UTF-8?q?rge=5Fprogress=20=E5=9C=A8=E5=A4=84=E7=90=86=20`1..1`=20`3..1`?= =?UTF-8?q?=20=E7=AD=89=E9=94=99=E8=AF=AF=E8=8C=83=E5=9B=B4=E6=97=B6?= =?UTF-8?q?=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-pull/src/base/merge.rs | 47 ++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/crates/fast-pull/src/base/merge.rs b/crates/fast-pull/src/base/merge.rs index 18edd8b..a2f5c57 100644 --- a/crates/fast-pull/src/base/merge.rs +++ b/crates/fast-pull/src/base/merge.rs @@ -13,6 +13,9 @@ pub trait Merge { impl Merge for Vec { fn merge_progress(&mut self, new: ProgressEntry) { + if new.start >= new.end { + return; + } let i = self.partition_point(|x| x.end < new.start); if i == self.len() { self.push(new); @@ -76,4 +79,48 @@ mod tests { v.merge_progress(0..90); assert_eq!(v, vec![0..90]); } + + #[test] + fn test_merge_empty_range_is_dropped() { + // An empty range contained inside an existing entry must be a no-op. + let mut v = vec![1..5, 10..20]; + v.merge_progress(3..3); + assert_eq!(v, vec![1..5, 10..20]); + + // An empty range landing in a gap or at the end must NOT be inserted + // as a degenerate entry. + v.merge_progress(7..7); + assert_eq!(v, vec![1..5, 10..20]); + v.merge_progress(25..25); + assert_eq!(v, vec![1..5, 10..20]); + } + + #[test] + fn test_merge_before_front_with_gap() { + #![allow(clippy::single_range_in_vec_init)] + // `new` sits entirely before `self[0]`, leaving a gap -> inserted at front. + let mut v = vec![5..10]; + v.merge_progress(1..3); + assert_eq!(v, vec![1..3, 5..10]); + } + + #[test] + fn test_merge_extends_before_first_and_spans_gaps() { + #![allow(clippy::single_range_in_vec_init)] + // `new` starts before the first entry and spans across multiple gaps, + // absorbing every overlapping/touching entry into a single coalesced range. + let mut v = vec![1..5, 8..10, 12..15]; + v.merge_progress(0..13); + assert_eq!(v, vec![0..15]); + } + + #[test] + fn test_merge_reversed_range_is_dropped() { + #![allow(clippy::single_range_in_vec_init)] + // A reversed range (`start > end`) is invalid and must not enter the list. + let mut v = vec![10..20]; + #[allow(clippy::reversed_empty_ranges)] + v.merge_progress(30..5); + assert_eq!(v, vec![10..20]); + } } From 467fea56daef717805bd2edfe95108e54ef1ee3b Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 18:44:20 +0800 Subject: [PATCH 21/26] =?UTF-8?q?fix(fast-down-api):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- .../{tests.bak => tests}/resume.rs | 65 +++++++++---------- 1 file changed, 32 insertions(+), 33 deletions(-) rename crates/fast-down-api/{tests.bak => tests}/resume.rs (94%) diff --git a/crates/fast-down-api/tests.bak/resume.rs b/crates/fast-down-api/tests/resume.rs similarity index 94% rename from crates/fast-down-api/tests.bak/resume.rs rename to crates/fast-down-api/tests/resume.rs index b7d1de1..b03142c 100644 --- a/crates/fast-down-api/tests.bak/resume.rs +++ b/crates/fast-down-api/tests/resume.rs @@ -22,7 +22,7 @@ use std::time::Duration; use bytes::Bytes; use fast_down_api::{ - DownloadHandle, Event, PartialConfig, ResumeError, Rx, WriteMethod, create_cancellation_token, + DownloadHandle, Event, PartialConfig, Rx, StateError, WriteMethod, create_cancellation_token, create_channel, }; use futures::StreamExt; @@ -227,7 +227,7 @@ fn make_config(save_dir: &Path) -> PartialConfig { save_dir: Some(save_dir.to_path_buf()), filename: Some("out.bin".to_string()), parse_filename: Some(false), - overwrite: Some(false), + overwrite: Some(true), write_method: Some(WriteMethod::Mmap), min_chunk_size: Some(1024 * 1024), threads: Some(1), @@ -255,8 +255,7 @@ async fn partial_download_via_cancel( let cfg = make_config(save_dir); let (tx, rx) = create_channel(); let _handle = - DownloadHandle::download(Url::parse(url).expect("valid url"), cfg, tx, cancel.clone()) - .expect("spawn download"); + DownloadHandle::download(Url::parse(url).expect("valid url"), cfg, tx, cancel.clone()); let mut events = Vec::new(); let mut started = false; @@ -311,8 +310,7 @@ async fn test_resume_success() { cfg, tx, resume_cancel, - ) - .expect("spawn resume"); + ); let events = drain(rx).await; let (_, size) = events @@ -364,8 +362,7 @@ async fn test_file_changed_download_falls_back() { let (tx, rx) = create_channel(); let dl_cancel = create_cancellation_token(); let _handle = - DownloadHandle::download(Url::parse(&url).expect("valid url"), cfg, tx, dl_cancel) - .expect("spawn download"); + DownloadHandle::download(Url::parse(&url).expect("valid url"), cfg, tx, dl_cancel); let events = drain(rx).await; assert!( @@ -390,7 +387,7 @@ async fn test_file_changed_download_falls_back() { } /// Case 2 (resume branch): a stale `.fd` (remote file changed) makes `resume()` -/// report `ResumeError::FileChanged` and keep the partial files untouched. +/// report `StateError::FileChanged` and keep the partial files untouched. #[tokio::test] async fn test_file_changed_resume_reports_error() { let dir = temp_dir("file_changed_resume"); @@ -412,20 +409,19 @@ async fn test_file_changed_resume_reports_error() { cfg, tx, resume_cancel, - ) - .expect("spawn resume"); + ); let events = drain(rx).await; let err = events .iter() .find_map(|e| match e { - Event::ResumeError(r) => Some(r.clone()), + Event::ResumeError(r) => Some(r), _ => None, }) .expect("expected Event::ResumeError"); assert!( - matches!(err, ResumeError::FileChanged { .. }), - "expected Event::ResumeError(ResumeError::FileChanged), got {err:?}" + matches!(err, StateError::FileChanged { .. }), + "expected Event::ResumeError(StateError::FileChanged), got {err:?}" ); assert!( @@ -441,7 +437,7 @@ async fn test_file_changed_resume_reports_error() { ); } -/// Case 3: `resume()` with no `.fd` state file reports `ResumeError::NoStateFile`. +/// Case 3: `resume()` with no `.fd` state file reports `StateError::Open`. #[tokio::test] async fn test_resume_no_state_file() { let dir = temp_dir("resume_no_state_file"); @@ -463,18 +459,20 @@ async fn test_resume_no_state_file() { cfg, tx, cancel, - ) - .expect("spawn resume"); + ); let events = drain(rx).await; let err = events .iter() .find_map(|e| match e { - Event::ResumeError(r) => Some(r.clone()), + Event::ResumeError(r) => Some(r), _ => None, }) .expect("expected Event::ResumeError"); - assert_eq!(err, ResumeError::NoStateFile); + assert!( + matches!(err, StateError::Open(_)), + "expected Event::ResumeError(StateError::Open) for missing .fd, got {err:?}" + ); assert!( !events.iter().any(|e| matches!(e, Event::Renamed(_))), "resume() must NOT rename when there is no .fd" @@ -482,16 +480,18 @@ async fn test_resume_no_state_file() { } /// Case 4: `resume()` against a server that does not support range requests -/// reports `ResumeError::NotResumable`. +/// reports `StateError::NotResumable`. #[tokio::test] async fn test_resume_not_resumable() { let dir = temp_dir("resume_not_resumable"); let (_server, url) = start_server(original_bytes(), "orig", "LM-A", false).await; - // `tmp_path` (.part) must exist so `force_resume` is engaged; the - // non-range server then reports `NotResumable`. + // A cancelled download leaves a valid `.fd` + `.part` for the non-range + // server; `resume()` must then report `NotResumable` (server can't range). + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + let final_path = dir.join("out.bin"); let part = final_path.with_added_extension("part"); - let _ = std::fs::File::create(&part).expect("create .part"); let cfg = make_config(&dir); let (tx, rx) = create_channel(); let cancel = create_cancellation_token(); @@ -501,18 +501,20 @@ async fn test_resume_not_resumable() { cfg, tx, cancel, - ) - .expect("spawn resume"); + ); let events = drain(rx).await; let err = events .iter() .find_map(|e| match e { - Event::ResumeError(r) => Some(r.clone()), + Event::ResumeError(r) => Some(r), _ => None, }) .expect("expected Event::ResumeError"); - assert_eq!(err, ResumeError::NotResumable); + assert!( + matches!(err, StateError::NotResumable(..)), + "expected Event::ResumeError(StateError::NotResumable), got {err:?}" + ); } /// Case 5 (explicit): the previous round's fix — a cancel must preserve `.part` @@ -546,8 +548,7 @@ async fn test_cancel_keeps_part_and_fd_then_resume() { cfg, tx, resume_cancel, - ) - .expect("spawn resume"); + ); let events = drain(rx).await; assert!( @@ -593,8 +594,7 @@ async fn test_resume_missing_tmp_path_falls_back_to_download() { cfg, tx, cancel, - ) - .expect("spawn resume"); + ); let events = drain(rx).await; assert!( @@ -632,8 +632,7 @@ async fn test_fresh_download_writes_full_file() { let cfg = make_config(&dir); let (tx, rx) = create_channel(); let cancel = create_cancellation_token(); - let _handle = DownloadHandle::download(Url::parse(&url).expect("valid url"), cfg, tx, cancel) - .expect("spawn download"); + let _handle = DownloadHandle::download(Url::parse(&url).expect("valid url"), cfg, tx, cancel); let events = drain(rx).await; assert!( From 3c92b9ade90ec1ff63dada5f9d4f01d2780dd9c1 Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 19:07:30 +0800 Subject: [PATCH 22/26] =?UTF-8?q?test:=20=E6=B7=BB=E5=8A=A0=E5=A4=A7?= =?UTF-8?q?=E9=87=8F=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/src/config.rs | 38 +++++ .../fast-down-api/src/utils/build_header.rs | 37 +++++ .../src/utils/filename_template.rs | 50 +++++++ .../fast-down/src/http/content_disposition.rs | 42 ++++++ crates/fast-down/src/http/manual_redirect.rs | 140 ++++++++++++++++++ crates/fast-down/src/proxy.rs | 40 +++++ crates/fast-down/src/url_info.rs | 46 ++++++ crates/fast-pull/src/base/invert.rs | 41 +++++ crates/fast-pull/src/base/merge.rs | 65 ++++++++ crates/fast-pull/src/base/progress.rs | 24 +++ crates/fast-pull/src/cache/merge.rs | 108 ++++++++++++++ crates/fast-pull/src/mem/pusher.rs | 54 +++++++ crates/fast-steal/src/task.rs | 61 ++++++++ 13 files changed, 746 insertions(+) diff --git a/crates/fast-down-api/src/config.rs b/crates/fast-down-api/src/config.rs index 0b16450..39f3345 100644 --- a/crates/fast-down-api/src/config.rs +++ b/crates/fast-down-api/src/config.rs @@ -183,3 +183,41 @@ impl PartialConfig { .merge_progress(progress); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn merge_progress_none_to_some() { + let mut c = PartialConfig::default(); + assert_eq!(c.downloaded_chunk, None); + c.merge_progress(1u64..5); + assert_eq!(c.downloaded_chunk, Some(vec![1u64..5])); + } + + #[test] + fn merge_progress_coalesces() { + let mut c = PartialConfig::default(); + c.merge_progress(1u64..5); + c.merge_progress(5u64..10); + c.merge_progress(10u64..20); + assert_eq!(c.downloaded_chunk, Some(vec![1u64..20])); + } + + #[test] + fn merge_progress_empty_is_noop() { + let mut c = PartialConfig::default(); + c.merge_progress(1u64..5); + c.merge_progress(3u64..3); + assert_eq!(c.downloaded_chunk, Some(vec![1u64..5])); + } + + #[test] + fn merge_progress_disjoint() { + let mut c = PartialConfig::default(); + c.merge_progress(1u64..5); + c.merge_progress(10u64..20); + assert_eq!(c.downloaded_chunk, Some(vec![1u64..5, 10u64..20])); + } +} diff --git a/crates/fast-down-api/src/utils/build_header.rs b/crates/fast-down-api/src/utils/build_header.rs index 30a5857..3b74eaf 100644 --- a/crates/fast-down-api/src/utils/build_header.rs +++ b/crates/fast-down-api/src/utils/build_header.rs @@ -10,3 +10,40 @@ pub fn build_header(headers: &HashMap) -> HeaderMap { } result } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn valid_headers_are_inserted() { + let mut map = HashMap::new(); + map.insert("User-Agent".to_string(), "test".to_string()); + map.insert("Accept".to_string(), "*/*".to_string()); + let h = build_header(&map); + assert_eq!(h.get("user-agent").unwrap(), "test"); + assert_eq!(h.get("accept").unwrap(), "*/*"); + } + + #[test] + fn invalid_name_is_skipped() { + let mut map = HashMap::new(); + // A space inside the header name makes it invalid. + map.insert("Invalid Name".to_string(), "x".to_string()); + assert!(build_header(&map).is_empty()); + } + + #[test] + fn invalid_value_is_skipped() { + let mut map = HashMap::new(); + // A newline in the value makes it invalid (control char rejected). + map.insert("X-Test".to_string(), "bad\nvalue".to_string()); + assert!(build_header(&map).is_empty()); + } + + #[test] + fn empty_map_is_empty() { + let map: HashMap = HashMap::new(); + assert!(build_header(&map).is_empty()); + } +} diff --git a/crates/fast-down-api/src/utils/filename_template.rs b/crates/fast-down-api/src/utils/filename_template.rs index 5c23186..1168a56 100644 --- a/crates/fast-down-api/src/utils/filename_template.rs +++ b/crates/fast-down-api/src/utils/filename_template.rs @@ -34,3 +34,53 @@ pub fn parse_filename_template(template: String, url: &Url, filename: &str) -> S .replace("{file_stem}", file_stem) .replace("{file_ext}", file_ext) } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use url::Url; + + #[test] + fn all_placeholders() { + let url = Url::parse("https://example.com/path/to/file.txt").unwrap(); + let t = "{host}/{parent_path}/{file_name}_{file_stem}{file_ext}"; + let out = parse_filename_template(t.to_string(), &url, "file.txt"); + assert!(out.starts_with("example.com")); + assert!(out.contains("path")); + assert!(out.contains("to")); + assert!(out.ends_with("file.txt_file.txt")); + } + + #[test] + fn no_placeholders_passthrough() { + let url = Url::parse("https://example.com/x").unwrap(); + assert_eq!(parse_filename_template("plain".to_string(), &url, "f.txt"), "plain"); + } + + #[test] + fn host_unknown_when_no_host() { + let url = Url::parse("file:///etc/hosts").unwrap(); + assert_eq!(parse_filename_template("{host}".to_string(), &url, "hosts"), "unknown"); + } + + #[test] + fn parent_path_root_when_no_dir() { + let url = Url::parse("https://example.com/file.txt").unwrap(); + assert_eq!(parse_filename_template("{parent_path}".to_string(), &url, "file.txt"), "."); + } + + #[test] + fn file_ext_includes_dot() { + let url = Url::parse("https://example.com/a/b.tar.gz").unwrap(); + let out = parse_filename_template("{file_stem}{file_ext}".to_string(), &url, "b.tar.gz"); + assert_eq!(out, "b.tar.gz"); + } + + #[test] + fn no_dot_file_has_empty_ext() { + let url = Url::parse("https://example.com/README").unwrap(); + let out = parse_filename_template("{file_stem}|{file_ext}".to_string(), &url, "README"); + assert_eq!(out, "README|"); + } +} diff --git a/crates/fast-down/src/http/content_disposition.rs b/crates/fast-down/src/http/content_disposition.rs index c6ac1ee..5da1a25 100644 --- a/crates/fast-down/src/http/content_disposition.rs +++ b/crates/fast-down/src/http/content_disposition.rs @@ -217,4 +217,46 @@ mod tests { let cd = ContentDisposition::parse(s); assert_eq!(cd.filename.unwrap(), ";\";;"); } + + #[test] + fn test_no_semicolon_is_none() { + assert_eq!(ContentDisposition::parse("attachment").filename, None); + } + + #[test] + fn test_empty_header() { + assert_eq!(ContentDisposition::parse("").filename, None); + assert_eq!(ContentDisposition::parse(" ").filename, None); + } + + #[test] + fn test_filename_star_non_utf8_ignored() { + // A non-UTF-8 charset is unsupported, so `filename*` is dropped. + let s = "attachment; filename*=ISO-8859-1''%A3.txt"; + assert_eq!(ContentDisposition::parse(s).filename, None); + } + + #[test] + fn test_filename_star_wins_over_filename() { + let s = r#"attachment; filename="old.txt"; filename*=UTF-8''%E6%B5%8B.txt"#; + assert_eq!(ContentDisposition::parse(s).filename.unwrap(), "测.txt"); + } + + #[test] + fn test_unquoted_token_stops_at_space() { + let s = "attachment; filename=foo bar.txt"; + assert_eq!(ContentDisposition::parse(s).filename.unwrap(), "foo"); + } + + #[test] + fn test_consecutive_semicolons() { + let s = "attachment;;; filename=foo.txt"; + assert_eq!(ContentDisposition::parse(s).filename.unwrap(), "foo.txt"); + } + + #[test] + fn test_quoted_with_escaped_quote() { + let s = r#"attachment; filename="a\"b.txt""#; + assert_eq!(ContentDisposition::parse(s).filename.unwrap(), "a\"b.txt"); + } } diff --git a/crates/fast-down/src/http/manual_redirect.rs b/crates/fast-down/src/http/manual_redirect.rs index 7d49a91..993aaed 100644 --- a/crates/fast-down/src/http/manual_redirect.rs +++ b/crates/fast-down/src/http/manual_redirect.rs @@ -122,3 +122,143 @@ pub fn compute_referer( Some(ReferrerPolicy::UnsafeUrl) => Some(referer_url(prev_url)), } } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use url::Url; + + fn u(s: &str) -> Url { + Url::parse(s).unwrap() + } + + #[test] + fn parse_all_tokens_case_insensitive() { + assert_eq!(ReferrerPolicy::parse("no-referrer"), Some(ReferrerPolicy::NoReferrer)); + assert_eq!(ReferrerPolicy::parse("NO-REFERRER"), Some(ReferrerPolicy::NoReferrer)); + assert_eq!( + ReferrerPolicy::parse("No-Referrer-When-Downgrade"), + Some(ReferrerPolicy::NoReferrerWhenDowngrade) + ); + assert_eq!(ReferrerPolicy::parse("origin"), Some(ReferrerPolicy::Origin)); + assert_eq!( + ReferrerPolicy::parse("origin-when-cross-origin"), + Some(ReferrerPolicy::OriginWhenCrossOrigin) + ); + assert_eq!(ReferrerPolicy::parse("same-origin"), Some(ReferrerPolicy::SameOrigin)); + assert_eq!(ReferrerPolicy::parse("strict-origin"), Some(ReferrerPolicy::StrictOrigin)); + assert_eq!( + ReferrerPolicy::parse("strict-origin-when-cross-origin"), + Some(ReferrerPolicy::StrictOriginWhenCrossOrigin) + ); + assert_eq!(ReferrerPolicy::parse("unsafe-url"), Some(ReferrerPolicy::UnsafeUrl)); + } + + #[test] + fn parse_last_token_wins() { + assert_eq!( + ReferrerPolicy::parse("origin, no-referrer"), + Some(ReferrerPolicy::NoReferrer) + ); + assert_eq!( + ReferrerPolicy::parse("no-referrer, origin"), + Some(ReferrerPolicy::Origin) + ); + } + + #[test] + fn parse_invalid_returns_none() { + assert_eq!(ReferrerPolicy::parse(""), None); + assert_eq!(ReferrerPolicy::parse("garbage"), None); + assert_eq!( + ReferrerPolicy::parse("no-referrer, garbage"), + Some(ReferrerPolicy::NoReferrer) + ); + } + + #[test] + fn compute_referer_matrix() { + let a = u("https://a.com/p"); + let b = u("https://b.com/q"); + let a_http = u("http://a.com/r"); + + // None / NoReferrerWhenDowngrade (default) + assert_eq!(compute_referer(None, &a, &b), Some("https://a.com/p".to_string())); + assert_eq!(compute_referer(None, &a, &a_http), None); + + // NoReferrer + assert_eq!(compute_referer(Some(ReferrerPolicy::NoReferrer), &a, &b), None); + + // Origin + assert_eq!( + compute_referer(Some(ReferrerPolicy::Origin), &a, &b), + Some("https://a.com".to_string()) + ); + assert_eq!( + compute_referer(Some(ReferrerPolicy::Origin), &a, &a_http), + Some("https://a.com".to_string()) + ); + + // OriginWhenCrossOrigin + assert_eq!( + compute_referer(Some(ReferrerPolicy::OriginWhenCrossOrigin), &a, &a), + Some("https://a.com/p".to_string()) + ); + assert_eq!( + compute_referer(Some(ReferrerPolicy::OriginWhenCrossOrigin), &a, &b), + Some("https://a.com".to_string()) + ); + + // SameOrigin + assert_eq!( + compute_referer(Some(ReferrerPolicy::SameOrigin), &a, &a), + Some("https://a.com/p".to_string()) + ); + assert_eq!(compute_referer(Some(ReferrerPolicy::SameOrigin), &a, &b), None); + + // StrictOrigin + assert_eq!( + compute_referer(Some(ReferrerPolicy::StrictOrigin), &a, &a), + Some("https://a.com".to_string()) + ); + assert_eq!(compute_referer(Some(ReferrerPolicy::StrictOrigin), &a, &a_http), None); + + // StrictOriginWhenCrossOrigin + assert_eq!( + compute_referer(Some(ReferrerPolicy::StrictOriginWhenCrossOrigin), &a, &a), + Some("https://a.com/p".to_string()) + ); + assert_eq!( + compute_referer(Some(ReferrerPolicy::StrictOriginWhenCrossOrigin), &a, &b), + Some("https://a.com".to_string()) + ); + assert_eq!( + compute_referer(Some(ReferrerPolicy::StrictOriginWhenCrossOrigin), &a, &a_http), + None + ); + + // UnsafeUrl (keeps full referer, no downgrade suppression) + assert_eq!( + compute_referer(Some(ReferrerPolicy::UnsafeUrl), &a, &a_http), + Some("https://a.com/p".to_string()) + ); + } + + #[test] + fn compute_referer_strips_userinfo_and_fragment() { + let with_auth = u("https://user:pass@a.com/p#frag"); + let other = u("https://b.com/q"); + let same = u("https://user:pass@a.com/r#x"); + // cross-origin -> only origin (no userinfo/fragment) + assert_eq!( + compute_referer(Some(ReferrerPolicy::OriginWhenCrossOrigin), &with_auth, &other), + Some("https://a.com".to_string()) + ); + // same-origin -> full referer with userinfo & fragment stripped + assert_eq!( + compute_referer(Some(ReferrerPolicy::OriginWhenCrossOrigin), &with_auth, &same), + Some("https://a.com/p".to_string()) + ); + } +} diff --git a/crates/fast-down/src/proxy.rs b/crates/fast-down/src/proxy.rs index 1a96fa9..fd9c7e3 100644 --- a/crates/fast-down/src/proxy.rs +++ b/crates/fast-down/src/proxy.rs @@ -53,3 +53,43 @@ impl Proxy { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn map_transforms_custom_only() { + assert_eq!(Proxy::::No.map(|x| x + 1), Proxy::No); + assert_eq!(Proxy::::System.map(|x| x + 1), Proxy::System); + assert_eq!(Proxy::Custom(5).map(|x| x + 1), Proxy::Custom(6)); + } + + #[test] + fn as_deref_borrows_custom() { + let p: Proxy = Proxy::Custom("http".to_string()); + assert_eq!(p.as_deref(), Proxy::Custom("http")); + assert_eq!(Proxy::::No.as_deref(), Proxy::No); + assert_eq!(Proxy::::System.as_deref(), Proxy::System); + } + + #[test] + fn as_ref_borrows_custom() { + let p: Proxy = Proxy::Custom(7); + assert_eq!(p.as_ref(), Proxy::Custom(&7)); + assert_eq!(Proxy::::No.as_ref(), Proxy::No); + assert_eq!(Proxy::::System.as_ref(), Proxy::System); + } + + #[test] + fn default_is_system() { + assert_eq!(Proxy::::default(), Proxy::System); + } + + #[test] + fn copy_and_equality() { + let p = Proxy::Custom(3); + let q = p; // Proxy is Copy + assert_eq!(p, q); + } +} diff --git a/crates/fast-down/src/url_info.rs b/crates/fast-down/src/url_info.rs index a2648ca..e0dfa1e 100644 --- a/crates/fast-down/src/url_info.rs +++ b/crates/fast-down/src/url_info.rs @@ -71,3 +71,49 @@ impl FileId { } } } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + + #[test] + fn file_id_new() { + let f = FileId::new(Some("abc"), Some("def")); + assert_eq!(f.etag, Some(Arc::from("abc"))); + assert_eq!(f.last_modified, Some(Arc::from("def"))); + } + + #[test] + fn file_id_none_fields() { + let f = FileId::new(None, None); + assert_eq!(f.etag, None); + assert_eq!(f.last_modified, None); + } + + #[test] + fn file_id_equality() { + assert_eq!(FileId::new(Some("x"), None), FileId::new(Some("x"), None)); + assert_ne!(FileId::new(Some("x"), None), FileId::new(Some("x"), Some("y"))); + } + + #[test] + fn url_info_filename_sanitizes() { + #![allow(unused_variables)] + let info = UrlInfo { + size: 10, + raw_name: "a/b:c*?.txt".to_string(), + supports_range: true, + fast_download: true, + final_url: Url::parse("http://example.com/x").unwrap(), + file_id: FileId::default(), + content_type: Some("text/plain".to_string()), + }; + #[cfg(feature = "sanitize-filename")] + { + let name = info.filename(); + assert!(!name.is_empty()); + assert!(!name.contains(['/', ':', '*', '?'])); + } + } +} diff --git a/crates/fast-pull/src/base/invert.rs b/crates/fast-pull/src/base/invert.rs index fe9f860..8d1055e 100644 --- a/crates/fast-pull/src/base/invert.rs +++ b/crates/fast-pull/src/base/invert.rs @@ -77,4 +77,45 @@ mod tests { assert_eq!(invert_vec(&[2..4, 6..8, 10..12], 15, 5), [0..15]); assert_eq!(invert_vec(&[0..2, 10..20], 30, 5), [2..10, 20..30]); } + + #[test] + fn test_invert_empty_progress() { + // Nothing downloaded of a 50-byte file -> one gap spanning everything. + assert_eq!(invert_vec(&[], 50, 1), [0..50]); + } + + #[test] + fn test_invert_zero_total_size() { + // total_size 0 -> no gaps, even if progress is present. + assert_eq!(invert_vec(&[0..5], 0, 1), []); + } + + #[test] + fn test_invert_full_cover_no_gaps() { + assert_eq!(invert_vec(&[0..30], 30, 1), []); + } + + #[test] + fn test_invert_window_zero_keeps_small_entries() { + #![allow(clippy::single_range_in_vec_init)] + // window=0 means every entry (even tiny) is kept, so small entries are + // not merged into the surrounding gap. + assert_eq!(invert_vec(&[10..12], 30, 0), [0..10, 12..30]); + } + + #[test] + fn test_invert_trailing_gap_only() { + assert_eq!(invert_vec(&[0..20], 30, 1), [20..30]); + } + + #[test] + fn test_invert_leading_gap_only() { + assert_eq!(invert_vec(&[10..30], 30, 1), [0..10]); + } + + #[test] + fn test_invert_contiguous_then_gap() { + #![allow(clippy::single_range_in_vec_init)] + assert_eq!(invert_vec(&[0..10, 10..20], 30, 1), [20..30]); + } } diff --git a/crates/fast-pull/src/base/merge.rs b/crates/fast-pull/src/base/merge.rs index a2f5c57..505f961 100644 --- a/crates/fast-pull/src/base/merge.rs +++ b/crates/fast-pull/src/base/merge.rs @@ -123,4 +123,69 @@ mod tests { v.merge_progress(30..5); assert_eq!(v, vec![10..20]); } + + #[test] + fn test_merge_into_empty_vec() { + #![allow(clippy::single_range_in_vec_init)] + let mut v: Vec = vec![]; + v.merge_progress(5..10); + assert_eq!(v, vec![5..10]); + } + + #[test] + fn test_merge_superset_absorbs_all() { + #![allow(clippy::single_range_in_vec_init)] + let mut v = vec![1..5, 8..10, 20..30]; + v.merge_progress(0..40); + assert_eq!(v, vec![0..40]); + } + + #[test] + fn test_merge_exact_duplicate_is_noop() { + #![allow(clippy::single_range_in_vec_init)] + let mut v = vec![1..5]; + v.merge_progress(1..5); + assert_eq!(v, vec![1..5]); + } + + #[test] + fn test_merge_touching_right_extends() { + #![allow(clippy::single_range_in_vec_init)] + let mut v = vec![1..5]; + v.merge_progress(5..8); + assert_eq!(v, vec![1..8]); + } + + #[test] + fn test_merge_touching_left_extends() { + #![allow(clippy::single_range_in_vec_init)] + let mut v = vec![5..10]; + v.merge_progress(1..5); + assert_eq!(v, vec![1..10]); + } + + /// Simulate concurrent, out-of-order chunk delivery where each chunk is + /// *adjacent* (touching but not overlapping) the next — the worst case for + /// `download_complete`'s `x.len() == 1` check in `overwrite.rs`. This must + /// still coalesce into a single `[0..200]` entry, otherwise the download + /// would be wrongly reported as incomplete and the `.part` never renamed. + #[test] + fn test_merge_out_of_order_adjacent_coalesces_to_single() { + let mut v: Vec = vec![]; + // 4 chunks of 50 bytes on a 200-byte file, arriving in a scrambled order. + v.merge_progress(0..50); + v.merge_progress(150..200); + v.merge_progress(100..150); + v.merge_progress(50..100); + assert_eq!(v, vec![0..200]); + + // And the canonical "fully covered, single entry" invariant holds for + // any interleaving that covers the whole span. + let mut w: Vec = vec![]; + for r in [0..70, 140..200, 70..140] { + w.merge_progress(r); + } + assert_eq!(w, vec![0..200]); + assert!(w.len() == 1 && w[0] == (0..200)); + } } diff --git a/crates/fast-pull/src/base/progress.rs b/crates/fast-pull/src/base/progress.rs index 1a88d46..bee05bd 100644 --- a/crates/fast-pull/src/base/progress.rs +++ b/crates/fast-pull/src/base/progress.rs @@ -26,3 +26,27 @@ impl Total for Vec { self.iter().map(Total::total).sum() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn progress_entry_total() { + assert_eq!((0..0).total(), 0); + assert_eq!((5..10).total(), 5); + assert_eq!((3..3).total(), 0); + // A reversed range must not underflow: saturating_sub yields 0. + let reversed = core::ops::Range { start: 10, end: 5 }; + assert_eq!(reversed.total(), 0); + } + + #[test] + fn vec_progress_total() { + #![allow(clippy::single_range_in_vec_init)] + let v: Vec = vec![1..5, 8..10, 12..15]; + assert_eq!(v.total(), 9); + let empty: Vec = vec![]; + assert_eq!(empty.total(), 0); + } +} diff --git a/crates/fast-pull/src/cache/merge.rs b/crates/fast-pull/src/cache/merge.rs index 52c3a40..81df302 100644 --- a/crates/fast-pull/src/cache/merge.rs +++ b/crates/fast-pull/src/cache/merge.rs @@ -145,3 +145,111 @@ impl Pusher for CacheMergePusher

{ self.inner.flush() } } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + /// A `Pusher` that records everything pushed into a shared buffer so tests + /// can inspect it without reaching into `CacheMergePusher`'s private fields. + #[derive(Clone)] + struct SharedSink { + pushes: Arc>>, + fail_next: Arc, + } + impl SharedSink { + fn new() -> Self { + Self { + pushes: Arc::new(Mutex::new(Vec::new())), + fail_next: Arc::new(AtomicBool::new(false)), + } + } + } + impl Pusher for SharedSink { + type Error = std::io::Error; + fn set_listener(&mut self, _: ProgressListener) {} + fn push( + &mut self, + range: &ProgressEntry, + bytes: Bytes, + ) -> Result<(), (Self::Error, Bytes)> { + if self.fail_next.fetch_and(false, Ordering::SeqCst) { + return Err((std::io::Error::new(std::io::ErrorKind::Other, "boom"), bytes)); + } + self.pushes.lock().unwrap().push((range.clone(), bytes)); + Ok(()) + } + fn flush(&mut self) -> Result<(), Self::Error> { + Ok(()) + } + } + + fn bb(s: &str) -> Bytes { + Bytes::copy_from_slice(s.as_bytes()) + } + + #[test] + fn test_cache_merge_evicts_contiguous_run() { + let sink = SharedSink::new(); + let mut p = CacheMergePusher::new(sink.clone(), 30, 0); + // Out-of-order insertion; not yet at watermark. + p.push(&(0..10), bb(&"A".repeat(10))).unwrap(); + p.push(&(20..30), bb(&"C".repeat(10))).unwrap(); + assert!(sink.pushes.lock().unwrap().is_empty()); + // This insertion reaches the high watermark and triggers a merge+evict. + p.push(&(10..20), bb(&"B".repeat(10))).unwrap(); + let pushes = sink.pushes.lock().unwrap(); + assert_eq!(pushes.len(), 1); + assert_eq!(pushes[0].0, 0..30); + assert_eq!(pushes[0].1.len(), 30); + // Merged bytes preserve ascending order: A(0..10) B(10..20) C(20..30). + assert_eq!(&pushes[0].1[..], b"AAAAAAAAAABBBBBBBBBBCCCCCCCCCC"); + } + + #[test] + fn test_cache_merge_no_evict_below_watermark() { + let sink = SharedSink::new(); + let mut p = CacheMergePusher::new(sink.clone(), 100, 0); + p.push(&(0..10), bb(&"A".repeat(10))).unwrap(); + p.push(&(10..20), bb(&"B".repeat(10))).unwrap(); + assert!(sink.pushes.lock().unwrap().is_empty()); + // Explicit flush must drain the buffered run to the inner pusher. + p.flush().unwrap(); + let pushes = sink.pushes.lock().unwrap(); + assert_eq!(pushes.len(), 1); + assert_eq!(pushes[0].0, 0..20); + } + + #[test] + fn test_cache_merge_inner_failure_propagates() { + let sink = SharedSink::new(); + sink.fail_next.store(true, Ordering::SeqCst); + let mut p = CacheMergePusher::new(sink.clone(), 10, 0); + let res = p.push(&(0..10), bb(&"A".repeat(10))); + assert!(res.is_err()); + } + + #[test] + fn test_cache_merge_empty_bytes_is_noop() { + let sink = SharedSink::new(); + let mut p = CacheMergePusher::new(sink.clone(), 1, 0); + p.push(&(0..0), Bytes::new()).unwrap(); + assert!(sink.pushes.lock().unwrap().is_empty()); + } + + #[test] + fn test_cache_merge_partial_overwrite_replaces() { + // A chunk at an already-cached start position replaces the old bytes. + let sink = SharedSink::new(); + let mut p = CacheMergePusher::new(sink.clone(), 100, 0); + p.push(&(0..10), bb(&"A".repeat(10))).unwrap(); + p.push(&(0..10), bb(&"B".repeat(10))).unwrap(); + p.flush().unwrap(); + let pushes = sink.pushes.lock().unwrap(); + assert_eq!(pushes.len(), 1); + assert_eq!(&pushes[0].1[..], b"BBBBBBBBBB"); + } +} diff --git a/crates/fast-pull/src/mem/pusher.rs b/crates/fast-pull/src/mem/pusher.rs index ae3b786..17fab5d 100644 --- a/crates/fast-pull/src/mem/pusher.rs +++ b/crates/fast-pull/src/mem/pusher.rs @@ -70,3 +70,57 @@ impl Pusher for MemPusher { Ok(()) } } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use std::sync::{Arc, Mutex}; + + #[test] + fn sequential_append() { + let mut p = MemPusher::new(); + p.push(&(0..3), Bytes::copy_from_slice(b"abc")).unwrap(); + p.push(&(3..6), Bytes::copy_from_slice(b"def")).unwrap(); + assert_eq!(&p.receive.lock()[..], b"abcdef"); + } + + #[test] + fn random_access_resizes_and_writes() { + let mut p = MemPusher::new(); + p.push(&(5..8), Bytes::copy_from_slice(b"xyz")).unwrap(); + // The gap before the write is filled with zeros via resize. + assert_eq!(p.receive.lock().len(), 8); + p.push(&(0..5), Bytes::copy_from_slice(b"hello")).unwrap(); + assert_eq!(&p.receive.lock()[..], b"helloxyz"); + } + + #[test] + fn empty_push_is_noop() { + let mut p = MemPusher::new(); + p.push(&(0..0), Bytes::new()).unwrap(); + assert!(p.receive.lock().is_empty()); + } + + #[test] + fn listener_invoked_with_range() { + let mut p = MemPusher::new(); + let seen = Arc::new(Mutex::new(None::)); + let seen2 = seen.clone(); + p.set_listener(Box::new(move |r| { + *seen2.lock().unwrap() = Some(r); + })); + p.push(&(0..4), Bytes::copy_from_slice(b"data")).unwrap(); + assert_eq!(*seen.lock().unwrap(), Some(0..4)); + } + + #[test] + fn clone_shares_receive() { + let mut p = MemPusher::new(); + p.push(&(0..2), Bytes::copy_from_slice(b"hi")).unwrap(); + let mut q = p.clone(); + q.push(&(2..4), Bytes::copy_from_slice(b"ya")).unwrap(); + // Both handles observe the same underlying vec. + assert_eq!(&p.receive.lock()[..], b"hiya"); + } +} diff --git a/crates/fast-steal/src/task.rs b/crates/fast-steal/src/task.rs index f2a24e5..d743c39 100644 --- a/crates/fast-steal/src/task.rs +++ b/crates/fast-steal/src/task.rs @@ -282,4 +282,65 @@ mod tests { assert_eq!(task.end(), 2); assert_eq!(range, None); } + + #[test] + fn test_safe_add_start_no_progress() { + let task = Task::new(10..20); + // bias 0 -> start does not advance + assert_eq!(task.safe_add_start(10, 0), Err(RangeError)); + // bias would not exceed current start + assert_eq!(task.safe_add_start(8, 2), Err(RangeError)); + } + + #[test] + fn test_safe_add_start_claims_span() { + let task = Task::new(10..20); + let span = task.safe_add_start(10, 5).unwrap(); + assert_eq!(span, 10..15); + assert_eq!(task.start(), 15); + assert_eq!(task.remain(), 5); + } + + #[test] + fn test_safe_add_start_capped_at_end() { + let task = Task::new(10..12); + let span = task.safe_add_start(10, 100).unwrap(); + assert_eq!(span, 10..12); + assert_eq!(task.remain(), 0); + } + + #[test] + fn test_take_empties() { + let task = Task::new(5..9); + assert_eq!(task.take(), Some(5..9)); + assert_eq!(task.take(), None); + assert_eq!(task.remain(), 0); + } + + #[test] + fn test_downgrade_upgrade() { + let task = Task::new(1..10); + let weak = task.downgrade(); + assert_eq!(weak.strong_count(), 1); + assert_eq!(weak.upgrade().unwrap().get(), 1..10); + drop(task); + assert_eq!(weak.upgrade(), None); + } + + #[test] + fn test_partial_eq_by_ptr() { + let a = Task::new(1..10); + let b = a.clone(); + assert_eq!(a, b); + let c = Task::new(1..10); + assert_ne!(a, c); + } + + #[test] + fn test_split_two_halves() { + let task = Task::new(0..100); + let range = task.split_two().unwrap().unwrap(); + assert_eq!(range, 50..100); + assert_eq!(task.get(), 0..50); + } } From f5933bb392470eaf9a98d3b0d108e9c6b39046cb Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 19:30:12 +0800 Subject: [PATCH 23/26] =?UTF-8?q?feat(fast-down-api):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=8E=87=E8=A6=86=E7=9B=96=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/README.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/crates/fast-down-api/README.md b/crates/fast-down-api/README.md index dbd6212..bfa5420 100644 --- a/crates/fast-down-api/README.md +++ b/crates/fast-down-api/README.md @@ -2,6 +2,7 @@ [![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) [![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![codecov](https://codecov.io/gh/fast-down/core/branch/main/graph/badge.svg)](https://codecov.io/gh/fast-down/core) [![Latest version](https://img.shields.io/crates/v/fast-down-api.svg)](https://crates.io/crates/fast-down-api) [![Documentation](https://docs.rs/fast-down-api/badge.svg)](https://docs.rs/fast-down-api) [![License](https://img.shields.io/crates/l/fast-down-api.svg)](https://github.com/fast-down/core/blob/main/LICENSE) @@ -90,16 +91,16 @@ token.cancel(); // stops fetching, keeps .part / .fd so you can resume later ## API overview -| Item | Purpose | -| --- | --- | -| [`DownloadHandle::download`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.download) | Start a download; auto-resume when a valid `.fd` + `.part` exist, else fresh. | -| [`DownloadHandle::resume`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.resume) | Resume a specific `.part` file; hard-error (`Event::ResumeError`) if it can't. | -| [`DownloadHandle::join`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.join) | Await the detached task; errors if the worker panicked. | -| [`create_channel`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_channel.html) | Create the `(Tx, Rx)` event channel. | -| [`create_cancellation_token`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_cancellation_token.html) | Create a `CancellationToken` for cooperative cancellation. | -| [`Event`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.Event.html) | The event enum delivered over the channel. | -| [`PartialConfig`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.PartialConfig.html) | Layered, optional configuration for a download. | -| [`StateError`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.StateError.html) | Errors surfaced via `Event::ResumeError`. | +| Item | Purpose | +| --------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| [`DownloadHandle::download`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.download) | Start a download; auto-resume when a valid `.fd` + `.part` exist, else fresh. | +| [`DownloadHandle::resume`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.resume) | Resume a specific `.part` file; hard-error (`Event::ResumeError`) if it can't. | +| [`DownloadHandle::join`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.join) | Await the detached task; errors if the worker panicked. | +| [`create_channel`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_channel.html) | Create the `(Tx, Rx)` event channel. | +| [`create_cancellation_token`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_cancellation_token.html) | Create a `CancellationToken` for cooperative cancellation. | +| [`Event`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.Event.html) | The event enum delivered over the channel. | +| [`PartialConfig`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.PartialConfig.html) | Layered, optional configuration for a download. | +| [`StateError`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.StateError.html) | Errors surfaced via `Event::ResumeError`. | ## How resume works From f74bf7b2a1b98735b021fc2566010cd838cfe1a3 Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 19:28:00 +0800 Subject: [PATCH 24/26] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E8=A6=86=E7=9B=96=E7=8E=87=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- .github/workflows/coverage.yml | 45 ++++++++++++++++++++++++++++++++++ README.md | 1 + crates/fast-down/README.md | 1 + crates/fast-pull/README.md | 1 + crates/fast-steal/README.md | 1 + 5 files changed, 49 insertions(+) create mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..a36f71f --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,45 @@ +name: Coverage + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + coverage: + name: Code Coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview + + - name: Cache + uses: swatinem/rust-cache@v2 + with: + shared-key: "ubuntu-stable-coverage" + add-job-id-key: "false" + cache-on-failure: "true" + cache-all-crates: "true" + cache-workspace-crates: "true" + save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' }} + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@v2 + with: + tool: cargo-llvm-cov + + - name: Generate coverage (lcov) + run: cargo llvm-cov --workspace --all-features --lcov --output-path lcov.info + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: lcov.info + fail_ci_if_error: false diff --git a/README.md b/README.md index c1b8cda..3b8dbb3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) [![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![codecov](https://codecov.io/gh/fast-down/core/branch/main/graph/badge.svg)](https://codecov.io/gh/fast-down/core) [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/fast-down/core/blob/main/LICENSE) `fast-down` **Fastest** concurrent downloader! diff --git a/crates/fast-down/README.md b/crates/fast-down/README.md index 04b196a..9d3cceb 100644 --- a/crates/fast-down/README.md +++ b/crates/fast-down/README.md @@ -2,6 +2,7 @@ [![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) [![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![codecov](https://codecov.io/gh/fast-down/core/branch/main/graph/badge.svg)](https://codecov.io/gh/fast-down/core) [![Latest version](https://img.shields.io/crates/v/fast-down.svg)](https://crates.io/crates/fast-down) [![Documentation](https://docs.rs/fast-down/badge.svg)](https://docs.rs/fast-down) [![License](https://img.shields.io/crates/l/fast-down.svg)](https://github.com/fast-down/core/blob/main/LICENSE) diff --git a/crates/fast-pull/README.md b/crates/fast-pull/README.md index f514828..1322a48 100644 --- a/crates/fast-pull/README.md +++ b/crates/fast-pull/README.md @@ -2,6 +2,7 @@ [![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) [![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![codecov](https://codecov.io/gh/fast-down/core/branch/main/graph/badge.svg)](https://codecov.io/gh/fast-down/core) [![Latest version](https://img.shields.io/crates/v/fast-pull.svg)](https://crates.io/crates/fast-pull) [![Documentation](https://docs.rs/fast-pull/badge.svg)](https://docs.rs/fast-pull) [![License](https://img.shields.io/crates/l/fast-pull.svg)](https://github.com/fast-down/core/blob/main/LICENSE) diff --git a/crates/fast-steal/README.md b/crates/fast-steal/README.md index 51fe1f0..f09639c 100644 --- a/crates/fast-steal/README.md +++ b/crates/fast-steal/README.md @@ -2,6 +2,7 @@ [![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) [![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![codecov](https://codecov.io/gh/fast-down/core/branch/main/graph/badge.svg)](https://codecov.io/gh/fast-down/core) [![Latest version](https://img.shields.io/crates/v/fast-steal.svg)](https://crates.io/crates/fast-steal) [![Documentation](https://docs.rs/fast-steal/badge.svg)](https://docs.rs/fast-steal) [![License](https://img.shields.io/crates/l/fast-steal.svg)](https://github.com/fast-down/core/blob/main/LICENSE) From e8113e74d9b951e558d79f6852e0b20444289da2 Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 19:57:27 +0800 Subject: [PATCH 25/26] ci: bump codecov-action to v7 (Node 24, drop Node 20 deprecation warning) --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a36f71f..0e30426 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -39,7 +39,7 @@ jobs: run: cargo llvm-cov --workspace --all-features --lcov --output-path lcov.info - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v7 with: files: lcov.info fail_ci_if_error: false From 76a6842bd7f493d91b15554225573dacc26a93bd Mon Sep 17 00:00:00 2001 From: share121 Date: Tue, 28 Jul 2026 20:10:44 +0800 Subject: [PATCH 26/26] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20ci=20?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: share121 --- crates/fast-down-api/src/config.rs | 1 + .../src/utils/filename_template.rs | 15 +++- crates/fast-down/src/http/manual_redirect.rs | 68 +++++++++++++++---- crates/fast-down/src/url_info.rs | 5 +- crates/fast-pull/src/cache/merge.rs | 7 +- 5 files changed, 77 insertions(+), 19 deletions(-) diff --git a/crates/fast-down-api/src/config.rs b/crates/fast-down-api/src/config.rs index 39f3345..963e75b 100644 --- a/crates/fast-down-api/src/config.rs +++ b/crates/fast-down-api/src/config.rs @@ -185,6 +185,7 @@ impl PartialConfig { } #[cfg(test)] +#[allow(clippy::single_range_in_vec_init)] mod tests { use super::*; diff --git a/crates/fast-down-api/src/utils/filename_template.rs b/crates/fast-down-api/src/utils/filename_template.rs index 1168a56..f887d9e 100644 --- a/crates/fast-down-api/src/utils/filename_template.rs +++ b/crates/fast-down-api/src/utils/filename_template.rs @@ -55,19 +55,28 @@ mod tests { #[test] fn no_placeholders_passthrough() { let url = Url::parse("https://example.com/x").unwrap(); - assert_eq!(parse_filename_template("plain".to_string(), &url, "f.txt"), "plain"); + assert_eq!( + parse_filename_template("plain".to_string(), &url, "f.txt"), + "plain" + ); } #[test] fn host_unknown_when_no_host() { let url = Url::parse("file:///etc/hosts").unwrap(); - assert_eq!(parse_filename_template("{host}".to_string(), &url, "hosts"), "unknown"); + assert_eq!( + parse_filename_template("{host}".to_string(), &url, "hosts"), + "unknown" + ); } #[test] fn parent_path_root_when_no_dir() { let url = Url::parse("https://example.com/file.txt").unwrap(); - assert_eq!(parse_filename_template("{parent_path}".to_string(), &url, "file.txt"), "."); + assert_eq!( + parse_filename_template("{parent_path}".to_string(), &url, "file.txt"), + "." + ); } #[test] diff --git a/crates/fast-down/src/http/manual_redirect.rs b/crates/fast-down/src/http/manual_redirect.rs index 993aaed..56cae12 100644 --- a/crates/fast-down/src/http/manual_redirect.rs +++ b/crates/fast-down/src/http/manual_redirect.rs @@ -135,24 +135,42 @@ mod tests { #[test] fn parse_all_tokens_case_insensitive() { - assert_eq!(ReferrerPolicy::parse("no-referrer"), Some(ReferrerPolicy::NoReferrer)); - assert_eq!(ReferrerPolicy::parse("NO-REFERRER"), Some(ReferrerPolicy::NoReferrer)); + assert_eq!( + ReferrerPolicy::parse("no-referrer"), + Some(ReferrerPolicy::NoReferrer) + ); + assert_eq!( + ReferrerPolicy::parse("NO-REFERRER"), + Some(ReferrerPolicy::NoReferrer) + ); assert_eq!( ReferrerPolicy::parse("No-Referrer-When-Downgrade"), Some(ReferrerPolicy::NoReferrerWhenDowngrade) ); - assert_eq!(ReferrerPolicy::parse("origin"), Some(ReferrerPolicy::Origin)); + assert_eq!( + ReferrerPolicy::parse("origin"), + Some(ReferrerPolicy::Origin) + ); assert_eq!( ReferrerPolicy::parse("origin-when-cross-origin"), Some(ReferrerPolicy::OriginWhenCrossOrigin) ); - assert_eq!(ReferrerPolicy::parse("same-origin"), Some(ReferrerPolicy::SameOrigin)); - assert_eq!(ReferrerPolicy::parse("strict-origin"), Some(ReferrerPolicy::StrictOrigin)); + assert_eq!( + ReferrerPolicy::parse("same-origin"), + Some(ReferrerPolicy::SameOrigin) + ); + assert_eq!( + ReferrerPolicy::parse("strict-origin"), + Some(ReferrerPolicy::StrictOrigin) + ); assert_eq!( ReferrerPolicy::parse("strict-origin-when-cross-origin"), Some(ReferrerPolicy::StrictOriginWhenCrossOrigin) ); - assert_eq!(ReferrerPolicy::parse("unsafe-url"), Some(ReferrerPolicy::UnsafeUrl)); + assert_eq!( + ReferrerPolicy::parse("unsafe-url"), + Some(ReferrerPolicy::UnsafeUrl) + ); } #[test] @@ -184,11 +202,17 @@ mod tests { let a_http = u("http://a.com/r"); // None / NoReferrerWhenDowngrade (default) - assert_eq!(compute_referer(None, &a, &b), Some("https://a.com/p".to_string())); + assert_eq!( + compute_referer(None, &a, &b), + Some("https://a.com/p".to_string()) + ); assert_eq!(compute_referer(None, &a, &a_http), None); // NoReferrer - assert_eq!(compute_referer(Some(ReferrerPolicy::NoReferrer), &a, &b), None); + assert_eq!( + compute_referer(Some(ReferrerPolicy::NoReferrer), &a, &b), + None + ); // Origin assert_eq!( @@ -215,14 +239,20 @@ mod tests { compute_referer(Some(ReferrerPolicy::SameOrigin), &a, &a), Some("https://a.com/p".to_string()) ); - assert_eq!(compute_referer(Some(ReferrerPolicy::SameOrigin), &a, &b), None); + assert_eq!( + compute_referer(Some(ReferrerPolicy::SameOrigin), &a, &b), + None + ); // StrictOrigin assert_eq!( compute_referer(Some(ReferrerPolicy::StrictOrigin), &a, &a), Some("https://a.com".to_string()) ); - assert_eq!(compute_referer(Some(ReferrerPolicy::StrictOrigin), &a, &a_http), None); + assert_eq!( + compute_referer(Some(ReferrerPolicy::StrictOrigin), &a, &a_http), + None + ); // StrictOriginWhenCrossOrigin assert_eq!( @@ -234,7 +264,11 @@ mod tests { Some("https://a.com".to_string()) ); assert_eq!( - compute_referer(Some(ReferrerPolicy::StrictOriginWhenCrossOrigin), &a, &a_http), + compute_referer( + Some(ReferrerPolicy::StrictOriginWhenCrossOrigin), + &a, + &a_http + ), None ); @@ -252,12 +286,20 @@ mod tests { let same = u("https://user:pass@a.com/r#x"); // cross-origin -> only origin (no userinfo/fragment) assert_eq!( - compute_referer(Some(ReferrerPolicy::OriginWhenCrossOrigin), &with_auth, &other), + compute_referer( + Some(ReferrerPolicy::OriginWhenCrossOrigin), + &with_auth, + &other + ), Some("https://a.com".to_string()) ); // same-origin -> full referer with userinfo & fragment stripped assert_eq!( - compute_referer(Some(ReferrerPolicy::OriginWhenCrossOrigin), &with_auth, &same), + compute_referer( + Some(ReferrerPolicy::OriginWhenCrossOrigin), + &with_auth, + &same + ), Some("https://a.com/p".to_string()) ); } diff --git a/crates/fast-down/src/url_info.rs b/crates/fast-down/src/url_info.rs index e0dfa1e..2061e98 100644 --- a/crates/fast-down/src/url_info.rs +++ b/crates/fast-down/src/url_info.rs @@ -94,7 +94,10 @@ mod tests { #[test] fn file_id_equality() { assert_eq!(FileId::new(Some("x"), None), FileId::new(Some("x"), None)); - assert_ne!(FileId::new(Some("x"), None), FileId::new(Some("x"), Some("y"))); + assert_ne!( + FileId::new(Some("x"), None), + FileId::new(Some("x"), Some("y")) + ); } #[test] diff --git a/crates/fast-pull/src/cache/merge.rs b/crates/fast-pull/src/cache/merge.rs index 81df302..83915e8 100644 --- a/crates/fast-pull/src/cache/merge.rs +++ b/crates/fast-pull/src/cache/merge.rs @@ -177,7 +177,7 @@ mod tests { bytes: Bytes, ) -> Result<(), (Self::Error, Bytes)> { if self.fail_next.fetch_and(false, Ordering::SeqCst) { - return Err((std::io::Error::new(std::io::ErrorKind::Other, "boom"), bytes)); + return Err((std::io::Error::other("boom"), bytes)); } self.pushes.lock().unwrap().push((range.clone(), bytes)); Ok(()) @@ -207,6 +207,7 @@ mod tests { assert_eq!(pushes[0].1.len(), 30); // Merged bytes preserve ascending order: A(0..10) B(10..20) C(20..30). assert_eq!(&pushes[0].1[..], b"AAAAAAAAAABBBBBBBBBBCCCCCCCCCC"); + drop(pushes); } #[test] @@ -221,13 +222,14 @@ mod tests { let pushes = sink.pushes.lock().unwrap(); assert_eq!(pushes.len(), 1); assert_eq!(pushes[0].0, 0..20); + drop(pushes); } #[test] fn test_cache_merge_inner_failure_propagates() { let sink = SharedSink::new(); sink.fail_next.store(true, Ordering::SeqCst); - let mut p = CacheMergePusher::new(sink.clone(), 10, 0); + let mut p = CacheMergePusher::new(sink, 10, 0); let res = p.push(&(0..10), bb(&"A".repeat(10))); assert!(res.is_err()); } @@ -251,5 +253,6 @@ mod tests { let pushes = sink.pushes.lock().unwrap(); assert_eq!(pushes.len(), 1); assert_eq!(&pushes[0].1[..], b"BBBBBBBBBB"); + drop(pushes); } }