diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..0e30426 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,45 @@ +name: Coverage + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + coverage: + name: Code Coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview + + - name: Cache + uses: swatinem/rust-cache@v2 + with: + shared-key: "ubuntu-stable-coverage" + add-job-id-key: "false" + cache-on-failure: "true" + cache-all-crates: "true" + cache-workspace-crates: "true" + save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' }} + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@v2 + with: + tool: cargo-llvm-cov + + - name: Generate coverage (lcov) + run: cargo llvm-cov --workspace --all-features --lcov --output-path lcov.info + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v7 + with: + files: lcov.info + fail_ci_if_error: false diff --git a/Cargo.lock b/Cargo.lock index 43a4bca..73301ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -11,6 +17,36 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -21,12 +57,30 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "atomic-waker" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "aws-lc-rs" version = "1.17.3" @@ -62,6 +116,27 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -76,9 +151,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -109,6 +184,19 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "cmake" version = "0.1.58" @@ -137,6 +225,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "cookie" version = "0.18.1" @@ -166,6 +274,16 @@ dependencies = [ "url", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -191,6 +309,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-utils" version = "0.8.22" @@ -278,6 +405,33 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "fast-down-api" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "crossfire", + "fast-down", + "futures", + "http-body-util", + "hyper", + "hyper-util", + "inherit-config", + "parking_lot", + "path_helper", + "reqwest", + "serde", + "soft-canonicalize", + "thiserror", + "tokio", + "tokio-util", + "toml", + "url", + "urlencoding", +] + [[package]] name = "fast-pull" version = "5.0.2" @@ -313,6 +467,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -597,9 +761,35 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", ] [[package]] @@ -715,6 +905,26 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inherit-config" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590c82229e5e5335b001ce25020931c800c382089234890f215e2a4d7f695201" +dependencies = [ + "inherit-config-derive", +] + +[[package]] +name = "inherit-config-derive" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcef06b7bbce7e228d5bf17f3718aa7141ce92bf5cd2ab378ce96c6dd80c0066" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -857,6 +1067,32 @@ dependencies = [ "libc", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -899,6 +1135,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -936,11 +1181,13 @@ dependencies = [ [[package]] name = "path_helper" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8b6f3d03172dc1dac2005cfb2486d79d20f3e256383e14bcc6f23d73af9bc" +checksum = "b1cf0534c5a78bd12bd2c8d9ad3231f22ef07fc2205b5093e63e40cab4c4a8fc" dependencies = [ + "mime_guess", "sanitize-filename", + "tokio", ] [[package]] @@ -997,6 +1244,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-canonicalize" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e8ce98d2141039e100a63f7ab04ee0dec503608d72dd6dbc9d56d7532da3ec" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1204,6 +1457,7 @@ dependencies = [ "cookie", "cookie_store", "futures-core", + "h2", "http", "http-body", "http-body-util", @@ -1314,7 +1568,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni", "log", @@ -1399,7 +1653,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -1464,6 +1718,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1482,6 +1745,22 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simd_cesu8" version = "1.2.0" @@ -1526,6 +1805,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "soft-canonicalize" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80111c43bbe801bcec3679d2ccba588a4618c11de4024abbd028ff50e33436dd" +dependencies = [ + "proc-canonicalize", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1580,6 +1868,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -1679,6 +1988,7 @@ dependencies = [ "mio", "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -1719,6 +2029,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + [[package]] name = "tower" version = "0.5.3" @@ -1740,12 +2089,17 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ + "async-compression", "bitflags", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -1789,6 +2143,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1959,12 +2319,76 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2047,6 +2471,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -2167,3 +2597,31 @@ name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 01e2de3..5954ef9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/fast-down", "crates/fast-pull", "crates/fast-steal", + "crates/fast-down-api", ] [workspace.package] diff --git a/README.md b/README.md index 05a277b..3b8dbb3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) [![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![codecov](https://codecov.io/gh/fast-down/core/branch/main/graph/badge.svg)](https://codecov.io/gh/fast-down/core) [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/fast-down/core/blob/main/LICENSE) `fast-down` **Fastest** concurrent downloader! @@ -9,7 +10,7 @@ - fast-steal: [![Latest version](https://img.shields.io/crates/v/fast-steal.svg)](https://crates.io/crates/fast-steal) [![Documentation](https://docs.rs/fast-steal/badge.svg)](https://docs.rs/fast-steal) - fast-pull: [![Latest version](https://img.shields.io/crates/v/fast-pull.svg)](https://crates.io/crates/fast-pull) [![Documentation](https://docs.rs/fast-pull/badge.svg)](https://docs.rs/fast-pull) - fast-down: [![Latest version](https://img.shields.io/crates/v/fast-down.svg)](https://crates.io/crates/fast-down) [![Documentation](https://docs.rs/fast-down/badge.svg)](https://docs.rs/fast-down) -- fast-down-ffi: [![Latest version](https://img.shields.io/crates/v/fast-down-ffi.svg)](https://crates.io/crates/fast-down-ffi) [![Documentation](https://docs.rs/fast-down-ffi/badge.svg)](https://docs.rs/fast-down-ffi) +- fast-down-api: [![Latest version](https://img.shields.io/crates/v/fast-down-api.svg)](https://crates.io/crates/fast-down-api) [![Documentation](https://docs.rs/fast-down-api/badge.svg)](https://docs.rs/fast-down-api) **[Official Website (Simplified Chinese)](https://fd.s121.top/)** diff --git a/crates/fast-down-api/Cargo.toml b/crates/fast-down-api/Cargo.toml new file mode 100644 index 0000000..28f9a01 --- /dev/null +++ b/crates/fast-down-api/Cargo.toml @@ -0,0 +1,64 @@ +[package] +name = "fast-down-api" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +description = "A convenient and easy-to-use FFI wrapper for fast-down" +documentation = "https://docs.rs/fast-down-api" +readme = "README.md" +homepage.workspace = true +repository.workspace = true +license.workspace = true +keywords = ["concurrency", "downloader", "parallel", "performance", "thread"] +categories = ["network-programming"] + +[dependencies] +fast-down = { workspace = true, features = [ + "cookie-store", + "fast-puller", + "file", + "getifaddrs", + "mem", + "reqwest-tls", + "serde", +] } +parking_lot.workspace = true +tokio.workspace = true +tokio-util = "0.7" +url = { workspace = true, features = ["serde"] } +serde = { version = "1.0.228", features = ["serde_derive"] } +reqwest = { version = "0.13.4", default-features = false, features = [ + "brotli", + "default-tls", + "deflate", + "gzip", + "http2", + "socks", + "system-proxy", + "zstd", + "cookies", +] } +thiserror.workspace = true +crossfire.workspace = true +anyhow = "1.0.103" +path_helper = { version = "0.1.9", features = [ + "auto_ext", + "sanitize", + "tokio", +] } +chrono = "0.4.45" +urlencoding = "2.1.3" +soft-canonicalize = "0.5.6" +inherit-config = "0.2.2" +toml = "1.1.2" + +[dev-dependencies] +bytes = { workspace = true } +futures = { workspace = true } +tokio = { workspace = true, features = ["full"] } +hyper = { version = "1", features = ["server", "http1"] } +hyper-util = { version = "0.1", features = ["tokio", "server", "http1"] } +http-body-util = "0.1" + +[lints] +workspace = true diff --git a/crates/fast-down-api/README.md b/crates/fast-down-api/README.md new file mode 100644 index 0000000..bfa5420 --- /dev/null +++ b/crates/fast-down-api/README.md @@ -0,0 +1,120 @@ +# fast-down-api + +[![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) +[![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![codecov](https://codecov.io/gh/fast-down/core/branch/main/graph/badge.svg)](https://codecov.io/gh/fast-down/core) +[![Latest version](https://img.shields.io/crates/v/fast-down-api.svg)](https://crates.io/crates/fast-down-api) +[![Documentation](https://docs.rs/fast-down-api/badge.svg)](https://docs.rs/fast-down-api) +[![License](https://img.shields.io/crates/l/fast-down-api.svg)](https://github.com/fast-down/core/blob/main/LICENSE) + +A convenient, high-level wrapper around [`fast-down`](https://github.com/fast-down/fast-down) +that turns the pull/push engine into a few lines of async code: spawn a download, +drain progress events, resume after interruption, and cancel cooperatively. + +- **Concurrent, resumable downloads** powered by the `fast-down` engine (work-stealing, range requests). +- **Two entry points**: `download` (auto-resume when possible) and `resume` (hard error if it can't continue). +- **Event stream**: a single channel carries prefetch, per-worker progress, rename, and error events. +- **Cooperative cancellation**: cancelling mid-flight preserves the `.part` / `.fd` files so you can resume later. +- **Configurable**: threads, chunk size, write method (`Mmap` / `Std`), proxies, headers, retries, and more via `PartialConfig`. + +## Quick start + +```rust,ignore +use fast_down_api::{create_cancellation_token, create_channel, DownloadHandle, Event, PartialConfig}; +use std::path::PathBuf; +use url::Url; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // 1. Channel for progress / lifecycle events, plus a cancellation token. + let (tx, rx) = create_channel(); + let token = create_cancellation_token(); + + // 2. Configure the download. Every field is optional; unset fields + // fall back to Config::default(). (save_dir is required at runtime.) + let config = PartialConfig { + save_dir: Some(PathBuf::from("./downloads")), + overwrite: Some(true), + threads: Some(16), + ..Default::default() + }; + + // 3. Start the download. This spawns a detached task and returns at once. + let url = Url::parse("https://example.com/large-file.bin")?; + let handle = DownloadHandle::download(url, config, tx, token.clone()); + + // 4. Drain events until the task finishes or is cancelled. + while let Ok(event) = rx.recv().await { + match event { + Event::PushProgress(p) => println!("wrote range: {p:?}"), + Event::Renamed(path) => { + println!("done -> {path:?}"); + break; + } + Event::RenameFailed(e) => { + eprintln!("rename failed: {e}"); + break; + } + Event::ResumeError(e) => eprintln!("resume error: {e}"), + _ => {} + } + } + + // 5. Optionally await the spawned task (returns Err if it panicked). + handle.join().await?; + Ok(()) +} +``` + +### Resuming an interrupted download + +`resume` targets the existing `.part` file. If the download cannot be continued +(no `.fd` state, no range support, or the remote file changed) it emits +`Event::ResumeError` and **does not** silently restart — unlike `download`, +which auto-resumes when it can and otherwise falls back to a fresh download. + +```rust,ignore +let handle = DownloadHandle::resume( + "./downloads/large-file.bin.part", // the .part file from a previous run + url, + config, + tx, + token, +); +``` + +### Cancelling cooperatively + +```rust,ignore +token.cancel(); // stops fetching, keeps .part / .fd so you can resume later +``` + +## API overview + +| Item | Purpose | +| --------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| [`DownloadHandle::download`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.download) | Start a download; auto-resume when a valid `.fd` + `.part` exist, else fresh. | +| [`DownloadHandle::resume`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.resume) | Resume a specific `.part` file; hard-error (`Event::ResumeError`) if it can't. | +| [`DownloadHandle::join`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadHandle.html#method.join) | Await the detached task; errors if the worker panicked. | +| [`create_channel`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_channel.html) | Create the `(Tx, Rx)` event channel. | +| [`create_cancellation_token`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_cancellation_token.html) | Create a `CancellationToken` for cooperative cancellation. | +| [`Event`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.Event.html) | The event enum delivered over the channel. | +| [`PartialConfig`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.PartialConfig.html) | Layered, optional configuration for a download. | +| [`StateError`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.StateError.html) | Errors surfaced via `Event::ResumeError`. | + +## How resume works + +During a download the engine periodically persists a `.fd` state file next to +the `.part` partial file. That state records the byte ranges already written +(`downloaded_chunk`) and the remote file identity (`etag` / `last_modified` / +size). On the next run: + +1. `download` loads the `.fd`, validates it still matches the remote (size + identity), and — if the `.part` file is present — emits `Event::Resumed` and continues from the recorded offset. +2. If validation fails (remote changed) or there is no `.part`, it starts fresh. +3. `resume` applies the same checks but, instead of falling back, reports `Event::ResumeError`. + +Cancellation leaves both files in place, so a later `resume` (or `download`) can pick up exactly where it stopped. + +## License + +MIT — see [LICENSE](https://github.com/fast-down/core/blob/main/LICENSE). diff --git a/crates/fast-down-api/src/config.rs b/crates/fast-down-api/src/config.rs new file mode 100644 index 0000000..963e75b --- /dev/null +++ b/crates/fast-down-api/src/config.rs @@ -0,0 +1,224 @@ +use fast_down::{Merge, ProgressEntry, Proxy}; +use inherit_config::InheritConfig; +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, net::IpAddr, path::PathBuf, time::Duration}; + +/// File write method for downloaded data. +/// +/// - `Mmap`: memory-mapped I/O (fastest, default) +/// - `Std`: buffered standard file I/O +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub enum WriteMethod { + #[default] + Mmap, + Std, +} + +/// Configuration for a download task. +/// +/// All fields have sensible defaults; see [`Config::default`] for values. +#[derive(Debug, Clone, Serialize, Deserialize, InheritConfig)] +#[allow(clippy::struct_excessive_bools)] +pub struct Config { + /// 保存的文件夹 + pub save_dir: PathBuf, + + /// 文件名解析 + pub parse_filename: bool, + + /// 文件名 + pub filename: String, + + /// 用于在 prefetch 阶段生成占位文件名 + pub gid: String, + + /// Number of threads. Recommended: `32` / `16` / `8`. More threads does not always mean faster. + #[config(default = 32)] + pub threads: usize, + + /// Proxy setting. Supports https, http, and socks5 proxies. + pub proxy: Proxy, + + /// Custom request headers. + pub headers: HashMap, + + /// Minimum chunk size in bytes. Recommended: `8 * 1024 * 1024` + /// + /// - Chunks that are too small may cause heavy contention. + /// - When chunking is no longer possible, speculative mode is used. + #[config(default = 8 * 1024 * 1024)] + pub min_chunk_size: u64, + + /// Whether to ensure data is fully flushed to disk. Recommended: `false` + /// + /// Set to `true` only if you need to power off immediately after download. + pub sync_all: bool, + + /// Write buffer size in bytes. Recommended: `16 * 1024 * 1024` + /// + /// - Only effective for [`WriteMethod::Std`]. Reduces the number of `write` syscalls + /// by batching small writes into larger ones via `BufWriter`. + /// - Not used for [`WriteMethod::Mmap`], as the buffer is managed by the OS. + #[config(default = 16 * 1024 * 1024)] + pub write_buffer_size: usize, + + /// Cache high watermark in bytes. Recommended: `16 * 1024 * 1024` + /// + /// When the byte merge buffer reaches this size, a merge flush is triggered + /// to reduce the buffer to `cache_low_watermark` or below. + /// + /// - Only effective for [`WriteMethod::Std`]. + /// - Not used for [`WriteMethod::Mmap`]. + #[config(default = 16 * 1024 * 1024)] + pub cache_high_watermark: usize, + + /// Cache low watermark in bytes. Recommended: `8 * 1024 * 1024` + /// + /// After a merge flush, the byte merge buffer size is reduced to this level or below. + /// + /// - Only effective for [`WriteMethod::Std`]. + /// - Not used for [`WriteMethod::Mmap`]. + #[config(default = 8 * 1024 * 1024)] + pub cache_low_watermark: usize, + + /// Write queue capacity. Recommended: `10240` + /// + /// If download threads fill the write queue, backpressure is applied to + /// slow down downloads and prevent excessive memory usage. + #[config(default = 10240)] + pub write_queue_cap: usize, + + /// Default retry interval after a request failure. Recommended: `500ms` + /// + /// If the server returns a `Retry-After` header, that value takes precedence. + #[config(default = Duration::from_millis(500))] + pub retry_gap: Duration, + + /// Pull timeout. Recommended: `5s` + /// + /// If no bytes are received within `pull_timeout` after sending the request, + /// the connection is dropped and re-established. This helps TCP detect + /// congestion and can improve download speed. + #[config(default = Duration::from_secs(5))] + pub pull_timeout: Duration, + + /// Whether to accept invalid certificates (dangerous). Recommended: `false` + pub accept_invalid_certs: bool, + + /// Whether to accept invalid hostnames (dangerous). Recommended: `false` + pub accept_invalid_hostnames: bool, + + /// Write method. Recommended: [`WriteMethod::Mmap`] + /// + /// - [`WriteMethod::Mmap`] is fastest — it delegates writes to the OS, but: + /// 1. On 32-bit systems, the maximum file size is 4 GB, so it automatically + /// falls back to [`WriteMethod::Std`]. + /// 2. Mmap requires the file size to be known and byte-range support from + /// the server; when the `fast_download` flag (set during prefetch) is false, + /// it falls back to [`WriteMethod::Std`]. + /// 3. In rare cases, the OS may cache all data in memory and flush it all + /// at once after the download completes, causing a long post-download delay. + /// - [`WriteMethod::Std`] has the best compatibility. Out-of-order chunks are + /// re-ordered into sequential order by the cache layer before being written. + pub write_method: WriteMethod, + + /// Number of retries for fetching metadata. Recommended: `10`. Note: this is not + /// the retry count during download. + #[config(default = 10)] + pub retry_times: usize, + + /// Local IP addresses to bind for outgoing requests. Recommended: `Vec::new()` + /// + /// If you have multiple network interfaces, you can provide their IP addresses; + /// each time the puller is cloned (e.g. on retry or work-stealing) the next + /// address in the list is used. This may not always improve speed. + pub local_address: Vec, + + /// Maximum number of speculative workers. Recommended: `3` + /// + /// When the remaining chunk is smaller than `min_chunk_size` and cannot be split, + /// speculative mode is used. Up to `max_speculative` workers compete on the same + /// chunk to prevent the download from stalling near 99%. + #[config(default = 3)] + pub max_speculative: usize, + + /// Already downloaded chunks. Pass `Vec::new()` to download the entire file. + pub downloaded_chunk: Vec, + + /// Smoothing window for downloaded chunks in bytes. Recommended: `8 * 1024` + /// + /// Filters out small gaps in `downloaded_chunk` that are smaller than + /// `chunk_window` to reduce the number of HTTP requests. + #[config(default = 8 * 1024)] + pub chunk_window: u64, + + /// Maximum number of redirects. Recommended value: `20` + #[config(default = 20)] + pub max_redirects: usize, + + /// Enable cookie store. When `true`, the client will automatically save + /// `Set-Cookie` headers from responses and send matching cookies in + /// subsequent requests (including across redirects). + pub cookie_store: bool, + + /// 是否尝试断点续传，推荐值: `true` + #[config(default = true)] + pub resume: bool, + + /// 是否覆盖已存在的文件，推荐值: `false` + pub overwrite: bool, +} + +impl PartialConfig { + /// Merge a freshly-written byte range into this partial config's progress. + /// + /// The range is folded into `downloaded_chunk` (created if absent), keeping + /// it normalized and de-duplicated. This is the in-memory counterpart of + /// [`DownloadState::merge_progress`](crate::DownloadState::merge_progress): + /// callers use it to record progress before handing the config to + /// [`DownloadHandle::resume`](crate::DownloadHandle::resume). + pub fn merge_progress(&mut self, progress: ProgressEntry) { + self.downloaded_chunk + .get_or_insert_default() + .merge_progress(progress); + } +} + +#[cfg(test)] +#[allow(clippy::single_range_in_vec_init)] +mod tests { + use super::*; + + #[test] + fn merge_progress_none_to_some() { + let mut c = PartialConfig::default(); + assert_eq!(c.downloaded_chunk, None); + c.merge_progress(1u64..5); + assert_eq!(c.downloaded_chunk, Some(vec![1u64..5])); + } + + #[test] + fn merge_progress_coalesces() { + let mut c = PartialConfig::default(); + c.merge_progress(1u64..5); + c.merge_progress(5u64..10); + c.merge_progress(10u64..20); + assert_eq!(c.downloaded_chunk, Some(vec![1u64..20])); + } + + #[test] + fn merge_progress_empty_is_noop() { + let mut c = PartialConfig::default(); + c.merge_progress(1u64..5); + c.merge_progress(3u64..3); + assert_eq!(c.downloaded_chunk, Some(vec![1u64..5])); + } + + #[test] + fn merge_progress_disjoint() { + let mut c = PartialConfig::default(); + c.merge_progress(1u64..5); + c.merge_progress(10u64..20); + assert_eq!(c.downloaded_chunk, Some(vec![1u64..5, 10u64..20])); + } +} diff --git a/crates/fast-down-api/src/core/download/mod.rs b/crates/fast-down-api/src/core/download/mod.rs new file mode 100644 index 0000000..1fecfbb --- /dev/null +++ b/crates/fast-down-api/src/core/download/mod.rs @@ -0,0 +1,259 @@ +use crate::core::download::overwrite::OverwriteOption; +use crate::utils::ForceSendExt; +use crate::{DownloadState, Event, StateError}; +use crate::{PartialConfig, Tx, prefetch, tx_err, utils::gen_path}; +use fast_down::handle::SharedHandle; +use inherit_config::ConfigLayer; +use overwrite::overwrite; +use path_helper::IterStemExt; +use std::path::Path; +use std::sync::Arc; +use tokio::fs::{self, OpenOptions}; +use tokio::task::JoinError; +use tokio_util::sync::CancellationToken; +use url::Url; + +mod overwrite; +mod pipeline; + +fn open_existing() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(false); + o +} +fn open_create() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create(true); + o +} +fn open_create_new() -> OpenOptions { + let mut o = OpenOptions::new(); + o.read(true).write(true).truncate(false).create_new(true); + o +} + +/// A handle to a spawned download task. +/// +/// [`download`](Self::download) and [`resume`](Self::resume) spawn a detached +/// background task and return a `DownloadHandle`. Progress and lifecycle events +/// are delivered through the [`Tx`](crate::Tx) channel you pass in; call +/// [`join`](Self::join) to await the task's completion (it errors if the task +/// panicked). +#[derive(Debug, Clone)] +pub struct DownloadHandle { + handle: SharedHandle<()>, +} + +impl DownloadHandle { + /// Returns the join of this [`DownloadHandle`]. + /// + /// # Errors + /// + /// This function will return an error if download thread panic + pub async fn join(&self) -> Result<(), Arc> { + self.handle.join().await + } + + /// Start a download, resuming automatically when possible. + /// + /// Spawns a detached task. The task first `prefetch`es metadata, then: + /// - if `config.resume` is enabled, a valid `.fd` state file and its `.part` + /// exist, and the remote file still matches ([`crate::Event::Resumed`]), it + /// continues from the recorded offset; + /// - otherwise it starts a fresh download. + /// + /// Unlike [`resume`](Self::resume), a failure to resume here is **not** an + /// error: the task silently falls back to a full re-download and emits the + /// normal event stream. Pass a + /// [`CancellationToken`](crate::create_cancellation_token) to cancel; on + /// cancellation the `.part`/`.fd` files are preserved so a later + /// [`resume`](Self::resume) can continue. + #[must_use] + pub fn download( + url: Url, + partial_config: PartialConfig, + tx: Tx, + token: CancellationToken, + ) -> Self { + let handle = tokio::spawn( + async move { + let token2 = token.clone(); + let opt = token + .run_until_cancelled( + async move { download(url, partial_config, tx, token2).await }, + ) + .await + .flatten(); + if let Some(opt) = opt { + overwrite(opt).await; + } + } + .force_send(), + ); + Self { + handle: SharedHandle::new(handle), + } + } + + /// Resume a previously interrupted download from its `.part` file. + /// + /// Spawns a detached task pinned to `tmp_path`. If the download cannot be + /// continued — the `.fd` state file is missing, the server does not support + /// range requests, or the remote file changed — the task emits + /// [`Event::ResumeError`](crate::Event::ResumeError) and returns **without** + /// falling back to a full re-download. This is the "hard error" counterpart + /// to [`download`](Self::download). + /// + /// If `tmp_path` itself does not exist, the call falls back to + /// [`download`](Self::download) (a fresh download under a unique name). + pub fn resume( + tmp_path: impl AsRef, + url: Url, + partial_config: PartialConfig, + tx: Tx, + token: CancellationToken, + ) -> Self { + let tmp_path = tmp_path.as_ref().to_path_buf(); + let handle = tokio::spawn( + async move { + let token2 = token.clone(); + let opt = Box::pin(token.run_until_cancelled(async move { + resume(&tmp_path, url, partial_config, tx, token2).await + })) + .await + .flatten(); + if let Some(opt) = opt { + overwrite(opt).await; + } + } + .force_send(), + ); + Self { + handle: SharedHandle::new(handle), + } + } +} + +async fn download( + url: Url, + partial_config: PartialConfig, + tx: Tx, + token: CancellationToken, +) -> Option { + let config = partial_config.clone().build(); + let (info, resp) = prefetch(&url, &config, &tx).await?; + let can_resume = config.resume && info.fast_download; + + let origin_path = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError, None); + + if config.overwrite { + let cfg_path = origin_path.with_added_extension("fd"); + let tmp_path = origin_path.with_added_extension("part"); + + let state = if can_resume + && let Ok(mut s) = DownloadState::load(&cfg_path).await + && s.validate(&info).is_ok() + && fs::try_exists(&tmp_path).await.unwrap_or(false) + { + s.merge_config(&partial_config); + let _ = tx.send(Event::Resumed { + config_path: cfg_path, + progress: s.get_progress(), + size: info.size, + }); + s + } else { + tx_err!( + open_create().open(tmp_path).await, + tx, + BuildPusherError, + None + ); + DownloadState::new(&url, &info, &partial_config, &cfg_path) + }; + return Some(OverwriteOption { + state, + final_path: origin_path, + info, + resp, + tx, + token, + }); + } + + for base_path in origin_path.iter_stem() { + let tmp_path = base_path.with_added_extension("part"); + let cfg_path = base_path.with_added_extension("fd"); + + let state = if can_resume + && let Ok(mut s) = DownloadState::load(&cfg_path).await + && s.validate(&info).is_ok() + && fs::try_exists(&tmp_path).await.unwrap_or(false) + { + s.merge_config(&partial_config); + let _ = tx.send(Event::Resumed { + config_path: cfg_path, + progress: s.get_progress(), + size: info.size, + }); + s + } else if open_create_new().open(&tmp_path).await.is_ok() { + DownloadState::new(&url, &info, &partial_config, &cfg_path) + } else { + continue; + }; + return Some(OverwriteOption { + state, + final_path: origin_path, + info, + resp, + tx, + token, + }); + } + unreachable!() +} + +async fn resume( + tmp_path: &Path, + url: Url, + mut partial_config: PartialConfig, + tx: Tx, + token: CancellationToken, +) -> Option { + partial_config.overwrite = Some(false); + let tmp_exits = fs::try_exists(tmp_path).await.unwrap_or(false); + if !tmp_exits { + partial_config.resume = Some(false); + return download(url, partial_config, tx, token).await; + } + + let cfg_path = tmp_path.with_extension("fd"); + let mut state = tx_err!(DownloadState::load(&cfg_path).await, tx, ResumeError, None); + + partial_config.resume = Some(true); + let config = partial_config.clone().build(); + let (info, resp) = prefetch(&url, &config, &tx).await?; + if !info.fast_download { + let _ = tx.send(Event::ResumeError(StateError::NotResumable(info, resp))); + return None; + } + tx_err!(state.validate(&info), tx, ResumeError, None); + + state.merge_config(&partial_config); + let _ = tx.send(Event::Resumed { + config_path: cfg_path, + progress: state.get_progress(), + size: info.size, + }); + + let final_path = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError, None); + Some(OverwriteOption { + state, + final_path, + info, + resp, + tx, + token, + }) +} diff --git a/crates/fast-down-api/src/core/download/overwrite.rs b/crates/fast-down-api/src/core/download/overwrite.rs new file mode 100644 index 0000000..ee87030 --- /dev/null +++ b/crates/fast-down-api/src/core/download/overwrite.rs @@ -0,0 +1,203 @@ +//! Full-pipeline download driver: persists state, runs the engine, then renames +//! the `.part` file into place. +//! +//! [`overwrite`] is the shared core of both [`crate::DownloadHandle::download`] +//! and [`crate::DownloadHandle::resume`]. It takes a fully-prepared +//! [`OverwriteOption`] (state + paths + prefetch result + channel + token), +//! builds the pull/push pipeline, drives [`fast_down::multi::download_multi`] or +//! [`fast_down::single::download_single`], forwards engine events to the public +//! [`crate::Event`] stream, periodically saves progress, and on success renames +//! the `.part` file to its final destination (or a unique variant when +//! `overwrite` is disabled). +use crate::{ + DownloadState, Event, PartialConfig, Tx, core::download::pipeline::build_pipeline, tx_err, +}; +use fast_down::{ + AnyError, DownloadResult, UrlInfo, + fast_puller::FastDownPuller, + http::HttpError, + invert, + multi::{TokioExecutor, download_multi}, + reqwest::SmartRedirectClient, + single::download_single, +}; +use inherit_config::ConfigLayer; +use path_helper::tokio::gen_unique_path; +use reqwest::Response; +use std::{ + path::PathBuf, + time::{Duration, Instant}, +}; +use tokio::fs; +use tokio_util::sync::CancellationToken; + +/// Fully-prepared inputs for [`overwrite`]. +/// +/// Callers build this once prefetch has succeeded and the [`DownloadState`] is +/// ready; [`overwrite`] then owns it and runs the download to completion. +pub struct OverwriteOption { + /// The download state backing resume (progress, identity, `.fd` path). + pub state: DownloadState, + /// Intended final destination of the file (before unique-path adjustment). + pub final_path: PathBuf, + /// Prefetch metadata for the remote file (size, identity, range support). + pub info: UrlInfo, + /// The prefetch HTTP response, reused to seed the first range request. + pub resp: Response, + /// Channel to forward public [`crate::Event`]s to the consumer. + pub tx: Tx, + /// Cancellation token; cancelling stops fetching and leaves `.part`/`.fd`. + pub token: CancellationToken, +} + +/// Run a complete download: persist state, drive the engine, rename into place. +/// +/// This is the shared core of [`crate::DownloadHandle::download`] and +/// [`crate::DownloadHandle::resume`]. It: +/// 1. Saves the `.fd` state up front. +/// 2. Builds the pull/push pipeline for the `.part` file. +/// 3. Emits [`crate::Event::Start`] and runs `download_multi` (fast downloads) +/// or `download_single` (single-stream) according to `info.fast_download`. +/// 4. Forwards every engine event to the public [`crate::Event`] channel and +/// merges `PushProgress` ranges into the state. +/// 5. Periodically (≈1s) re-saves the `.fd` so progress survives interruption. +/// 6. On success, renames `.part` to `final_path` (or a unique variant when +/// `overwrite` is disabled) and emits [`crate::Event::Renamed`]. +/// +/// If the token is cancelled or the download did not complete, the `.part` and +/// `.fd` files are left in place so a later resume can continue. +#[allow(clippy::too_many_lines)] +pub async fn overwrite(option: OverwriteOption) { + let OverwriteOption { + mut state, + final_path, + info, + resp, + tx, + token, + } = option; + tx_err!(state.store().await, tx, StateSaveError); + + let inner = state.clone().build(); + let tmp_path = state.tmp_path(); + let url = &inner.url; + let config = &inner.config; + + let pipeline = build_pipeline(url, config, &info, resp, &tmp_path, &tx, &token).await; + let Some((puller, pusher)) = pipeline else { + return; + }; + + let _ = tx.send(Event::Start { + tmp_path: tmp_path.clone(), + config_path: state.config_path.clone(), + parsed_config: state.config.clone().unwrap_or_default(), + }); + + let res = if info.fast_download { + download_multi( + puller, + pusher, + fast_down::multi::DownloadOptions { + download_chunks: invert( + config.downloaded_chunk.iter().cloned(), + info.size, + config.chunk_window, + ), + concurrent: config.threads, + retry_gap: config.retry_gap, + pull_timeout: config.pull_timeout, + push_queue_cap: config.write_queue_cap, + min_chunk_size: config.min_chunk_size, + max_speculative: config.max_speculative, + }, + ) + } else { + download_single( + puller, + pusher, + fast_down::single::DownloadOptions { + retry_gap: config.retry_gap, + push_queue_cap: config.write_queue_cap, + }, + ) + }; + let _guard = abort_ctrl(&token, &res); + + let mut store_time = Instant::now(); + while let Ok(e) = res.event_chain().recv().await { + if let fast_down::Event::PushProgress(range) = &e { + state.merge_progress(range.clone()); + } + let _ = match e { + fast_down::Event::Pulling(id) => tx.send(Event::Pulling(id)), + fast_down::Event::PullError(id, e) => tx.send(Event::PullError(id, anyhow::anyhow!(e))), + fast_down::Event::PullTimeout(id) => tx.send(Event::PullTimeout(id)), + fast_down::Event::PullProgress(id, range) => tx.send(Event::PullProgress(id, range)), + fast_down::Event::Pushing(id, range) => tx.send(Event::Pushing(id, range)), + fast_down::Event::PushError(id, range, e) => { + tx.send(Event::PushError(id, range, anyhow::anyhow!(e))) + } + fast_down::Event::PushProgress(range) => tx.send(Event::PushProgress(range)), + fast_down::Event::Flushing => tx.send(Event::Flushing), + fast_down::Event::FlushError(e) => tx.send(Event::FlushError(anyhow::anyhow!(e))), + fast_down::Event::Finished(id) => tx.send(Event::Finished(id)), + }; + if store_time.elapsed() > Duration::from_secs(1) { + if let Err(e) = state.store().await { + let _ = tx.send(Event::StateSaveError(e)); + } + store_time = Instant::now(); + } + } + + if let Err(e) = res.join().await { + let _ = tx.send(Event::JoinError(e)); + } + let download_complete = info.size == 0 + || matches!(&state.config, Some(PartialConfig { downloaded_chunk: Some(x), .. }) if x.len() == 1 && x[0] == (0..info.size)); + if token.is_cancelled() || !download_complete { + if let Err(e) = state.store().await { + let _ = tx.send(Event::StateSaveError(e)); + } + return; + } + + let final_path = if config.overwrite { + final_path + } else { + tx_err!(gen_unique_path(final_path).await, tx, GenPathError) + }; + if let Err(e) = fs::rename(tmp_path, &final_path).await { + if !config.overwrite { + let _ = fs::remove_file(&final_path).await; + } + let _ = tx.send(Event::RenameFailed(e)); + return; + } + let _ = fs::remove_file(&state.config_path).await; + let _ = tx.send(Event::Renamed(final_path)); +} + +struct Guard(tokio::task::JoinHandle<()>); +impl Drop for Guard { + fn drop(&mut self) { + self.0.abort(); + } +} + +fn abort_ctrl(token: &CancellationToken, res: &MyDownloadResult) -> Guard { + let token = token.clone(); + let res = res.clone(); + let handle = tokio::spawn(async move { + token.cancelled().await; + res.abort(); + }); + Guard(handle) +} + +type MyDownloadResult = DownloadResult< + TokioExecutor>, + HttpError, + Box, +>; diff --git a/crates/fast-down-api/src/core/download/pipeline.rs b/crates/fast-down-api/src/core/download/pipeline.rs new file mode 100644 index 0000000..b132b59 --- /dev/null +++ b/crates/fast-down-api/src/core/download/pipeline.rs @@ -0,0 +1,96 @@ +//! Builds the pull/push pipeline shared by `download` and `resume`. +//! +//! [`build_pipeline`] constructs a [`FastDownPuller`] (network side) and a +//! [`BoxPusher`] (file side) for the `.part` file, choosing the memory-mapped +//! writer on 64-bit targets when the server supports fast (resumable) downloads +//! and `Mmap` writing is configured, and the buffered/cache writer otherwise. +use crate::{Config, Event, Tx, WriteMethod, core::download::open_existing, utils::build_header}; +use fast_down::{ + BoxPusher, UrlInfo, + fast_puller::{FastDownPuller, FastDownPullerOptions}, + file::{CacheFilePusher, MmapFilePusher}, +}; +use parking_lot::Mutex; +use reqwest::Response; +use std::{path::Path, sync::Arc}; +use tokio_util::sync::CancellationToken; +use url::Url; + +/// Construct the (puller, pusher) pipeline for a `.part` file. +/// +/// Returns `None` (after forwarding the failure as a public +/// [`crate::Event`]) if the HTTP client or the output file cannot be created, +/// or if `token` is cancelled before construction finishes. +/// +/// * `url` / `config` drive the puller (headers, proxy, cert handling, range +/// identity, local bind address, redirect limit). +/// * `info` supplies the file identity used for range validation and selects +/// the writer: on 64-bit targets a resumable `info.fast_download` download +/// with [`WriteMethod::Mmap`] uses [`MmapFilePusher`]; otherwise +/// [`CacheFilePusher`] (buffered + out-of-order reordering). +/// * `resp` is the prefetch response, reused to seed the first range request +/// without an extra round-trip. +/// * `path` is the `.part` file; `tx` receives error events; `token` makes +/// construction cancellable. +pub async fn build_pipeline( + url: &Url, + config: &Config, + info: &UrlInfo, + resp: Response, + path: &Path, + tx: &Tx, + token: &CancellationToken, +) -> Option<(FastDownPuller, BoxPusher)> { + let resp = Some(Arc::new(Mutex::new(Some(resp)))); + let built = token + .run_until_cancelled(async move { + let puller = FastDownPuller::new(FastDownPullerOptions { + url: url.clone(), + headers: build_header(&config.headers).into(), + proxy: config.proxy.as_deref(), + accept_invalid_certs: config.accept_invalid_certs, + accept_invalid_hostnames: config.accept_invalid_hostnames, + cookie_store: config.cookie_store, + file_id: info.file_id.clone(), + resp, + available_ips: config.local_address.clone().into(), + max_redirects: config.max_redirects, + }) + .map_err(Event::BuildClientError)?; + + let file = open_existing() + .open(path) + .await + .map_err(Event::BuildPusherError)?; + let pusher = if cfg!(target_pointer_width = "64") + && info.fast_download + && config.write_method == WriteMethod::Mmap + { + MmapFilePusher::new(&file, info.size, config.sync_all) + .await + .map(BoxPusher::new) + } else { + CacheFilePusher::new( + file, + info.size, + config.sync_all, + config.cache_high_watermark, + config.cache_low_watermark, + config.write_buffer_size, + ) + .await + .map(BoxPusher::new) + } + .map_err(Event::BuildPusherError)?; + Ok::<_, Event>((puller, pusher)) + }) + .await; + match built { + Some(Ok(b)) => Some(b), + Some(Err(e)) => { + let _ = tx.send(e); + None + } + None => None, + } +} diff --git a/crates/fast-down-api/src/core/mod.rs b/crates/fast-down-api/src/core/mod.rs new file mode 100644 index 0000000..625df38 --- /dev/null +++ b/crates/fast-down-api/src/core/mod.rs @@ -0,0 +1,7 @@ +mod download; +mod prefetch; +mod state; + +pub use download::*; +pub use prefetch::*; +pub use state::*; diff --git a/crates/fast-down-api/src/core/prefetch.rs b/crates/fast-down-api/src/core/prefetch.rs new file mode 100644 index 0000000..1dad30d --- /dev/null +++ b/crates/fast-down-api/src/core/prefetch.rs @@ -0,0 +1,34 @@ +use crate::{Config, Event, Tx, tx_err, utils::build_header}; +use fast_down::{UrlInfo, fast_puller::build_client, http::Prefetch}; +use reqwest::Response; +use url::Url; + +pub async fn prefetch(url: &Url, config: &Config, tx: &Tx) -> Option<(UrlInfo, Response)> { + let client = build_client( + build_header(&config.headers), + config.proxy.as_deref(), + config.accept_invalid_certs, + config.accept_invalid_hostnames, + config.cookie_store, + config.local_address.first().copied(), + config.max_redirects, + ); + let client = tx_err!(client, tx, BuildClientError, None); + let mut retry_count = 0; + loop { + match client.prefetch(url.clone()).await { + Ok(t) => { + let _ = tx.send(Event::Prefetch(t.0.clone())); + break Some(t); + } + Err((e, t)) => { + let _ = tx.send(Event::PrefetchError(e)); + retry_count += 1; + if retry_count >= config.retry_times { + return None; + } + tokio::time::sleep(t.unwrap_or(config.retry_gap)).await; + } + } + } +} diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs new file mode 100644 index 0000000..3f24dd2 --- /dev/null +++ b/crates/fast-down-api/src/core/state.rs @@ -0,0 +1,271 @@ +use crate::{Config, PartialConfig}; +use fast_down::{FileId, ProgressEntry, UrlInfo}; +use inherit_config::{ConfigLayer, InheritConfig}; +use path_helper::tokio::safe_replace; +use reqwest::Response; +use std::{ + ops::Deref, + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::fs; +use url::Url; + +/// Errors that can occur when an explicit [`DownloadHandle::resume`](crate::DownloadHandle::resume) +/// cannot continue an interrupted download. +/// +/// These are surfaced through [`crate::Event::ResumeError`] so the caller is notified +/// instead of silently falling back to a full re-download. +#[derive(Debug, thiserror::Error)] +#[allow(clippy::large_enum_variant)] +pub enum StateError { + /// Failed to open or read the `.fd` state file from disk. + /// + /// This usually means the file is missing or unreadable (permission error, + /// removed by another process, etc.). + #[error("failed to open .fd state file: {0}")] + Open(std::io::Error), + /// Failed to persist the `.fd` state file to disk. + /// + /// The state was computed but could not be written (disk full, permission + /// error, etc.). + #[error("failed to save .fd state file: {0}")] + Save(std::io::Error), + /// Failed to deserialize the `.fd` state file. + /// + /// The file was read but is not valid TOML, or its schema no longer matches + /// the current [`DownloadStateInner`] definition. + #[error("failed to decode .fd state file: {0}")] + Decode(#[from] toml::de::Error), + /// Failed to serialize the download state into the `.fd` state file. + /// + /// The state could not be encoded as TOML (e.g. a field holds a value that + /// has no valid TOML representation). + #[error("failed to encode .fd state file: {0}")] + Encode(#[from] toml::ser::Error), + /// The remote file changed (size / etag / last-modified mismatch), so resuming would corrupt the output. + #[error( + "remote file changed, cannot resume\n local: size={local_file_size}, id={local_file_id:?}\n remote: size={remote_file_size}, id={remote_file_id:?}" + )] + FileChanged { + local_file_id: FileId, + local_file_size: u64, + remote_file_id: FileId, + remote_file_size: u64, + }, + /// The server does not support resumable (range) downloads. + #[error( + "server does not support resumable download\n url_info: {:?}\n url: {}\n status: {}\n headers: {:?}", + .0, .1.url(), .1.status(), .1.headers() + )] + NotResumable(UrlInfo, Response), +} + +/// Full (resolved) download state that is serialized into the `.fd` file. +/// +/// `DownloadStateInner` is the non-partial form of the persisted state: every +/// field is present. It is encoded to TOML by [`DownloadState::store`] and read +/// back as the generated partial [`PartialDownloadStateInner`] on resume. The +/// `size` field is recorded for resume validation; older `.fd` files omit it and +/// read back as `None` in the partial form. +#[derive(Debug, Clone, InheritConfig)] +pub struct DownloadStateInner { + #[config(default = Url::parse("about:blank").unwrap())] + pub url: Url, + pub etag: Option>, + pub last_modified: Option>, + #[config(nest)] + pub config: Config, + /// Total file size recorded at save time, compared against the server + /// `UrlInfo.size` during resume validation. + /// + /// An older `.fd` written without this field reads back as `None`, which + /// makes `validate` fail and safely fall back to a full re-download. + pub size: u64, +} + +/// On-disk state for an in-progress download, backing resume support. +/// +/// `DownloadState` pairs a [`PartialDownloadStateInner`] (the persisted config, +/// whose fields may be absent if the `.fd` file predates them) with the path of +/// the `.fd` file and a dirty flag. It is the bridge between a saved `.fd` file +/// and a fresh [`crate::PartialConfig`] handed to +/// [`crate::DownloadHandle::resume`]: [`DownloadState::merge_config`] folds the +/// loaded progress into the new request so a resumed download continues from the +/// correct byte offset instead of restarting from zero. +/// +/// `DownloadState` derefs to [`PartialDownloadStateInner`], so the saved `url`, +/// `etag`, `last_modified`, `config` and `size` are reachable directly. +#[derive(Debug)] +pub struct DownloadState { + inner: PartialDownloadStateInner, + is_dirty: bool, + pub config_path: PathBuf, +} + +impl DownloadState { + /// Build a fresh, dirty download state from the initial prefetch metadata. + /// + /// `url` and `url_info` come from the prefetch step, `config` is the + /// caller-provided [`PartialConfig`], and `config_path` is where the `.fd` + /// file will be written. The returned state is marked dirty so the first + /// [`DownloadState::store`] persists it. + #[must_use] + pub fn new(url: &Url, url_info: &UrlInfo, config: &PartialConfig, config_path: &Path) -> Self { + Self { + inner: PartialDownloadStateInner { + url: Some(url.clone()), + etag: Some(url_info.file_id.etag.clone()), + last_modified: Some(url_info.file_id.last_modified.clone()), + config: Some(config.clone()), + size: Some(url_info.size), + }, + is_dirty: true, + config_path: config_path.to_path_buf(), + } + } + + /// Load a download state from disk. + /// + /// # Errors + /// Returns an error if the file cannot be read or deserialized. + #[allow(clippy::result_large_err)] + pub async fn load(config_path: &Path) -> Result { + let inner = fs::read(&config_path).await.map_err(StateError::Open)?; + let inner: PartialDownloadStateInner = toml::from_slice(&inner)?; + Ok(Self { + inner, + is_dirty: false, + config_path: config_path.to_path_buf(), + }) + } + + /// Persist the download state to disk when it is dirty. + /// + /// # Errors + /// Returns an error if serializing or writing the state fails. + #[allow(clippy::result_large_err)] + pub async fn store(&mut self) -> Result<(), StateError> { + if self.is_dirty { + self.inner + .simplify_from(&PartialDownloadStateInner::default()); + let inner = toml::to_string_pretty(&self.inner)?; + safe_replace(&self.config_path, inner.as_bytes()) + .await + .map_err(StateError::Save)?; + self.is_dirty = false; + } + Ok(()) + } + + /// Apply a mutation to the inner partial state and mark it dirty. + /// + /// Any change made through `cb` (e.g. updating `etag`/`last_modified` or the + /// nested `config`) schedules the next [`DownloadState::store`]. + pub fn update(&mut self, cb: impl FnOnce(&mut PartialDownloadStateInner)) { + cb(&mut self.inner); + self.is_dirty = true; + } + + /// Check whether the server-side file is still the same one this state was + /// saved for, so a resumed download continues from the correct offset. + /// + /// The comparison requires the recorded `size` to match and (unless both + /// sides are missing identity headers) the `FileId` (`etag` + + /// `last_modified`) to be equal. Resumable downloads also require the + /// server to support range requests. + /// + /// # Errors + #[allow(clippy::result_large_err)] + pub fn validate(&self, info: &UrlInfo) -> Result<(), StateError> { + let local_file_id = self.file_id(); + let local_file_size = self.size.unwrap_or(0); + let is_same = local_file_size == info.size && local_file_id == info.file_id; + if is_same { + Ok(()) + } else { + Err(StateError::FileChanged { + local_file_id, + local_file_size, + remote_file_id: info.file_id.clone(), + remote_file_size: info.size, + }) + } + } + + /// Return the list of byte ranges already written to the `.part` file. + /// + /// This is the authoritative on-disk progress. It is empty for a brand new + /// download and grows as [`DownloadState::merge_progress`] is called. + #[must_use] + pub fn get_progress(&self) -> Vec { + self.config + .as_ref() + .and_then(|c| c.downloaded_chunk.clone()) + .unwrap_or_default() + } + + /// Merge a freshly-written byte range into the recorded progress. + /// + /// The progress list is the authoritative set of on-disk ranges; new ranges + /// are merged, de-duplicated and normalized. Marks the state dirty. + pub fn merge_progress(&mut self, range: ProgressEntry) { + self.inner + .config + .get_or_insert_default() + .merge_progress(range); + self.is_dirty = true; + } + + /// Fold a freshly-built [`PartialConfig`] into this loaded state for resume. + /// + /// This is the key bridge that preserves download progress across a resume: + /// the ranges already recorded in `self.config.downloaded_chunk` are merged + /// into `partial_config` first, so that even if `partial_config` starts from + /// an empty progress list the resumed download keeps the already-downloaded + /// bytes. `partial_config` is then layered on top via + /// `inherit_from`, and the merged result replaces the stored + /// config. Marks the state dirty. + pub fn merge_config(&mut self, partial_config: &PartialConfig) { + if let Some(config) = &self.config { + let mut pc = partial_config.clone(); + if let Some(downloaded_chunk) = &config.downloaded_chunk { + for i in downloaded_chunk { + pc.merge_progress(i.clone()); + } + } + pc.inherit_from(config); + self.inner.config = Some(pc); + self.is_dirty = true; + } + } + + /// Reconstruct the identity of the file this state was saved for. + /// + /// Returns a [`FileId`] from the stored `etag` / `last_modified`. Missing + /// headers collapse to `None`, which makes [`DownloadState::validate`] treat + /// "no identity on either side" as a match. + #[must_use] + pub fn file_id(&self) -> FileId { + FileId { + etag: self.etag.clone().flatten(), + last_modified: self.last_modified.clone().flatten(), + } + } + + /// Path of the partial (`.part`) output file paired with this state. + /// + /// Derived from `config_path` by swapping the extension to `.part`. + #[must_use] + pub fn tmp_path(&self) -> PathBuf { + self.config_path.with_extension("part") + } +} + +impl Deref for DownloadState { + type Target = PartialDownloadStateInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs new file mode 100644 index 0000000..48d59a0 --- /dev/null +++ b/crates/fast-down-api/src/event.rs @@ -0,0 +1,107 @@ +use crate::{PartialConfig, StateError}; +use fast_down::{ProgressEntry, UrlInfo, WorkerId, reqwest::ReqwestResponseError}; +use std::{path::PathBuf, sync::Arc}; + +/// Events emitted by a download run, consumed through the crossfire channel +/// returned by [`crate::create_channel`]. +/// +/// Events cover the full lifecycle: prefetch ([`Event::Prefetch`]), pipeline +/// setup ([`Event::Start`]), per-worker fetch/write progress +/// ([`Event::Pulling`] … [`Event::Finished`]), resume ([`Event::Resumed`], +/// [`Event::ResumeError`]), and completion ([`Event::Renamed`]). Error variants +/// (`*Error`) report failures without aborting the stream, so a consumer can +/// decide whether to retry, cancel, or surface them in a UI. +#[allow(clippy::large_enum_variant)] +pub enum Event { + /// Emitted after the prefetch step resolves the remote file's metadata. + /// + /// Carries the [`UrlInfo`] (size, identity headers, range support) so a + /// caller can inspect the remote before or while the download proceeds. + Prefetch(UrlInfo), + /// The prefetch request failed (server unreachable, non-2xx response, etc.). + /// + /// Fatal for the download: without [`UrlInfo`] the engine cannot plan ranges. + PrefetchError(ReqwestResponseError), + /// Failed to compute the output / `.part` / `.fd` paths. + /// + /// For example the target directory is not writable or the file name is + /// invalid on this platform. + GenPathError(std::io::Error), + /// Persisting the `.fd` state file failed (see [`StateError`]). + /// + /// Non-fatal for the current run, but the download can no longer be resumed + /// reliably if it is interrupted afterwards. + StateSaveError(StateError), + /// Building the HTTP client failed (TLS / backend initialization, etc.). + BuildClientError(reqwest::Error), + /// Creating the output sink — opening the `.part` file — failed. + BuildPusherError(std::io::Error), + /// A spawned download worker task panicked or was cancelled. + /// + /// The engine joins its worker tasks; if a join handle reports an error it is + /// surfaced here rather than silently dropping the panic. + JoinError(Arc), + /// The final rename of the `.part` file to its destination failed. + /// + /// The success counterpart is [`Event::Renamed`]. The bytes are already on + /// disk under the `.part` name, so they can still be resumed or retried. + RenameFailed(std::io::Error), + /// Emitted after the `.part` file is successfully renamed to its final + /// destination. Carries the actual landing path, which in unique mode may + /// differ from the originally-planned name (e.g. `xxx (1).mp4`) when the + /// target got occupied during the download. + Renamed(PathBuf), + /// Emitted once the pipeline is set up and writing is about to begin. + /// + /// Carries the `.part` path, the `.fd` state-file path, and the resolved + /// [`PartialConfig`] actually used for this run (after merging any resumed + /// progress and applying defaults). + Start { + tmp_path: PathBuf, + config_path: PathBuf, + parsed_config: PartialConfig, + }, + /// Emitted when a download resumes from a previously-saved state, before + /// [`Event::Start`]. Carries the progress that will be continued from and + /// the total file size, so a UI can show e.g. "resuming from 42%". + Resumed { + config_path: PathBuf, + progress: Vec, + size: u64, + }, + /// Emitted when an explicit `resume()` call cannot continue the download. + /// Unlike `download()` (which silently falls back to a full re-download), + /// `resume()` reports the failure so the caller can decide what to do. + ResumeError(StateError), + + /// Worker `id` started fetching its assigned byte range from the network. + Pulling(WorkerId), + /// Worker `id` failed to fetch its assigned range (network / decode error). + PullError(WorkerId, anyhow::Error), + /// Worker `id`'s fetch exceeded its time budget and was aborted. + PullTimeout(WorkerId), + /// Worker `id` pulled some bytes into memory. + /// + /// `ProgressEntry` describes the contiguous range that just arrived from the + /// network but is not yet written to disk. + PullProgress(WorkerId, ProgressEntry), + /// Worker `id` is handing a pulled range to the sink (writing it to `.part`). + /// + /// `ProgressEntry` is the range being written. + Pushing(WorkerId, ProgressEntry), + /// Worker `id` failed to write `ProgressEntry` to the sink. + PushError(WorkerId, ProgressEntry, anyhow::Error), + /// A range was written to the sink. + /// + /// `ProgressEntry` is the range persisted (at least to the OS page cache) by + /// this write. This drives the progress bar and the resume bookkeeping; it is + /// intentionally emitted before `fsync`, so it reflects "written", not + /// "durably flushed". + PushProgress(ProgressEntry), + /// The sink is being flushed and synced to the `.part` file. + Flushing, + /// Flushing / syncing the sink failed. + FlushError(anyhow::Error), + /// Worker `id` completed its assigned range and exited. + Finished(WorkerId), +} diff --git a/crates/fast-down-api/src/lib.rs b/crates/fast-down-api/src/lib.rs new file mode 100644 index 0000000..cb07433 --- /dev/null +++ b/crates/fast-down-api/src/lib.rs @@ -0,0 +1,39 @@ +#![doc = include_str!("../README.md")] + +mod config; +mod core; +mod event; +pub(crate) mod utils; + +pub use config::*; +pub use core::*; +pub use event::*; + +pub use fast_down; + +use tokio_util::sync::CancellationToken; + +/// Sender half of the event channel, used to push [`Event`]s from the download task. +pub type Tx = crossfire::MTx>; +/// Receiver half of the event channel, used to receive [`Event`]s from the download task. +pub type Rx = crossfire::MAsyncRx>; + +/// Create a new unbounded event channel for receiving download progress events. +/// +/// Returns a sender (`Tx`) and receiver (`Rx`) pair. +#[must_use] +pub fn create_channel() -> (Tx, Rx) { + crossfire::mpmc::unbounded_async() +} + +/// Create a new cancellation token for use with download tasks. +/// +/// Pass the token to [`DownloadHandle::download`] or [`DownloadHandle::resume`] +/// to cancel the download at any time. Cancellation is cooperative: the running +/// task stops fetching, leaves the `.part`/`.fd` files in place, and returns +/// without renaming — so a later [`DownloadHandle::resume`] call can continue +/// from where it stopped. +#[must_use] +pub fn create_cancellation_token() -> CancellationToken { + CancellationToken::new() +} diff --git a/crates/fast-down-api/src/utils/build_header.rs b/crates/fast-down-api/src/utils/build_header.rs new file mode 100644 index 0000000..3b74eaf --- /dev/null +++ b/crates/fast-down-api/src/utils/build_header.rs @@ -0,0 +1,49 @@ +use reqwest::header::{HeaderMap, HeaderName}; +use std::{collections::HashMap, str::FromStr}; + +pub fn build_header(headers: &HashMap) -> HeaderMap { + let mut result = HeaderMap::with_capacity(headers.len()); + for (k, v) in headers { + if let (Ok(k), Ok(v)) = (HeaderName::from_str(k), v.parse()) { + result.insert(k, v); + } + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn valid_headers_are_inserted() { + let mut map = HashMap::new(); + map.insert("User-Agent".to_string(), "test".to_string()); + map.insert("Accept".to_string(), "*/*".to_string()); + let h = build_header(&map); + assert_eq!(h.get("user-agent").unwrap(), "test"); + assert_eq!(h.get("accept").unwrap(), "*/*"); + } + + #[test] + fn invalid_name_is_skipped() { + let mut map = HashMap::new(); + // A space inside the header name makes it invalid. + map.insert("Invalid Name".to_string(), "x".to_string()); + assert!(build_header(&map).is_empty()); + } + + #[test] + fn invalid_value_is_skipped() { + let mut map = HashMap::new(); + // A newline in the value makes it invalid (control char rejected). + map.insert("X-Test".to_string(), "bad\nvalue".to_string()); + assert!(build_header(&map).is_empty()); + } + + #[test] + fn empty_map_is_empty() { + let map: HashMap = HashMap::new(); + assert!(build_header(&map).is_empty()); + } +} diff --git a/crates/fast-down-api/src/utils/filename_template.rs b/crates/fast-down-api/src/utils/filename_template.rs new file mode 100644 index 0000000..f887d9e --- /dev/null +++ b/crates/fast-down-api/src/utils/filename_template.rs @@ -0,0 +1,95 @@ +use chrono::Local; +use path_helper::sanitize_filename; +use std::panic; +use url::Url; + +pub fn parse_filename_template(template: String, url: &Url, filename: &str) -> String { + let template = + panic::catch_unwind(|| Local::now().format(&template).to_string()).unwrap_or(template); + let host = sanitize_filename(url.host_str().unwrap_or("unknown"), 255); + let mut parent_path: Vec<_> = url + .path_segments() + .into_iter() + .flat_map(|segments| { + segments.map(|seg| { + let decoded = urlencoding::decode_binary(seg.as_bytes()); + sanitize_filename(String::from_utf8_lossy(&decoded), 255) + }) + }) + .collect(); + parent_path.pop(); + let parent_path = if parent_path.is_empty() { + ".".to_string() + } else { + parent_path.join(std::path::MAIN_SEPARATOR_STR) + }; + let (file_stem, file_ext) = filename + .rfind('.') + .map_or((filename, ""), |pos| (&filename[..pos], &filename[pos..])); + #[allow(clippy::literal_string_with_formatting_args)] + template + .replace("{host}", &host) + .replace("{parent_path}", &parent_path) + .replace("{file_name}", filename) + .replace("{file_stem}", file_stem) + .replace("{file_ext}", file_ext) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use url::Url; + + #[test] + fn all_placeholders() { + let url = Url::parse("https://example.com/path/to/file.txt").unwrap(); + let t = "{host}/{parent_path}/{file_name}_{file_stem}{file_ext}"; + let out = parse_filename_template(t.to_string(), &url, "file.txt"); + assert!(out.starts_with("example.com")); + assert!(out.contains("path")); + assert!(out.contains("to")); + assert!(out.ends_with("file.txt_file.txt")); + } + + #[test] + fn no_placeholders_passthrough() { + let url = Url::parse("https://example.com/x").unwrap(); + assert_eq!( + parse_filename_template("plain".to_string(), &url, "f.txt"), + "plain" + ); + } + + #[test] + fn host_unknown_when_no_host() { + let url = Url::parse("file:///etc/hosts").unwrap(); + assert_eq!( + parse_filename_template("{host}".to_string(), &url, "hosts"), + "unknown" + ); + } + + #[test] + fn parent_path_root_when_no_dir() { + let url = Url::parse("https://example.com/file.txt").unwrap(); + assert_eq!( + parse_filename_template("{parent_path}".to_string(), &url, "file.txt"), + "." + ); + } + + #[test] + fn file_ext_includes_dot() { + let url = Url::parse("https://example.com/a/b.tar.gz").unwrap(); + let out = parse_filename_template("{file_stem}{file_ext}".to_string(), &url, "b.tar.gz"); + assert_eq!(out, "b.tar.gz"); + } + + #[test] + fn no_dot_file_has_empty_ext() { + let url = Url::parse("https://example.com/README").unwrap(); + let out = parse_filename_template("{file_stem}|{file_ext}".to_string(), &url, "README"); + assert_eq!(out, "README|"); + } +} diff --git a/crates/fast-down-api/src/utils/force_send.rs b/crates/fast-down-api/src/utils/force_send.rs new file mode 100644 index 0000000..1f8fe4f --- /dev/null +++ b/crates/fast-down-api/src/utils/force_send.rs @@ -0,0 +1,29 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; + +pub struct ForceSend(pub F); + +#[allow(clippy::non_send_fields_in_send_ty)] +unsafe impl Send for ForceSend {} + +impl Future for ForceSend { + type Output = F::Output; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + unsafe { self.map_unchecked_mut(|s| &mut s.0).poll(cx) } + } +} + +pub trait ForceSendExt { + fn force_send(self) -> ForceSend + where + Self: Sized, + { + ForceSend(self) + } +} + +impl ForceSendExt for F { + fn force_send(self) -> ForceSend { + ForceSend(self) + } +} diff --git a/crates/fast-down-api/src/utils/gen_path.rs b/crates/fast-down-api/src/utils/gen_path.rs new file mode 100644 index 0000000..bb33483 --- /dev/null +++ b/crates/fast-down-api/src/utils/gen_path.rs @@ -0,0 +1,37 @@ +use crate::{Config, utils::parse_filename_template}; +use fast_down::UrlInfo; +use path_helper::{auto_ext, sanitize_filename, sanitize_path}; +use soft_canonicalize::soft_canonicalize; +use std::{borrow::Cow, path::PathBuf}; +use tokio::fs; +use url::Url; + +pub async fn gen_path(url: &Url, info: &UrlInfo, config: &Config) -> std::io::Result { + let mut filename = sanitize_filename( + if config.filename.is_empty() || config.parse_filename { + auto_ext(&info.raw_name, info.content_type.as_deref()) + } else { + Cow::Borrowed(config.filename.as_str()) + }, + 248, + ); + let mut save_dir = soft_canonicalize::soft_canonicalize(&config.save_dir)?; + if config.parse_filename && !config.filename.is_empty() { + let path = PathBuf::from(parse_filename_template( + config.filename.clone(), + url, + &filename, + )); + if let Some(s) = path.file_name() { + filename = sanitize_filename(s.to_string_lossy(), 248); + } + if let Some(parent_path) = path.parent() + && let Ok(new_save_dir) = soft_canonicalize(save_dir.join(sanitize_path(parent_path))) + && new_save_dir.starts_with(&save_dir) + { + save_dir = new_save_dir; + } + } + fs::create_dir_all(&save_dir).await?; + Ok(save_dir.join(&filename)) +} diff --git a/crates/fast-down-api/src/utils/mod.rs b/crates/fast-down-api/src/utils/mod.rs new file mode 100644 index 0000000..8938650 --- /dev/null +++ b/crates/fast-down-api/src/utils/mod.rs @@ -0,0 +1,10 @@ +mod build_header; +mod filename_template; +mod force_send; +mod gen_path; +mod tx_err; + +pub use build_header::*; +pub use filename_template::*; +pub use force_send::*; +pub use gen_path::*; diff --git a/crates/fast-down-api/src/utils/tx_err.rs b/crates/fast-down-api/src/utils/tx_err.rs new file mode 100644 index 0000000..3db9f0e --- /dev/null +++ b/crates/fast-down-api/src/utils/tx_err.rs @@ -0,0 +1,22 @@ +#[macro_export] +#[doc(hidden)] +macro_rules! tx_err { + ($x: expr, $tx: expr, $event: ident) => { + match $x { + Ok(r) => r, + Err(e) => { + let _ = $tx.send(Event::$event(e)); + return; + } + } + }; + ($x: expr, $tx: expr, $event: ident, $ret: expr) => { + match $x { + Ok(r) => r, + Err(e) => { + let _ = $tx.send(Event::$event(e)); + return $ret; + } + } + }; +} diff --git a/crates/fast-down-api/tests/resume.rs b/crates/fast-down-api/tests/resume.rs new file mode 100644 index 0000000..b03142c --- /dev/null +++ b/crates/fast-down-api/tests/resume.rs @@ -0,0 +1,670 @@ +//! End-to-end integration tests for the resume (断点续传) feature of +//! `fast_down_api`. +//! +//! These tests stand up a minimal ranged HTTP server (built on `hyper`) and drive +//! the public `DownloadHandle::{download, resume}` API through real, resumable +//! scenarios: a successful resume after a mid-download cancel, a remote file +//! change (both the `download` silent-fallback and `resume` error-reporting +//! branches), a missing `.fd` state file, a server that does not support range +//! requests, and the core "cancel keeps `.part`/`.fd` so it can be resumed" +//! contract. +#![allow( + clippy::too_many_lines, + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::similar_names +)] + +use std::convert::Infallible; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use fast_down_api::{ + DownloadHandle, Event, PartialConfig, Rx, StateError, WriteMethod, create_cancellation_token, + create_channel, +}; +use futures::StreamExt; +use futures::stream::unfold; +use http_body_util::combinators::BoxBody; +use http_body_util::{BodyExt, StreamBody}; +use hyper::body::{Frame, Incoming}; +use hyper::header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, ETAG, LAST_MODIFIED, RANGE}; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use tokio::net::TcpListener; +use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; +use url::Url; + +/// Total size of the served file (5 MiB). Large enough that a throttled download +/// cannot finish before our cancel fires, so the partial state is always real. +const FILE_SIZE: usize = 5 * 1024 * 1024; +/// Bytes served per throttled chunk, with a small sleep between chunks, so the +/// download is slow enough for the cancel to land in the middle. +const THROTTLE_CHUNK: usize = 64 * 1024; +const THROTTLE_MS: u64 = 150; + +type RespBody = BoxBody; + +/// In-memory file served by [`TestServer`]. +struct FileData { + body: Vec, + etag: String, + last_modified: String, + supports_range: bool, +} + +#[derive(Clone)] +struct TestServer { + data: Arc>, +} + +impl TestServer { + /// Bind to a random loopback port and start serving. Returns the base URL. + async fn serve(&self) -> String { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind server socket"); + let addr = listener.local_addr().expect("resolve local addr"); + let server = self.clone(); + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.expect("accept connection"); + let conn_server = server.clone(); + tokio::spawn(async move { + let io = TokioIo::new(stream); + let service = service_fn(move |req| { + let s = conn_server.clone(); + handle(s, req) + }); + let _ = http1::Builder::new().serve_connection(io, service).await; + }); + } + }); + format!("http://{addr}") + } + + /// Swap the served content (and identity headers), e.g. to simulate the + /// remote file having changed between an interrupted and a resumed download. + async fn set_content(&self, body: Vec, etag: &str, last_modified: &str) { + let mut data = self.data.write().await; + data.body = body; + data.etag = etag.to_string(); + data.last_modified = last_modified.to_string(); + } +} + +/// The hyper request handler: serves the current [`FileData`], honouring range +/// requests (with throttling) when `supports_range` is set, and always answering +/// with `ETag`/`Last-Modified` so the prefetch can build a `FileId`. +async fn handle( + server: TestServer, + req: Request, +) -> Result, Infallible> { + let data = server.data.read().await; + let total = data.body.len(); + let supports_range = data.supports_range; + let etag = data.etag.clone(); + let last_modified = data.last_modified.clone(); + let body = data.body.clone(); + drop(data); + + let range = req + .headers() + .get(RANGE) + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + + if supports_range + && let Some(header) = range + && let Some((start, end)) = parse_range(&header, total) + { + let chunk = body[start..end].to_vec(); + let end_inclusive = end - 1; + let content_range = format!("bytes {start}-{end_inclusive}/{total}"); + return Ok(Response::builder() + .status(StatusCode::PARTIAL_CONTENT) + .header(CONTENT_RANGE, content_range) + .header(ACCEPT_RANGES, "bytes") + .header(CONTENT_LENGTH, chunk.len().to_string()) + .header(ETAG, etag.as_str()) + .header(LAST_MODIFIED, last_modified.as_str()) + .body(throttled_stream(chunk)) + .expect("build 206 response")); + } + + // Fallback: full body, no `Accept-Ranges`, so a `Range` probe (used by the + // prefetch to detect resumability) will not see a `Content-Range` header. + // Throttled the same way as the ranged branch so a fresh (non-resumed) + // download is still slow enough for a mid-flight cancel to land. + Ok(Response::builder() + .status(StatusCode::OK) + .header(CONTENT_LENGTH, body.len().to_string()) + .header(ETAG, etag.as_str()) + .header(LAST_MODIFIED, last_modified.as_str()) + .body(throttled_stream(body)) + .expect("build 200 response")) +} + +/// Parse a `bytes=START-END` (or `bytes=START-`) header into an exclusive +/// `[start, end)` range against `total` bytes. +fn parse_range(header: &str, total: usize) -> Option<(usize, usize)> { + let spec = header.trim().strip_prefix("bytes=")?; + let (start_s, end_s) = spec.split_once('-')?; + let start = if start_s.is_empty() { + 0 + } else { + start_s.trim().parse().ok()? + }; + let end = if end_s.is_empty() { + total + } else { + end_s.trim().parse::().ok()?.saturating_add(1) + }; + let end = end.min(total); + if start >= end { + return None; + } + Some((start, end)) +} + +/// Build a streaming body that emits `THROTTLE_CHUNK` slices of `data` with a +/// small `sleep` between them, keeping the connection open long enough for a +/// cancel to arrive mid-transfer. +fn throttled_stream(data: Vec) -> RespBody { + let stream = unfold((0usize, data), |(pos, data)| async move { + if pos >= data.len() { + return None; + } + tokio::time::sleep(Duration::from_millis(THROTTLE_MS)).await; + let end = (pos + THROTTLE_CHUNK).min(data.len()); + Some(( + Ok::<_, Infallible>(Bytes::copy_from_slice(&data[pos..end])), + (end, data), + )) + }); + let framed = stream.map(|result| result.map(Frame::data)); + BodyExt::boxed(StreamBody::new(framed)) +} + +/// Start a ranged server serving `body` under the given identity headers. +async fn start_server( + body: Vec, + etag: &str, + last_modified: &str, + supports_range: bool, +) -> (TestServer, String) { + let server = TestServer { + data: Arc::new(RwLock::new(FileData { + body, + etag: etag.to_string(), + last_modified: last_modified.to_string(), + supports_range, + })), + }; + let url = server.serve().await; + (server, url) +} + +/// A fresh, unique, empty temp directory per test (runs in parallel safe). +fn temp_dir(name: &str) -> PathBuf { + static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("fast_down_api_resume_{name}_{n}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir +} + +/// Build a deterministic config that targets `/out.bin` and downloads with +/// a single thread and small chunks so the cancel lands predictably mid-flight. +fn make_config(save_dir: &Path) -> PartialConfig { + PartialConfig { + save_dir: Some(save_dir.to_path_buf()), + filename: Some("out.bin".to_string()), + parse_filename: Some(false), + overwrite: Some(true), + write_method: Some(WriteMethod::Mmap), + min_chunk_size: Some(1024 * 1024), + threads: Some(1), + ..Default::default() + } +} + +/// Drain all events from `rx` until the task ends (channel closed). +async fn drain(rx: Rx) -> Vec { + let mut events = Vec::new(); + while let Ok(e) = rx.recv().await { + events.push(e); + } + events +} + +/// Start a normal `download()` and cancel it a fixed time after `Event::Start`, +/// leaving a real partial `.part`/`.fd` on disk. Returns the collected events. +/// `DownloadHelper` 的 `partial_download_via_cancel` 版本——开始下载后收到 `Start` 立即取消。 +async fn partial_download_via_cancel( + url: &str, + save_dir: &Path, + cancel: CancellationToken, +) -> Vec { + let cfg = make_config(save_dir); + let (tx, rx) = create_channel(); + let _handle = + DownloadHandle::download(Url::parse(url).expect("valid url"), cfg, tx, cancel.clone()); + + let mut events = Vec::new(); + let mut started = false; + while let Ok(e) = rx.recv().await { + if matches!(e, Event::Start { .. }) && !started { + started = true; + cancel.cancel(); + } + events.push(e); + } + assert!(started, "expected Event::Start during the partial download"); + assert!( + !events.iter().any(|e| matches!(e, Event::Renamed(_))), + "a cancelled download must not rename the .part file" + ); + events +} + +fn original_bytes() -> Vec { + vec![0xAA; FILE_SIZE] +} + +fn new_bytes() -> Vec { + vec![0xBB; FILE_SIZE] +} + +/// Case 1 (+ Case 5 emphasis): cancel mid-download, then `resume()` finishes it. +#[tokio::test] +async fn test_resume_success() { + let dir = temp_dir("resume_success"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + assert!(part.exists(), ".part must exist after cancel"); + assert!(fd.exists(), ".fd must exist after cancel"); + assert!( + !final_path.exists(), + "final file must NOT exist after a cancelled download" + ); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let resume_cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + resume_cancel, + ); + let events = drain(rx).await; + + let (_, size) = events + .iter() + .find_map(|e| match e { + Event::Resumed { progress, size, .. } => Some((progress, *size)), + _ => None, + }) + .expect("expected Event::Resumed"); + assert_eq!( + size, FILE_SIZE as u64, + "Event::Resumed.size must equal file size" + ); + + let renamed = events + .iter() + .find_map(|e| match e { + Event::Renamed(p) => Some(p.clone()), + _ => None, + }) + .expect("expected Event::Renamed"); + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "resume() must not emit a ResumeError on a valid state" + ); + + let got = tokio::fs::read(&renamed).await.expect("read final file"); + assert_eq!(got, original_bytes(), "resumed file content mismatch"); + assert!( + !fd.exists(), + ".fd must be removed after a successful resume (contract §1)" + ); +} + +/// Case 2 (download branch): a stale `.fd` (remote file changed) makes +/// `download()` silently fall back to a full re-download of the NEW content. +#[tokio::test] +async fn test_file_changed_download_falls_back() { + let dir = temp_dir("file_changed_download"); + let (server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + // The `.fd` now records the ORIGINAL etag/size. + + server.set_content(new_bytes(), "new", "LM-B").await; + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let dl_cancel = create_cancellation_token(); + let _handle = + DownloadHandle::download(Url::parse(&url).expect("valid url"), cfg, tx, dl_cancel); + let events = drain(rx).await; + + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "download() must complete (Renamed) after a silent fallback" + ); + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "download() must NOT emit a ResumeError" + ); + + let final_path = dir.join("out.bin"); + let got = tokio::fs::read(&final_path) + .await + .expect("final file should exist"); + assert_eq!(got, new_bytes(), "final file must be the NEW content"); + let fd = final_path.with_added_extension("fd"); + assert!( + !fd.exists(), + ".fd must be removed after a successful download (contract §1)" + ); +} + +/// Case 2 (resume branch): a stale `.fd` (remote file changed) makes `resume()` +/// report `StateError::FileChanged` and keep the partial files untouched. +#[tokio::test] +async fn test_file_changed_resume_reports_error() { + let dir = temp_dir("file_changed_resume"); + let (server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + server.set_content(new_bytes(), "new", "LM-B").await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let resume_cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + resume_cancel, + ); + let events = drain(rx).await; + + let err = events + .iter() + .find_map(|e| match e { + Event::ResumeError(r) => Some(r), + _ => None, + }) + .expect("expected Event::ResumeError"); + assert!( + matches!(err, StateError::FileChanged { .. }), + "expected Event::ResumeError(StateError::FileChanged), got {err:?}" + ); + + assert!( + !events.iter().any(|e| matches!(e, Event::Renamed(_))), + "resume() must NOT rename on FileChanged" + ); + + assert!(part.exists(), ".part must be kept on resume error"); + assert!(fd.exists(), ".fd must be kept on resume error"); + assert!( + !final_path.exists(), + "final file must not exist on resume error" + ); +} + +/// Case 3: `resume()` with no `.fd` state file reports `StateError::Open`. +#[tokio::test] +async fn test_resume_no_state_file() { + let dir = temp_dir("resume_no_state_file"); + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + // `tmp_path` (.part) exists but the `.fd` state file is missing → `resume` + // must report `NoStateFile` (the resume contract), not silently fall back. + let _ = tokio::fs::remove_file(&fd).await; + let _ = std::fs::File::create(&part).expect("create .part"); + + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + cancel, + ); + let events = drain(rx).await; + + let err = events + .iter() + .find_map(|e| match e { + Event::ResumeError(r) => Some(r), + _ => None, + }) + .expect("expected Event::ResumeError"); + assert!( + matches!(err, StateError::Open(_)), + "expected Event::ResumeError(StateError::Open) for missing .fd, got {err:?}" + ); + assert!( + !events.iter().any(|e| matches!(e, Event::Renamed(_))), + "resume() must NOT rename when there is no .fd" + ); +} + +/// Case 4: `resume()` against a server that does not support range requests +/// reports `StateError::NotResumable`. +#[tokio::test] +async fn test_resume_not_resumable() { + let dir = temp_dir("resume_not_resumable"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", false).await; + // A cancelled download leaves a valid `.fd` + `.part` for the non-range + // server; `resume()` must then report `NotResumable` (server can't range). + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + cancel, + ); + let events = drain(rx).await; + + let err = events + .iter() + .find_map(|e| match e { + Event::ResumeError(r) => Some(r), + _ => None, + }) + .expect("expected Event::ResumeError"); + assert!( + matches!(err, StateError::NotResumable(..)), + "expected Event::ResumeError(StateError::NotResumable), got {err:?}" + ); +} + +/// Case 5 (explicit): the previous round's fix — a cancel must preserve `.part` +/// and `.fd` (not delete them), and a subsequent `resume()` must finish cleanly. +#[tokio::test] +async fn test_cancel_keeps_part_and_fd_then_resume() { + let dir = temp_dir("cancel_keeps_state"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + assert!( + part.exists() && fd.exists(), + "cancel must preserve both .part and .fd" + ); + assert!( + !final_path.exists(), + "cancel must NOT create the final file" + ); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let resume_cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + resume_cancel, + ); + let events = drain(rx).await; + + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "resume() must complete with Renamed after a cancel" + ); + assert!( + !fd.exists(), + ".fd must be removed after the resume completes" + ); + let got = tokio::fs::read(&final_path) + .await + .expect("final file should exist after resume"); + assert_eq!( + got, + original_bytes(), + "final content must match after resume" + ); +} + +/// Case 5b (explicit new behavior): when the provided `tmp_path` does **not** +/// exist, `resume()` must fall back to a fresh full download (`force_resume = +/// false`) at that path — it must complete (Renamed), produce the correct +/// content, and never emit a `ResumeError`. +#[tokio::test] +async fn test_resume_missing_tmp_path_falls_back_to_download() { + let dir = temp_dir("resume_missing_tmp"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + assert!( + !part.exists(), + "tmp_path must not exist before resume (the point of this test)" + ); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel = create_cancellation_token(); + let _handle = DownloadHandle::resume( + part.clone(), + Url::parse(&url).expect("valid url"), + cfg, + tx, + cancel, + ); + let events = drain(rx).await; + + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "missing tmp_path must fall back to a full download (Renamed)" + ); + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "missing tmp_path must NOT emit a ResumeError" + ); + + let got = tokio::fs::read(&final_path) + .await + .expect("final file should exist after the fallback download"); + assert_eq!( + got, + original_bytes(), + "fallback download must produce correct content" + ); + assert!( + !part.exists(), + ".part must be renamed away after a successful fallback download" + ); +} + +/// Case 6 (fresh download, sanity guard): a brand-new `download()` with no +/// cancel and no resume must write the COMPLETE and CORRECT file. This is the +/// direct regression guard for BUG-1 — an empty `downloaded_chunk` (before the +/// fix) made `run_download` request zero chunks and produce a 0-byte file. +#[tokio::test] +async fn test_fresh_download_writes_full_file() { + let dir = temp_dir("fresh_download"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel = create_cancellation_token(); + let _handle = DownloadHandle::download(Url::parse(&url).expect("valid url"), cfg, tx, cancel); + let events = drain(rx).await; + + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "a fresh download must complete with Renamed" + ); + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "a fresh download must NOT emit a ResumeError" + ); + + let final_path = dir.join("out.bin"); + assert!( + final_path.exists(), + "final file must exist after a fresh download" + ); + + let got = tokio::fs::read(&final_path).await.expect("read final file"); + assert_eq!( + got.len(), + FILE_SIZE, + "fresh download must write the full file (BUG-1 regression guard)" + ); + assert_eq!( + got, + original_bytes(), + "fresh download content must match source exactly" + ); + + let fd = final_path.with_added_extension("fd"); + assert!( + !fd.exists(), + ".fd must be removed after a successful fresh download" + ); +} diff --git a/crates/fast-down/README.md b/crates/fast-down/README.md index db44313..9d3cceb 100644 --- a/crates/fast-down/README.md +++ b/crates/fast-down/README.md @@ -2,24 +2,95 @@ [![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) [![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![codecov](https://codecov.io/gh/fast-down/core/branch/main/graph/badge.svg)](https://codecov.io/gh/fast-down/core) [![Latest version](https://img.shields.io/crates/v/fast-down.svg)](https://crates.io/crates/fast-down) [![Documentation](https://docs.rs/fast-down/badge.svg)](https://docs.rs/fast-down) [![License](https://img.shields.io/crates/l/fast-down.svg)](https://github.com/fast-down/core/blob/main/LICENSE) -`fast-down` **Fastest** concurrent downloader! +`fast-down` is a fast, concurrent file downloader library built on top of the +[`fast_pull`](https://crates.io/crates/fast-pull) pull/push engine, with first-class +HTTP support. **[Official Website (Simplified Chinese)](https://fd.s121.top/)** -## Features - -1. **⚡️ Fastest Download**\ - We created [fast-steal](https://github.com/fast-down/fast-steal) With optimized Work Stealing, **1.43 x faster** than - NDM. -2. **🔄 File consistency**\ - Switching Wi-Fi, Turn Off Wi-Fi, Switch proxies. **We guarantee the consistency**. -3. **⛓️‍💥 Resuming Downloads**\ - You can **interrupt** at any time, and **resume downloading** after. -4. **⛓️‍💥 Incremental Downloads**\ - 1000 more lines server logs? Don't worry, we **only download new lines**. -5. **💰 Free and open-source**\ - The code stays free and open-source. Thanks to [share121](https://github.com/share121), [Cyan](https://github.com/CyanChanges) and other fast-down contributors. +## What this crate provides + +`fast-down` re-exports the entire [`fast_pull`](https://crates.io/crates/fast-pull) +pull/push engine (the `download_single` / `download_multi` entry points, the `Puller` +and `Pusher` traits, and the file/memory pushers) and adds the following on top of it: + +1. **HTTP / reqwest puller** — `FastDownPuller` (with `FastDownPullerOptions`) is the + default `Puller` for HTTP(S) sources. It builds on a `SmartRedirectClient` that + follows redirects _manually_ so it can honor the `Referrer-Policy` header and strip + resource-specific headers (`Origin` / `Authorization` / `Cookie`) on cross-origin + hops, per RFC 9110 §15.4. +2. **URL info resolution** — `UrlInfo` and `FileId` capture a resource's size, suggested + filename, content type, range support, and a stable identity derived from the + `ETag` / `Last-Modified` headers, which powers incremental and resumable downloads. +3. **Proxy support** — the `Proxy` enum selects no proxy, the system proxy, or a custom + proxy URL for outgoing requests. +4. **Task handles** — the re-exported `SharedHandle` task-handle type lets you await + and abort an in-flight download from multiple owners. + +Supporting building blocks (behind feature flags) include the backend-agnostic +`http` module (`HttpClient` traits, `HttpPuller`, `Prefetch`, `ContentDisposition`, +`HttpError`) and the `reqwest` module (`SmartRedirectClient`, +`ManualRedirectRequestBuilder`). + +## Example + +Download a file concurrently to disk. This requires the `reqwest` feature +(which also enables `http` and `fast-puller`) and a network connection, so the +block is marked `ignore` to keep doctests hermetic: + +```rust,ignore +use std::sync::Arc; +use std::time::Duration; +use url::Url; + +use fast_down::{FastDownPuller, FastDownPullerOptions, FileId, Proxy}; +use fast_pull::file::StdFilePusher; +use fast_pull::multi::DownloadOptions; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let url = Url::parse("https://example.com/large.bin")?; + let file = tokio::fs::File::create("large.bin").await?; + // Pre-size the file; pass `true` to fsync on flush. + let pusher = StdFilePusher::new(file, /* size */ 0, /* sync_all */ false).await?; + + let puller = FastDownPuller::new(FastDownPullerOptions { + url, + headers: Arc::new(Default::default()), + proxy: Proxy::System, + accept_invalid_certs: false, + accept_invalid_hostnames: false, + cookie_store: false, + file_id: FileId::default(), + resp: None, + available_ips: Arc::from(Vec::::new()), + max_redirects: 10, + })?; + + let result = fast_pull::download_multi( + puller, + pusher, + DownloadOptions { + download_chunks: vec![0..u64::MAX].into_iter(), + concurrent: 8, + retry_gap: Duration::from_secs(1), + pull_timeout: Duration::from_secs(30), + push_queue_cap: 1024, + min_chunk_size: 1 << 20, + max_speculative: 3, + }, + ); + + result.join().await?; + Ok(()) +} +``` + +For a sequential, single-threaded download, swap `download_multi` for +`fast_pull::download_single` and use `fast_pull::single::DownloadOptions` +(which only has `retry_gap` and `push_queue_cap`). diff --git a/crates/fast-down/src/http/content_disposition.rs b/crates/fast-down/src/http/content_disposition.rs index 95715bc..5da1a25 100644 --- a/crates/fast-down/src/http/content_disposition.rs +++ b/crates/fast-down/src/http/content_disposition.rs @@ -1,3 +1,10 @@ +//! Parser for the HTTP `Content-Disposition` header. +//! +//! Used during prefetch to derive a suggested filename for a downloaded +//! resource. [`ContentDisposition::parse`] handles both the quoted `filename` +//! form and the RFC 5987 `filename*` (UTF-8, percent-encoded) form, with the +//! latter taking precedence when both are present. + use std::{iter::Peekable, str::Chars}; /// Parsed `Content-Disposition` header, extracting the `filename` parameter. @@ -210,4 +217,46 @@ mod tests { let cd = ContentDisposition::parse(s); assert_eq!(cd.filename.unwrap(), ";\";;"); } + + #[test] + fn test_no_semicolon_is_none() { + assert_eq!(ContentDisposition::parse("attachment").filename, None); + } + + #[test] + fn test_empty_header() { + assert_eq!(ContentDisposition::parse("").filename, None); + assert_eq!(ContentDisposition::parse(" ").filename, None); + } + + #[test] + fn test_filename_star_non_utf8_ignored() { + // A non-UTF-8 charset is unsupported, so `filename*` is dropped. + let s = "attachment; filename*=ISO-8859-1''%A3.txt"; + assert_eq!(ContentDisposition::parse(s).filename, None); + } + + #[test] + fn test_filename_star_wins_over_filename() { + let s = r#"attachment; filename="old.txt"; filename*=UTF-8''%E6%B5%8B.txt"#; + assert_eq!(ContentDisposition::parse(s).filename.unwrap(), "测.txt"); + } + + #[test] + fn test_unquoted_token_stops_at_space() { + let s = "attachment; filename=foo bar.txt"; + assert_eq!(ContentDisposition::parse(s).filename.unwrap(), "foo"); + } + + #[test] + fn test_consecutive_semicolons() { + let s = "attachment;;; filename=foo.txt"; + assert_eq!(ContentDisposition::parse(s).filename.unwrap(), "foo.txt"); + } + + #[test] + fn test_quoted_with_escaped_quote() { + let s = r#"attachment; filename="a\"b.txt""#; + assert_eq!(ContentDisposition::parse(s).filename.unwrap(), "a\"b.txt"); + } } diff --git a/crates/fast-down/src/http/manual_redirect.rs b/crates/fast-down/src/http/manual_redirect.rs index 86f2445..56cae12 100644 --- a/crates/fast-down/src/http/manual_redirect.rs +++ b/crates/fast-down/src/http/manual_redirect.rs @@ -1,3 +1,11 @@ +//! Manual redirect handling that respects the `Referrer-Policy` header. +//! +//! When `SmartRedirectClient` follows redirects itself (rather +//! than letting `reqwest` do it), it must compute the correct `Referer` header +//! for each hop. [`compute_referer`] implements the W3C Referrer Policy +//! algorithm (with RFC 9110 §7.4 defaults), and [`ReferrerPolicy`] parses the +//! policy tokens sent by servers. + use url::Url; /// Serialize a URL for use as a `Referer` header value, @@ -51,6 +59,7 @@ impl ReferrerPolicy { } } +/// Returns `true` when following `from` -> `to` would downgrade from HTTPS to HTTP. fn is_downgrade(from: &Url, to: &Url) -> bool { from.scheme() == "https" && to.scheme() == "http" } @@ -113,3 +122,185 @@ pub fn compute_referer( Some(ReferrerPolicy::UnsafeUrl) => Some(referer_url(prev_url)), } } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use url::Url; + + fn u(s: &str) -> Url { + Url::parse(s).unwrap() + } + + #[test] + fn parse_all_tokens_case_insensitive() { + assert_eq!( + ReferrerPolicy::parse("no-referrer"), + Some(ReferrerPolicy::NoReferrer) + ); + assert_eq!( + ReferrerPolicy::parse("NO-REFERRER"), + Some(ReferrerPolicy::NoReferrer) + ); + assert_eq!( + ReferrerPolicy::parse("No-Referrer-When-Downgrade"), + Some(ReferrerPolicy::NoReferrerWhenDowngrade) + ); + assert_eq!( + ReferrerPolicy::parse("origin"), + Some(ReferrerPolicy::Origin) + ); + assert_eq!( + ReferrerPolicy::parse("origin-when-cross-origin"), + Some(ReferrerPolicy::OriginWhenCrossOrigin) + ); + assert_eq!( + ReferrerPolicy::parse("same-origin"), + Some(ReferrerPolicy::SameOrigin) + ); + assert_eq!( + ReferrerPolicy::parse("strict-origin"), + Some(ReferrerPolicy::StrictOrigin) + ); + assert_eq!( + ReferrerPolicy::parse("strict-origin-when-cross-origin"), + Some(ReferrerPolicy::StrictOriginWhenCrossOrigin) + ); + assert_eq!( + ReferrerPolicy::parse("unsafe-url"), + Some(ReferrerPolicy::UnsafeUrl) + ); + } + + #[test] + fn parse_last_token_wins() { + assert_eq!( + ReferrerPolicy::parse("origin, no-referrer"), + Some(ReferrerPolicy::NoReferrer) + ); + assert_eq!( + ReferrerPolicy::parse("no-referrer, origin"), + Some(ReferrerPolicy::Origin) + ); + } + + #[test] + fn parse_invalid_returns_none() { + assert_eq!(ReferrerPolicy::parse(""), None); + assert_eq!(ReferrerPolicy::parse("garbage"), None); + assert_eq!( + ReferrerPolicy::parse("no-referrer, garbage"), + Some(ReferrerPolicy::NoReferrer) + ); + } + + #[test] + fn compute_referer_matrix() { + let a = u("https://a.com/p"); + let b = u("https://b.com/q"); + let a_http = u("http://a.com/r"); + + // None / NoReferrerWhenDowngrade (default) + assert_eq!( + compute_referer(None, &a, &b), + Some("https://a.com/p".to_string()) + ); + assert_eq!(compute_referer(None, &a, &a_http), None); + + // NoReferrer + assert_eq!( + compute_referer(Some(ReferrerPolicy::NoReferrer), &a, &b), + None + ); + + // Origin + assert_eq!( + compute_referer(Some(ReferrerPolicy::Origin), &a, &b), + Some("https://a.com".to_string()) + ); + assert_eq!( + compute_referer(Some(ReferrerPolicy::Origin), &a, &a_http), + Some("https://a.com".to_string()) + ); + + // OriginWhenCrossOrigin + assert_eq!( + compute_referer(Some(ReferrerPolicy::OriginWhenCrossOrigin), &a, &a), + Some("https://a.com/p".to_string()) + ); + assert_eq!( + compute_referer(Some(ReferrerPolicy::OriginWhenCrossOrigin), &a, &b), + Some("https://a.com".to_string()) + ); + + // SameOrigin + assert_eq!( + compute_referer(Some(ReferrerPolicy::SameOrigin), &a, &a), + Some("https://a.com/p".to_string()) + ); + assert_eq!( + compute_referer(Some(ReferrerPolicy::SameOrigin), &a, &b), + None + ); + + // StrictOrigin + assert_eq!( + compute_referer(Some(ReferrerPolicy::StrictOrigin), &a, &a), + Some("https://a.com".to_string()) + ); + assert_eq!( + compute_referer(Some(ReferrerPolicy::StrictOrigin), &a, &a_http), + None + ); + + // StrictOriginWhenCrossOrigin + assert_eq!( + compute_referer(Some(ReferrerPolicy::StrictOriginWhenCrossOrigin), &a, &a), + Some("https://a.com/p".to_string()) + ); + assert_eq!( + compute_referer(Some(ReferrerPolicy::StrictOriginWhenCrossOrigin), &a, &b), + Some("https://a.com".to_string()) + ); + assert_eq!( + compute_referer( + Some(ReferrerPolicy::StrictOriginWhenCrossOrigin), + &a, + &a_http + ), + None + ); + + // UnsafeUrl (keeps full referer, no downgrade suppression) + assert_eq!( + compute_referer(Some(ReferrerPolicy::UnsafeUrl), &a, &a_http), + Some("https://a.com/p".to_string()) + ); + } + + #[test] + fn compute_referer_strips_userinfo_and_fragment() { + let with_auth = u("https://user:pass@a.com/p#frag"); + let other = u("https://b.com/q"); + let same = u("https://user:pass@a.com/r#x"); + // cross-origin -> only origin (no userinfo/fragment) + assert_eq!( + compute_referer( + Some(ReferrerPolicy::OriginWhenCrossOrigin), + &with_auth, + &other + ), + Some("https://a.com".to_string()) + ); + // same-origin -> full referer with userinfo & fragment stripped + assert_eq!( + compute_referer( + Some(ReferrerPolicy::OriginWhenCrossOrigin), + &with_auth, + &same + ), + Some("https://a.com/p".to_string()) + ); + } +} diff --git a/crates/fast-down/src/http/mod.rs b/crates/fast-down/src/http/mod.rs index 3fd28e1..c1daf77 100644 --- a/crates/fast-down/src/http/mod.rs +++ b/crates/fast-down/src/http/mod.rs @@ -1,3 +1,21 @@ +//! A backend-agnostic HTTP download layer. +//! +//! This module defines a set of small traits — [`HttpClient`], +//! [`HttpRequestBuilder`], [`HttpResponse`], and [`HttpHeaders`] — that abstract +//! over any HTTP client implementation (for example the `reqwest` wrapper in +//! the `reqwest` module). On top of them it provides: +//! +//! * [`HttpPuller`]: a [`fast_pull::Puller`] that streams bytes over HTTP, with +//! range support and file-identity (resumability) checks. +//! * [`Prefetch`]: resolves a [`crate::UrlInfo`] for a URL via a prefetch request. +//! * [`ContentDisposition`]: parses the `Content-Disposition` header for filenames. +//! * [`manual_redirect`]: RFC 9110-aware `Referer` computation for redirect following. +//! * [`HttpError`]: the error type produced by this layer. +//! +//! Most users do not use these types directly; instead they use +//! `FastDownPuller` (from the `fast-puller` feature), which wraps +//! [`HttpPuller`] with a smart-redirecting `reqwest` client. + mod content_disposition; pub mod manual_redirect; mod prefetch; @@ -61,13 +79,13 @@ pub type GetHeaderError = as HttpHeaders>::GetHeaderE /// detecting mismatched file identity, and irrecoverable failures. #[derive(thiserror::Error)] pub enum HttpError { - #[error("HTTP request failed")] + #[error("HTTP request failed: {0:?}")] Request(GetRequestError), - #[error("HTTP chunk read failed")] + #[error("HTTP chunk read failed: {0:?}\n response: {1:?}")] Chunk(GetChunkError, GetResponse), #[error("irrecoverable pull error")] Irrecoverable, - #[error("body mismatch: expected file {0:?}, got different content")] + #[error("body mismatch: expected file {0:?}, got different content\n response: {1:?}")] MismatchedBody(FileId, GetResponse), } diff --git a/crates/fast-down/src/http/prefetch.rs b/crates/fast-down/src/http/prefetch.rs index 827a568..42dbf3c 100644 --- a/crates/fast-down/src/http/prefetch.rs +++ b/crates/fast-down/src/http/prefetch.rs @@ -1,8 +1,15 @@ +//! Prefetch metadata for a downloadable URL over HTTP. +//! +//! [`Prefetch::prefetch`] issues the initial GET (and a range probe) through a +//! [`crate::http::HttpClient`], then assembles a [`crate::UrlInfo`] describing +//! the resource: its size, suggested filename, content type, range support, and +//! the [`crate::FileId`] used for resumable downloads. + use crate::{ UrlInfo, http::{ - ContentDisposition, GetResponse, HttpClient, HttpError, HttpHeaders, HttpRequestBuilder, - HttpResponse, + ContentDisposition, GetRequestError, GetResponse, HttpClient, HttpError, HttpHeaders, + HttpRequestBuilder, HttpResponse, }, url_info::FileId, }; @@ -11,7 +18,7 @@ use url::Url; /// Result of a prefetch operation: the metadata ([`UrlInfo`]) and the initial HTTP response. pub type PrefetchResult = - Result<(UrlInfo, GetResponse), (HttpError, Option)>; + Result<(UrlInfo, GetResponse), (GetRequestError, Option)>; /// Trait for fetching resource metadata (size, filename, range support) from a URL. /// @@ -50,8 +57,8 @@ fn get_filename(headers: &impl HttpHeaders, url: &Url) -> String { }) .filter(|s| !s.trim().is_empty()) }) - .or_else(|| url.host_str().map(ToString::to_string)) - .unwrap_or_else(|| url.to_string()) + .or_else(|| url.host_str().map(|s| s.replace('.', "_"))) + .unwrap_or_else(|| url.to_string().replace('.', "_")) } async fn prefetch(client: &Client, url: Url) -> PrefetchResult { @@ -76,11 +83,7 @@ async fn prefetch_no_range( client: &Client, url: Url, ) -> PrefetchResult { - let resp = client - .get(url, None) - .send() - .await - .map_err(|(e, d)| (HttpError::Request(e), d))?; + let resp = client.get(url, None).send().await?; let headers = resp.headers(); let size = headers .get("content-length") diff --git a/crates/fast-down/src/http/puller.rs b/crates/fast-down/src/http/puller.rs index 7925415..2658d2a 100644 --- a/crates/fast-down/src/http/puller.rs +++ b/crates/fast-down/src/http/puller.rs @@ -1,3 +1,11 @@ +//! An HTTP implementation of the [`fast_pull::Puller`] trait. +//! +//! [`HttpPuller`] builds range requests through the generic [`crate::http::HttpClient`] +//! trait and streams the response body back as a [`fast_pull::PullStream`]. It +//! verifies the server's `ETag` / `Last-Modified` headers against the expected +//! [`crate::FileId`] so that a changed file is reported as [`crate::http::HttpError::MismatchedBody`] +//! rather than silently corrupting an incremental download. + use crate::http::{ FileId, GetRequestError, GetResponse, HttpClient, HttpError, HttpHeaders, HttpRequestBuilder, HttpResponse, @@ -39,6 +47,14 @@ impl Clone for HttpPuller { } } impl HttpPuller { + /// Create a new [`HttpPuller`]. + /// + /// * `url` — the resource to download. + /// * `client` — the HTTP client used to issue requests. + /// * `resp` — an optional already-open response to reuse for the first + /// (full-file) request, typically the one produced by a prefetch. + /// * `file_id` — the expected [`crate::FileId`], compared against the + /// server's headers to detect a changed resource. pub const fn new( url: Arc, client: Client, diff --git a/crates/fast-down/src/proxy.rs b/crates/fast-down/src/proxy.rs index 2261d97..fd9c7e3 100644 --- a/crates/fast-down/src/proxy.rs +++ b/crates/fast-down/src/proxy.rs @@ -1,3 +1,9 @@ +//! Proxy selection for outgoing HTTP requests. +//! +//! This module defines the [`Proxy`] enum, which tells a `FastDownPuller` +//! how to route its connections: no proxy, the system proxy (honoring +//! platform/Environment settings), or a caller-supplied custom proxy URL. + use std::ops::Deref; /// Proxy configuration for outgoing HTTP requests. @@ -14,6 +20,8 @@ pub enum Proxy { } impl Proxy { + /// Transform the inner custom value, leaving [`Proxy::No`] and + /// [`Proxy::System`] unchanged. pub fn map(self, f: impl FnOnce(T) -> U) -> Proxy { match self { Self::No => Proxy::No, @@ -22,6 +30,8 @@ impl Proxy { } } + /// Borrow the inner custom value as a reference, leaving [`Proxy::No`] and + /// [`Proxy::System`] unchanged. Requires `T: Deref`. pub fn as_deref(&self) -> Proxy<&T::Target> where T: Deref, @@ -33,6 +43,8 @@ impl Proxy { } } + /// Borrow the inner custom value as a shared reference, leaving + /// [`Proxy::No`] and [`Proxy::System`] unchanged. pub const fn as_ref(&self) -> Proxy<&T> { match self { Self::No => Proxy::No, @@ -41,3 +53,43 @@ impl Proxy { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn map_transforms_custom_only() { + assert_eq!(Proxy::::No.map(|x| x + 1), Proxy::No); + assert_eq!(Proxy::::System.map(|x| x + 1), Proxy::System); + assert_eq!(Proxy::Custom(5).map(|x| x + 1), Proxy::Custom(6)); + } + + #[test] + fn as_deref_borrows_custom() { + let p: Proxy = Proxy::Custom("http".to_string()); + assert_eq!(p.as_deref(), Proxy::Custom("http")); + assert_eq!(Proxy::::No.as_deref(), Proxy::No); + assert_eq!(Proxy::::System.as_deref(), Proxy::System); + } + + #[test] + fn as_ref_borrows_custom() { + let p: Proxy = Proxy::Custom(7); + assert_eq!(p.as_ref(), Proxy::Custom(&7)); + assert_eq!(Proxy::::No.as_ref(), Proxy::No); + assert_eq!(Proxy::::System.as_ref(), Proxy::System); + } + + #[test] + fn default_is_system() { + assert_eq!(Proxy::::default(), Proxy::System); + } + + #[test] + fn copy_and_equality() { + let p = Proxy::Custom(3); + let q = p; // Proxy is Copy + assert_eq!(p, q); + } +} diff --git a/crates/fast-down/src/reqwest/mod.rs b/crates/fast-down/src/reqwest/mod.rs index 2b56ace..0a27e2c 100644 --- a/crates/fast-down/src/reqwest/mod.rs +++ b/crates/fast-down/src/reqwest/mod.rs @@ -1,5 +1,19 @@ #![cfg(not(target_family = "wasm"))] +//! A `reqwest`-based implementation of the [`crate::http`] HTTP traits with +//! smart redirect handling. +//! +//! This module adapts `reqwest` to the backend-agnostic [`crate::http::HttpClient`] +//! trait family and, more importantly, provides [`SmartRedirectClient`]: a +//! `reqwest::Client` wrapper that follows redirects **manually** so it can honor +//! the `Referrer-Policy` header and strip resource-specific headers +//! (`Origin` / `Authorization` / `Cookie`) on cross-origin hops, per RFC 9110 +//! §15.4. The corresponding request builder is [`ManualRedirectRequestBuilder`]. +//! +//! Most users do not construct these types directly; instead they build a +//! `FastDownPuller` via `build_client`, which creates a +//! correctly-configured [`SmartRedirectClient`]. + use crate::http::{ HttpClient, HttpHeaders, HttpRequestBuilder, HttpResponse, manual_redirect::{ReferrerPolicy, compute_referer}, @@ -37,13 +51,13 @@ impl HttpRequestBuilder for RequestBuilder { let res = self .send() .await - .map_err(|e| (ReqwestResponseError::Reqwest(e), None))?; + .map_err(|e| (ReqwestResponseError::Request(e), None))?; let status = res.status(); if status.is_success() { Ok(res) } else { let retry_after = parse_retry_after(res.headers()); - Err((ReqwestResponseError::StatusCode(status), retry_after)) + Err((ReqwestResponseError::StatusCode(res), retry_after)) } } } @@ -81,9 +95,9 @@ pub enum ReqwestGetHeaderError { #[derive(thiserror::Error, Debug)] pub enum ReqwestResponseError { #[error("Reqwest error {0:?}")] - Reqwest(reqwest::Error), - #[error("Status code {0:?}")] - StatusCode(reqwest::StatusCode), + Request(reqwest::Error), + #[error("Url: {}, Status Code: {}, Headers: {:?}", .0.url(), .0.status(), .0.headers())] + StatusCode(Response), } /// Parse the `Retry-After` response header into a [`Duration`]. @@ -118,6 +132,18 @@ pub struct SmartRedirectClient { } impl SmartRedirectClient { + /// Build a [`SmartRedirectClient`] from an already-constructed `reqwest::Client`. + /// + /// * `client` — the underlying client. **Must** be built with + /// `redirect(reqwest::redirect::Policy::none())`, otherwise the manual + /// redirect logic here conflicts with reqwest's own auto-follow. + /// * `initial_referer` — the `Referer` sent on the first request. + /// * `referrer_policy` — the policy applied when no `Referrer-Policy` + /// header is present on a response; per-hop headers override it. + /// * `origin` / `authorization` / `cookie` — resource-specific headers + /// injected only on the first hop and stripped on redirect (RFC 9110 §15.4). + /// * `max_redirects` — the maximum number of redirects to follow before + /// failing with a `StatusCode` error. #[must_use] pub const fn new( client: Client, @@ -215,7 +241,7 @@ impl HttpRequestBuilder for ManualRedirectRequestBuilder { let resp = req .send() .await - .map_err(|e| (ReqwestResponseError::Reqwest(e), None))?; + .map_err(|e| (ReqwestResponseError::Request(e), None))?; // DEBUG ASSERT: If reqwest auto-followed redirects, resp.url() will differ // from the URL we sent the request to. This means the inner Client was NOT @@ -234,12 +260,12 @@ impl HttpRequestBuilder for ManualRedirectRequestBuilder { Ok(resp) } else { let retry_after = parse_retry_after(resp.headers()); - Err((ReqwestResponseError::StatusCode(status), retry_after)) + Err((ReqwestResponseError::StatusCode(resp), retry_after)) }; } if self.redirect_count >= self.max_redirects { let retry_after = parse_retry_after(resp.headers()); - return Err((ReqwestResponseError::StatusCode(status), retry_after)); + return Err((ReqwestResponseError::StatusCode(resp), retry_after)); } let location = if let Some(v) = resp.headers().get(header::LOCATION) && let Ok(s) = v.to_str() @@ -247,11 +273,11 @@ impl HttpRequestBuilder for ManualRedirectRequestBuilder { s } else { let retry_after = parse_retry_after(resp.headers()); - return Err((ReqwestResponseError::StatusCode(status), retry_after)); + return Err((ReqwestResponseError::StatusCode(resp), retry_after)); }; let Ok(mut next_url) = self.url.join(location) else { let retry_after = parse_retry_after(resp.headers()); - return Err((ReqwestResponseError::StatusCode(status), retry_after)); + return Err((ReqwestResponseError::StatusCode(resp), retry_after)); }; // RFC 9110 §10.2.2: If the Location header lacks a fragment, // inherit it from the original request URI. @@ -296,7 +322,7 @@ mod tests { )] use super::*; use crate::{ - http::{HttpError, HttpPuller, Prefetch}, + http::{HttpPuller, Prefetch}, url_info::FileId, }; use fast_pull::{ @@ -411,17 +437,9 @@ mod tests { match client.prefetch(url).await { Ok(info) => unreachable!("404 status code should not success: {info:?}"), Err((err, _)) => match err { - HttpError::Request(e) => match e { - ReqwestResponseError::Reqwest(error) => unreachable!("{error:?}"), - ReqwestResponseError::StatusCode(status_code) => { - assert_eq!(status_code, StatusCode::NOT_FOUND); - } - }, - HttpError::Chunk(_, _) | HttpError::Irrecoverable => { - unreachable!() - } - HttpError::MismatchedBody(file_id, _) => { - unreachable!("404 status code should not return mismatched body: {file_id:?}") + ReqwestResponseError::Request(error) => unreachable!("{error:?}"), + ReqwestResponseError::StatusCode(resp) => { + assert_eq!(resp.status(), StatusCode::NOT_FOUND); } }, } @@ -494,7 +512,7 @@ mod tests { // `PushProgress` now flows on the same `event_chain` as the engine events // (the sink's listener emits it the moment data is actually written), so a // single drain collects both pull and push progress. - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { match e { Event::PullProgress(_, p) => pull_progress.merge_progress(p), Event::PushProgress(p) => push_progress.merge_progress(p), @@ -549,7 +567,7 @@ mod tests { // `PushProgress` now flows on the same `event_chain` as the engine events // (the sink's listener emits it the moment data is actually written), so a // single drain collects both pull and push progress. - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { match e { Event::PullProgress(_, p) => pull_progress.merge_progress(p), Event::PushProgress(p) => push_progress.merge_progress(p), diff --git a/crates/fast-down/src/url_info.rs b/crates/fast-down/src/url_info.rs index d9cb02b..2061e98 100644 --- a/crates/fast-down/src/url_info.rs +++ b/crates/fast-down/src/url_info.rs @@ -1,3 +1,11 @@ +//! Resolution of downloadable-resource metadata from an initial HTTP request. +//! +//! After a `FastDownPuller` performs a prefetch, it produces a +//! [`UrlInfo`] describing the resource (size, name, content type, range support) +//! together with a [`FileId`] (derived from the `ETag` / `Last-Modified` +//! headers) used to detect when a previously-downloaded file is still valid for +//! incremental/resumable downloads. + use std::sync::Arc; use url::Url; @@ -8,6 +16,7 @@ use url::Url; #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Debug, Clone, PartialEq, Eq)] pub struct UrlInfo { + /// Total size of the resource in bytes, taken from the `Content-Length` header (0 if absent). pub size: u64, /// Raw filename returned by the server. Sanitize invalid characters before using it safely. #[cfg_attr( @@ -19,10 +28,15 @@ pub struct UrlInfo { doc = "Enable the `sanitize-filename` feature to use the `filename()` method for sanitization." )] pub raw_name: String, + /// Whether the server supports HTTP range requests (used to split the download into concurrent chunks). pub supports_range: bool, + /// Whether the resource can be downloaded with the optimized fast (multi-chunk) path. pub fast_download: bool, + /// The URL the response was actually served from, after any redirects. pub final_url: Url, + /// Stable identity of the file, used to validate incremental/resumable downloads. pub file_id: FileId, + /// The `Content-Type` header value, if the server provided one. pub content_type: Option, } @@ -37,14 +51,19 @@ impl UrlInfo { /// File identity used for incremental and resumable downloads. /// /// Combines the `ETag` and `Last-Modified` headers into a stable identifier. +/// Two downloads of the same logical file share a [`FileId`] only if neither +/// header changed, which lets the engine resume instead of re-downloading. #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct FileId { + /// The `ETag` header value, if present. pub etag: Option>, + /// The `Last-Modified` header value, if present. pub last_modified: Option>, } impl FileId { + /// Build a [`FileId`] from borrowed header values, copying them into `Arc`. pub fn new(etag: Option<&str>, last_modified: Option<&str>) -> Self { Self { etag: etag.map(Arc::from), @@ -52,3 +71,52 @@ impl FileId { } } } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + + #[test] + fn file_id_new() { + let f = FileId::new(Some("abc"), Some("def")); + assert_eq!(f.etag, Some(Arc::from("abc"))); + assert_eq!(f.last_modified, Some(Arc::from("def"))); + } + + #[test] + fn file_id_none_fields() { + let f = FileId::new(None, None); + assert_eq!(f.etag, None); + assert_eq!(f.last_modified, None); + } + + #[test] + fn file_id_equality() { + assert_eq!(FileId::new(Some("x"), None), FileId::new(Some("x"), None)); + assert_ne!( + FileId::new(Some("x"), None), + FileId::new(Some("x"), Some("y")) + ); + } + + #[test] + fn url_info_filename_sanitizes() { + #![allow(unused_variables)] + let info = UrlInfo { + size: 10, + raw_name: "a/b:c*?.txt".to_string(), + supports_range: true, + fast_download: true, + final_url: Url::parse("http://example.com/x").unwrap(), + file_id: FileId::default(), + content_type: Some("text/plain".to_string()), + }; + #[cfg(feature = "sanitize-filename")] + { + let name = info.filename(); + assert!(!name.is_empty()); + assert!(!name.contains(['/', ':', '*', '?'])); + } + } +} diff --git a/crates/fast-down/src/utils/fast_puller.rs b/crates/fast-down/src/utils/fast_puller.rs index 7795502..3f6bc04 100644 --- a/crates/fast-down/src/utils/fast_puller.rs +++ b/crates/fast-down/src/utils/fast_puller.rs @@ -1,3 +1,13 @@ +//! The default HTTP [`fast_pull::Puller`] for this crate. +//! +//! [`FastDownPuller`] ties together the [`crate::http::HttpPuller`] engine and a +//! `SmartRedirectClient`, adding proxy support, optional +//! multi-interface IP rotation, and file-identity-based resumability. Construct +//! one from [`FastDownPullerOptions`] (typically via [`build_client`] to wire up +//! the underlying reqwest client), then pass it to `fast_pull::download_multi` +//! or `fast_pull::download_single` alongside a `Pusher` such as +//! `fast_pull::file::StdFilePusher` (requires the `file` feature of `fast-pull`). + use crate::Proxy; use crate::{ FileId, ProgressEntry, PullResult, PullStream, @@ -81,19 +91,31 @@ pub struct FastDownPuller { turn: Arc, max_redirects: usize, } +// Field-level docs live on [`FastDownPullerOptions`], the public construction +// surface; the runtime struct mirrors those fields. /// Options for constructing a [`FastDownPuller`]. #[derive(Debug)] pub struct FastDownPullerOptions<'a> { + /// The URL to download. pub url: Url, + /// Extra request headers sent on every request. pub headers: Arc, + /// Proxy selection (no / system / custom URL). pub proxy: Proxy<&'a str>, + /// Accept invalid TLS certificates (requires the `reqwest-tls` feature). pub accept_invalid_certs: bool, + /// Accept invalid TLS hostnames (requires the `reqwest-tls` feature). pub accept_invalid_hostnames: bool, + /// Enable a cookie store (requires the `cookie-store` feature). pub cookie_store: bool, + /// The expected [`FileId`], used to detect a changed resource and resume safely. pub file_id: FileId, + /// An already-open response to reuse for the first request (e.g. from a prefetch). pub resp: Option>>>, + /// Candidate local source IPs for outbound connections; rotated across clones. pub available_ips: Arc<[std::net::IpAddr]>, + /// Maximum number of redirects to follow before failing. pub max_redirects: usize, } diff --git a/crates/fast-down/src/utils/getifaddrs.rs b/crates/fast-down/src/utils/getifaddrs.rs index 493c3e5..ee39ac5 100644 --- a/crates/fast-down/src/utils/getifaddrs.rs +++ b/crates/fast-down/src/utils/getifaddrs.rs @@ -1,3 +1,10 @@ +//! Enumeration of usable local network interfaces. +//! +//! [`get_available_local_ips`] returns the non-loopback, non-virtual, +//! non-link-local IP addresses of the host. This powers the multi-interface +//! IP-rotation feature of `FastDownPuller`, where each clone can bind +//! to a different local address. + use std::net::IpAddr; /// # Errors diff --git a/crates/fast-down/src/utils/mod.rs b/crates/fast-down/src/utils/mod.rs index f0a4792..7fd8ce2 100644 --- a/crates/fast-down/src/utils/mod.rs +++ b/crates/fast-down/src/utils/mod.rs @@ -1,3 +1,11 @@ +//! Internal helpers backing the higher-level pullers. +//! +//! * [`fast_puller`] (feature `fast-puller`): the `FastDownPuller` +//! type and its `FastDownPullerOptions`, plus `build_client` +//! which constructs a correctly-configured `SmartRedirectClient`. +//! * [`getifaddrs`] (feature `getifaddrs`): enumerates the machine's non-virtual +//! local IP addresses for multi-interface download setups. + #[cfg(feature = "fast-puller")] #[cfg(not(target_family = "wasm"))] pub mod fast_puller; diff --git a/crates/fast-pull/README.md b/crates/fast-pull/README.md index 032e155..1322a48 100644 --- a/crates/fast-pull/README.md +++ b/crates/fast-pull/README.md @@ -2,24 +2,99 @@ [![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) [![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![codecov](https://codecov.io/gh/fast-down/core/branch/main/graph/badge.svg)](https://codecov.io/gh/fast-down/core) [![Latest version](https://img.shields.io/crates/v/fast-pull.svg)](https://crates.io/crates/fast-pull) [![Documentation](https://docs.rs/fast-pull/badge.svg)](https://docs.rs/fast-pull) [![License](https://img.shields.io/crates/l/fast-pull.svg)](https://github.com/fast-down/core/blob/main/LICENSE) -`fast-pull` **Fastest** concurrent pull/push engine for data streaming! +`fast-pull` is a low-level concurrent **pull/push streaming engine** for moving +byte ranges from any source to any sink. **[Official Website (Simplified Chinese)](https://fd.s121.top/)** ## Features -1. **⚡️ Fastest Download**\ - We created [fast-steal](https://github.com/fast-down/fast-steal) With optimized Work Stealing, **1.43 x faster** than - NDM. -2. **🔄 File consistency**\ - Switching Wi-Fi, Turn Off Wi-Fi, Switch proxies. **We guarantee the consistency**. -3. **⛓️‍💥 Resuming Downloads**\ - You can **interrupt** at any time, and **resume downloading** after. -4. **⛓️‍💥 Incremental Downloads**\ - 1000 more lines server logs? Don't worry, we **only download new lines**. -5. **💰 Free and open-source**\ - The code stays free and open-source. Thanks to [share121](https://github.com/share121), [Cyan](https://github.com/CyanChanges) and other fast-down contributors. +1. **⚡️ Concurrent pull/push** + Built on [`fast-steal`](https://github.com/fast-down/fast-steal) with optimized + work-stealing across worker tasks, plus a single-threaded sequential path + (`download_single`). +2. **🔌 `Puller` / `Pusher` abstractions** + A download is just a `Puller` (source) feeding a `Pusher` (sink). Bring your + own, or use the built-ins. `Puller` must be `Clone` so work can be stolen and + retried; `Pusher` reports partial failures so the engine can retry them. +3. **💾 Multiple write paths** (feature-gated) + - `file` — `StdFilePusher` (raw `std::fs::File` random-access writes) and + `MmapFilePusher` (memory-mapped zero-copy writes), plus the ready-made + `CacheFilePusher` stack. + - `mem` — `MemPusher`, an in-memory sink backed by a shared `Vec`. +4. **🧩 Out-of-order & buffered writes** + Cache decorators `CacheDirectPusher`, `CacheMergePusher`, and + `CacheSeqPusher` absorb out-of-order chunks (keyed by `range.start`) and + flush runs once a watermark is reached; `BufWriterPusher` batches contiguous + writes like `std::io::BufWriter`. +5. **📈 Progress & cancellation** + Streaming `Event`s (pull/push progress, errors, completion) are delivered on + `DownloadResult::event_chain`, and a session is cancelled by + `DownloadResult::abort` or simply dropping the last handle clone. +6. **🧪 Testing-friendly** + `MockPuller` + `build_mock_data` give you a deterministic in-memory source for + tests — no network or disk required. + +## Usage + +```rust +use std::sync::{Arc, Mutex}; + +use bytes::Bytes; +use fast_pull::{ + mock::{build_mock_data, MockPuller}, + single::{download_single, DownloadOptions}, + ProgressEntry, Pusher, +}; + +/// A minimal in-memory [`Pusher`] so this example compiles with **no** optional +/// features. In real code, prefer `fast_pull::mem::MemPusher` (feature `mem`) or +/// a file pusher (feature `file`). +#[derive(Clone, Default)] +struct VecPusher { + data: Arc>>, +} + +impl Pusher for VecPusher { + type Error = std::convert::Infallible; + fn push(&mut self, range: &ProgressEntry, content: Bytes) -> Result<(), (Self::Error, Bytes)> { + let mut g = self.data.lock().unwrap(); + if g.len() < range.end as usize { + g.resize(range.end as usize, 0); + } + g[range.start as usize..range.end as usize].copy_from_slice(&content); + Ok(()) + } +} + +#[tokio::main] +async fn main() { + let expected = build_mock_data(1024); + let puller = MockPuller::new(&expected); + let pusher = VecPusher::default(); + let out = pusher.data.clone(); + + let result = download_single( + puller, + pusher, + DownloadOptions { + retry_gap: std::time::Duration::from_secs(1), + push_queue_cap: 16, + }, + ); + result.join().await.unwrap(); + + assert_eq!(&*out.lock().unwrap(), &expected); +} +``` + +## License + +Licensed under the same terms as the rest of the `fast-down` workspace. Thanks to +[share121](https://github.com/share121), [Cyan](https://github.com/CyanChanges) +and other `fast-down` contributors. diff --git a/crates/fast-pull/src/base/event.rs b/crates/fast-pull/src/base/event.rs index 891760d..c7d0f04 100644 --- a/crates/fast-pull/src/base/event.rs +++ b/crates/fast-pull/src/base/event.rs @@ -1,9 +1,12 @@ +//! Download lifecycle events emitted on the +//! [`event_chain`](crate::DownloadResult::event_chain). + use crate::ProgressEntry; /// Numeric identifier assigned to each worker thread/task. pub type WorkerId = usize; -/// Events emitted during a download session, received via [`DownloadResultInner::event_chain`](crate::DownloadResultInner::event_chain). +/// Events emitted during a download session, received via [`DownloadResult::event_chain`](crate::DownloadResult::event_chain). /// /// Each variant records a state change: pulling, pushing, progress, errors, or completion. #[derive(Debug)] diff --git a/crates/fast-pull/src/base/invert.rs b/crates/fast-pull/src/base/invert.rs index 171b554..8d1055e 100644 --- a/crates/fast-pull/src/base/invert.rs +++ b/crates/fast-pull/src/base/invert.rs @@ -1,3 +1,6 @@ +//! Iterator and helper for computing the *gaps* (not-yet-downloaded ranges) +//! from a set of [`ProgressEntry`](crate::ProgressEntry)s. + use crate::ProgressEntry; /// Iterator that yields the *gaps* (non-downloaded ranges) from a list of [`ProgressEntry`]s. @@ -5,9 +8,13 @@ use crate::ProgressEntry; /// Entries shorter than `window` are merged into adjacent gaps to reduce fragmentation. #[derive(Debug)] pub struct InvertIter> { + /// Iterator over the already-downloaded (sorted) ranges. iter: I, + /// End offset of the last range consumed from `iter`. prev_end: u64, + /// Total size of the source. total_size: u64, + /// Merge entries shorter than this into the surrounding gap. window: u64, } @@ -70,4 +77,45 @@ mod tests { assert_eq!(invert_vec(&[2..4, 6..8, 10..12], 15, 5), [0..15]); assert_eq!(invert_vec(&[0..2, 10..20], 30, 5), [2..10, 20..30]); } + + #[test] + fn test_invert_empty_progress() { + // Nothing downloaded of a 50-byte file -> one gap spanning everything. + assert_eq!(invert_vec(&[], 50, 1), [0..50]); + } + + #[test] + fn test_invert_zero_total_size() { + // total_size 0 -> no gaps, even if progress is present. + assert_eq!(invert_vec(&[0..5], 0, 1), []); + } + + #[test] + fn test_invert_full_cover_no_gaps() { + assert_eq!(invert_vec(&[0..30], 30, 1), []); + } + + #[test] + fn test_invert_window_zero_keeps_small_entries() { + #![allow(clippy::single_range_in_vec_init)] + // window=0 means every entry (even tiny) is kept, so small entries are + // not merged into the surrounding gap. + assert_eq!(invert_vec(&[10..12], 30, 0), [0..10, 12..30]); + } + + #[test] + fn test_invert_trailing_gap_only() { + assert_eq!(invert_vec(&[0..20], 30, 1), [20..30]); + } + + #[test] + fn test_invert_leading_gap_only() { + assert_eq!(invert_vec(&[10..30], 30, 1), [0..10]); + } + + #[test] + fn test_invert_contiguous_then_gap() { + #![allow(clippy::single_range_in_vec_init)] + assert_eq!(invert_vec(&[0..10, 10..20], 30, 1), [20..30]); + } } diff --git a/crates/fast-pull/src/base/merge.rs b/crates/fast-pull/src/base/merge.rs index 9fe6e61..505f961 100644 --- a/crates/fast-pull/src/base/merge.rs +++ b/crates/fast-pull/src/base/merge.rs @@ -1,14 +1,21 @@ +//! Merging of [`ProgressEntry`](crate::ProgressEntry) ranges into a sorted list. + use crate::ProgressEntry; /// Trait for merging a new [`ProgressEntry`] into a sorted list of existing entries. /// /// Used to consolidate downloaded ranges and remove redundant gaps. pub trait Merge { + /// Merge `new` into the existing (sorted) progress list, coalescing overlaps + /// so the list stays sorted and gap-free where ranges touch. fn merge_progress(&mut self, new: ProgressEntry); } impl Merge for Vec { fn merge_progress(&mut self, new: ProgressEntry) { + if new.start >= new.end { + return; + } let i = self.partition_point(|x| x.end < new.start); if i == self.len() { self.push(new); @@ -72,4 +79,113 @@ mod tests { v.merge_progress(0..90); assert_eq!(v, vec![0..90]); } + + #[test] + fn test_merge_empty_range_is_dropped() { + // An empty range contained inside an existing entry must be a no-op. + let mut v = vec![1..5, 10..20]; + v.merge_progress(3..3); + assert_eq!(v, vec![1..5, 10..20]); + + // An empty range landing in a gap or at the end must NOT be inserted + // as a degenerate entry. + v.merge_progress(7..7); + assert_eq!(v, vec![1..5, 10..20]); + v.merge_progress(25..25); + assert_eq!(v, vec![1..5, 10..20]); + } + + #[test] + fn test_merge_before_front_with_gap() { + #![allow(clippy::single_range_in_vec_init)] + // `new` sits entirely before `self[0]`, leaving a gap -> inserted at front. + let mut v = vec![5..10]; + v.merge_progress(1..3); + assert_eq!(v, vec![1..3, 5..10]); + } + + #[test] + fn test_merge_extends_before_first_and_spans_gaps() { + #![allow(clippy::single_range_in_vec_init)] + // `new` starts before the first entry and spans across multiple gaps, + // absorbing every overlapping/touching entry into a single coalesced range. + let mut v = vec![1..5, 8..10, 12..15]; + v.merge_progress(0..13); + assert_eq!(v, vec![0..15]); + } + + #[test] + fn test_merge_reversed_range_is_dropped() { + #![allow(clippy::single_range_in_vec_init)] + // A reversed range (`start > end`) is invalid and must not enter the list. + let mut v = vec![10..20]; + #[allow(clippy::reversed_empty_ranges)] + v.merge_progress(30..5); + assert_eq!(v, vec![10..20]); + } + + #[test] + fn test_merge_into_empty_vec() { + #![allow(clippy::single_range_in_vec_init)] + let mut v: Vec = vec![]; + v.merge_progress(5..10); + assert_eq!(v, vec![5..10]); + } + + #[test] + fn test_merge_superset_absorbs_all() { + #![allow(clippy::single_range_in_vec_init)] + let mut v = vec![1..5, 8..10, 20..30]; + v.merge_progress(0..40); + assert_eq!(v, vec![0..40]); + } + + #[test] + fn test_merge_exact_duplicate_is_noop() { + #![allow(clippy::single_range_in_vec_init)] + let mut v = vec![1..5]; + v.merge_progress(1..5); + assert_eq!(v, vec![1..5]); + } + + #[test] + fn test_merge_touching_right_extends() { + #![allow(clippy::single_range_in_vec_init)] + let mut v = vec![1..5]; + v.merge_progress(5..8); + assert_eq!(v, vec![1..8]); + } + + #[test] + fn test_merge_touching_left_extends() { + #![allow(clippy::single_range_in_vec_init)] + let mut v = vec![5..10]; + v.merge_progress(1..5); + assert_eq!(v, vec![1..10]); + } + + /// Simulate concurrent, out-of-order chunk delivery where each chunk is + /// *adjacent* (touching but not overlapping) the next — the worst case for + /// `download_complete`'s `x.len() == 1` check in `overwrite.rs`. This must + /// still coalesce into a single `[0..200]` entry, otherwise the download + /// would be wrongly reported as incomplete and the `.part` never renamed. + #[test] + fn test_merge_out_of_order_adjacent_coalesces_to_single() { + let mut v: Vec = vec![]; + // 4 chunks of 50 bytes on a 200-byte file, arriving in a scrambled order. + v.merge_progress(0..50); + v.merge_progress(150..200); + v.merge_progress(100..150); + v.merge_progress(50..100); + assert_eq!(v, vec![0..200]); + + // And the canonical "fully covered, single entry" invariant holds for + // any interleaving that covers the whole span. + let mut w: Vec = vec![]; + for r in [0..70, 140..200, 70..140] { + w.merge_progress(r); + } + assert_eq!(w, vec![0..200]); + assert!(w.len() == 1 && w[0] == (0..200)); + } } diff --git a/crates/fast-pull/src/base/mod.rs b/crates/fast-pull/src/base/mod.rs index a47f46f..8d90063 100644 --- a/crates/fast-pull/src/base/mod.rs +++ b/crates/fast-pull/src/base/mod.rs @@ -1,3 +1,11 @@ +//! Core abstractions shared by every download engine in `fast-pull`. +//! +//! This module defines the two central traits — [`Puller`](crate::Puller) and +//! [`Pusher`](crate::Pusher) — plus the supporting types used to describe +//! progress and events: [`ProgressEntry`](crate::ProgressEntry), +//! [`Event`](crate::Event), [`WorkerId`](crate::WorkerId), and helpers for +//! merging and inverting progress ranges. + mod event; mod invert; mod merge; diff --git a/crates/fast-pull/src/base/progress.rs b/crates/fast-pull/src/base/progress.rs index 5894189..bee05bd 100644 --- a/crates/fast-pull/src/base/progress.rs +++ b/crates/fast-pull/src/base/progress.rs @@ -1,3 +1,5 @@ +//! Progress range type and total-size computation. + use core::ops::Range; /// A byte-range representing downloaded or to-be-downloaded progress. @@ -7,6 +9,7 @@ pub type ProgressEntry = Range; /// Trait for computing the total size from one or more [`ProgressEntry`] values. pub trait Total { + /// Total number of bytes represented by this progress value. fn total(&self) -> u64; } @@ -23,3 +26,27 @@ impl Total for Vec { self.iter().map(Total::total).sum() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn progress_entry_total() { + assert_eq!((0..0).total(), 0); + assert_eq!((5..10).total(), 5); + assert_eq!((3..3).total(), 0); + // A reversed range must not underflow: saturating_sub yields 0. + let reversed = core::ops::Range { start: 10, end: 5 }; + assert_eq!(reversed.total(), 0); + } + + #[test] + fn vec_progress_total() { + #![allow(clippy::single_range_in_vec_init)] + let v: Vec = vec![1..5, 8..10, 12..15]; + assert_eq!(v.total(), 9); + let empty: Vec = vec![]; + assert_eq!(empty.total(), 0); + } +} diff --git a/crates/fast-pull/src/base/puller.rs b/crates/fast-pull/src/base/puller.rs index d37b641..524c2f1 100644 --- a/crates/fast-pull/src/base/puller.rs +++ b/crates/fast-pull/src/base/puller.rs @@ -1,3 +1,5 @@ +//! The [`Puller`](crate::Puller) trait: an abstraction over a chunked data source. + use crate::ProgressEntry; use bytes::Bytes; use core::time::Duration; @@ -25,6 +27,12 @@ pub type PullResult = Result)>; /// specific byte range. Cloning is required for retry and work-stealing scenarios. pub trait Puller: Send + Sync + Clone + 'static { type Error: PullerError; + /// Pull a (sub)range of the source as a stream of byte chunks. + /// + /// Passing `None` for `range` requests the entire source. The returned + /// [`PullStream`] yields [`Bytes`] chunks; each error carries an optional + /// retry delay that the engine honors via its retry backoff. Implementors + /// must be `Clone` so workers can be spawned and work can be stolen/retried. fn pull( &mut self, range: Option<&ProgressEntry>, @@ -33,6 +41,11 @@ pub trait Puller: Send + Sync + Clone + 'static { /// Extension trait for pull errors, distinguishing recoverable from irrecoverable failures. pub trait PullerError: std::error::Error + Send + Sync + Unpin + 'static { + /// Whether an error is fatal and must **not** be retried. + /// + /// The default (`false`) means the error is recoverable and the engine will + /// retry after the configured backoff. Return `true` to stop retrying and + /// abort the affected worker. fn is_irrecoverable(&self) -> bool { false } diff --git a/crates/fast-pull/src/base/pusher.rs b/crates/fast-pull/src/base/pusher.rs index 9ad0378..fdcc3a0 100644 --- a/crates/fast-pull/src/base/pusher.rs +++ b/crates/fast-pull/src/base/pusher.rs @@ -1,3 +1,5 @@ +//! The [`Pusher`](crate::Pusher) trait: an abstraction over a chunked data sink. + use crate::ProgressEntry; use bytes::Bytes; @@ -14,8 +16,18 @@ pub type ProgressListener = Box; /// The pusher writes data to its destination and can optionally flush. pub trait Pusher: Send + 'static { type Error: std::error::Error + Send + Sync + Unpin + 'static; + /// Write `content` covering the given `range` to the destination. + /// + /// On success returns `Ok(())`. On failure returns `Err((error, bytes))` + /// where `bytes` is the (possibly partial) payload that was **not** written, + /// so the engine can retry it. Implementors should keep already-written + /// bytes internally on failure rather than dropping them. #[allow(clippy::missing_errors_doc)] fn push(&mut self, range: &ProgressEntry, content: Bytes) -> Result<(), (Self::Error, Bytes)>; + /// Flush any buffered data to the destination. + /// + /// The default implementation is a no-op. File-backed pushers use this to + /// issue `fsync` / `flush` on the underlying file. #[allow(clippy::missing_errors_doc)] fn flush(&mut self) -> Result<(), Self::Error> { Ok(()) @@ -41,6 +53,7 @@ impl std::error::Error for Box {} /// Useful for FFI boundaries or heterogeneous collections of pushers. #[allow(missing_debug_implementations)] pub struct BoxPusher { + /// The boxed, type-erased inner pusher. pub pusher: Box>>, } impl Pusher for BoxPusher { diff --git a/crates/fast-pull/src/cache/buf_writer.rs b/crates/fast-pull/src/cache/buf_writer.rs index 8337fe3..5e32db9 100644 --- a/crates/fast-pull/src/cache/buf_writer.rs +++ b/crates/fast-pull/src/cache/buf_writer.rs @@ -1,3 +1,6 @@ +//! A `std::io::BufWriter`-style contiguous write buffer for +//! [`Pusher`](crate::Pusher)s. + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::{Bytes, BytesMut}; diff --git a/crates/fast-pull/src/cache/direct.rs b/crates/fast-pull/src/cache/direct.rs index 702ceb7..9c287ab 100644 --- a/crates/fast-pull/src/cache/direct.rs +++ b/crates/fast-pull/src/cache/direct.rs @@ -1,3 +1,5 @@ +//! Pusher cache that flushes contiguous runs without byte merging. + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::Bytes; use std::collections::BTreeMap; @@ -18,6 +20,10 @@ pub struct CacheDirectPusher

{ } impl CacheDirectPusher

{ + /// Wrap `inner` with the given `high_watermark` / `low_watermark` (in bytes). + /// + /// Eviction to the inner pusher triggers once the buffered size reaches + /// `high_watermark`, and stops once it falls back to `low_watermark`. pub const fn new(inner: P, high_watermark: usize, low_watermark: usize) -> Self { Self { inner, diff --git a/crates/fast-pull/src/cache/merge.rs b/crates/fast-pull/src/cache/merge.rs index 64d3ea7..83915e8 100644 --- a/crates/fast-pull/src/cache/merge.rs +++ b/crates/fast-pull/src/cache/merge.rs @@ -1,3 +1,5 @@ +//! Pusher cache that merges each flush run into a single contiguous buffer. + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::{Bytes, BytesMut}; use std::collections::{BTreeMap, btree_map::Entry}; @@ -18,6 +20,10 @@ pub struct CacheMergePusher

{ } impl CacheMergePusher

{ + /// Wrap `inner` with the given `high_watermark` / `low_watermark` (in bytes). + /// + /// Eviction to the inner pusher triggers once the buffered size reaches + /// `high_watermark`, and stops once it falls back to `low_watermark`. pub const fn new(inner: P, high_watermark: usize, low_watermark: usize) -> Self { Self { inner, @@ -139,3 +145,114 @@ impl Pusher for CacheMergePusher

{ self.inner.flush() } } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + /// A `Pusher` that records everything pushed into a shared buffer so tests + /// can inspect it without reaching into `CacheMergePusher`'s private fields. + #[derive(Clone)] + struct SharedSink { + pushes: Arc>>, + fail_next: Arc, + } + impl SharedSink { + fn new() -> Self { + Self { + pushes: Arc::new(Mutex::new(Vec::new())), + fail_next: Arc::new(AtomicBool::new(false)), + } + } + } + impl Pusher for SharedSink { + type Error = std::io::Error; + fn set_listener(&mut self, _: ProgressListener) {} + fn push( + &mut self, + range: &ProgressEntry, + bytes: Bytes, + ) -> Result<(), (Self::Error, Bytes)> { + if self.fail_next.fetch_and(false, Ordering::SeqCst) { + return Err((std::io::Error::other("boom"), bytes)); + } + self.pushes.lock().unwrap().push((range.clone(), bytes)); + Ok(()) + } + fn flush(&mut self) -> Result<(), Self::Error> { + Ok(()) + } + } + + fn bb(s: &str) -> Bytes { + Bytes::copy_from_slice(s.as_bytes()) + } + + #[test] + fn test_cache_merge_evicts_contiguous_run() { + let sink = SharedSink::new(); + let mut p = CacheMergePusher::new(sink.clone(), 30, 0); + // Out-of-order insertion; not yet at watermark. + p.push(&(0..10), bb(&"A".repeat(10))).unwrap(); + p.push(&(20..30), bb(&"C".repeat(10))).unwrap(); + assert!(sink.pushes.lock().unwrap().is_empty()); + // This insertion reaches the high watermark and triggers a merge+evict. + p.push(&(10..20), bb(&"B".repeat(10))).unwrap(); + let pushes = sink.pushes.lock().unwrap(); + assert_eq!(pushes.len(), 1); + assert_eq!(pushes[0].0, 0..30); + assert_eq!(pushes[0].1.len(), 30); + // Merged bytes preserve ascending order: A(0..10) B(10..20) C(20..30). + assert_eq!(&pushes[0].1[..], b"AAAAAAAAAABBBBBBBBBBCCCCCCCCCC"); + drop(pushes); + } + + #[test] + fn test_cache_merge_no_evict_below_watermark() { + let sink = SharedSink::new(); + let mut p = CacheMergePusher::new(sink.clone(), 100, 0); + p.push(&(0..10), bb(&"A".repeat(10))).unwrap(); + p.push(&(10..20), bb(&"B".repeat(10))).unwrap(); + assert!(sink.pushes.lock().unwrap().is_empty()); + // Explicit flush must drain the buffered run to the inner pusher. + p.flush().unwrap(); + let pushes = sink.pushes.lock().unwrap(); + assert_eq!(pushes.len(), 1); + assert_eq!(pushes[0].0, 0..20); + drop(pushes); + } + + #[test] + fn test_cache_merge_inner_failure_propagates() { + let sink = SharedSink::new(); + sink.fail_next.store(true, Ordering::SeqCst); + let mut p = CacheMergePusher::new(sink, 10, 0); + let res = p.push(&(0..10), bb(&"A".repeat(10))); + assert!(res.is_err()); + } + + #[test] + fn test_cache_merge_empty_bytes_is_noop() { + let sink = SharedSink::new(); + let mut p = CacheMergePusher::new(sink.clone(), 1, 0); + p.push(&(0..0), Bytes::new()).unwrap(); + assert!(sink.pushes.lock().unwrap().is_empty()); + } + + #[test] + fn test_cache_merge_partial_overwrite_replaces() { + // A chunk at an already-cached start position replaces the old bytes. + let sink = SharedSink::new(); + let mut p = CacheMergePusher::new(sink.clone(), 100, 0); + p.push(&(0..10), bb(&"A".repeat(10))).unwrap(); + p.push(&(0..10), bb(&"B".repeat(10))).unwrap(); + p.flush().unwrap(); + let pushes = sink.pushes.lock().unwrap(); + assert_eq!(pushes.len(), 1); + assert_eq!(&pushes[0].1[..], b"BBBBBBBBBB"); + drop(pushes); + } +} diff --git a/crates/fast-pull/src/cache/mod.rs b/crates/fast-pull/src/cache/mod.rs index caa539c..5361d7c 100644 --- a/crates/fast-pull/src/cache/mod.rs +++ b/crates/fast-pull/src/cache/mod.rs @@ -1,3 +1,13 @@ +//! Pusher decorators that buffer out-of-order chunks before writing. +//! +//! Each decorator in this module wraps an inner [`Pusher`](crate::Pusher) and +//! absorbs out-of-order writes using a `BTreeMap` keyed by `range.start`, +//! flushing runs to the inner sink once a watermark is reached. Variants differ +//! in how runs are emitted: [`CacheDirectPusher`] flushes each chunk as-is, +//! [`CacheMergePusher`] coalesces each run into one contiguous buffer, and +//! [`CacheSeqPusher`] reorders runs into sequential order. [`BufWriterPusher`] +//! instead mimics `std::io::BufWriter` with a fixed-size contiguous buffer. + mod buf_writer; mod direct; mod merge; diff --git a/crates/fast-pull/src/cache/seq.rs b/crates/fast-pull/src/cache/seq.rs index eccb1b0..2cc71ee 100644 --- a/crates/fast-pull/src/cache/seq.rs +++ b/crates/fast-pull/src/cache/seq.rs @@ -1,3 +1,5 @@ +//! Pusher cache that reorders out-of-order chunks into sequential order. + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::Bytes; use std::collections::BTreeMap; @@ -17,6 +19,10 @@ pub struct CacheSeqPusher

{ } impl CacheSeqPusher

{ + /// Wrap `inner` with the given `high_watermark` / `low_watermark` (in bytes). + /// + /// Eviction to the inner pusher triggers once the buffered size reaches + /// `high_watermark`, and stops once it falls back to `low_watermark`. pub const fn new(inner: P, high_watermark: usize, low_watermark: usize) -> Self { Self { inner, diff --git a/crates/fast-pull/src/core/handle.rs b/crates/fast-pull/src/core/handle.rs index 4cbc452..ff6a8bc 100644 --- a/crates/fast-pull/src/core/handle.rs +++ b/crates/fast-pull/src/core/handle.rs @@ -1,3 +1,5 @@ +//! A shareable, multi-consumer handle to a tokio task. + use std::sync::Arc; use tokio::{ sync::watch, @@ -9,7 +11,7 @@ use tokio::{ /// Unlike a raw [`JoinHandle`], [`SharedHandle`] can be cloned and awaited /// concurrently without consuming the result. The first awaiter gets the result, /// subsequent awaiters will see the same cached result. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct SharedHandle { rx: watch::Receiver>>>, } diff --git a/crates/fast-pull/src/core/mock.rs b/crates/fast-pull/src/core/mock.rs index cd9db51..8eb4bb8 100644 --- a/crates/fast-pull/src/core/mock.rs +++ b/crates/fast-pull/src/core/mock.rs @@ -1,3 +1,6 @@ +//! A deterministic in-memory [`Puller`](crate::Puller) for tests, plus a helper +//! to build mock payloads. + use crate::{ProgressEntry, PullResult, PullStream, Puller}; use futures::stream; use std::{sync::Arc, vec::Vec}; diff --git a/crates/fast-pull/src/core/mod.rs b/crates/fast-pull/src/core/mod.rs index 291d29d..850dd4d 100644 --- a/crates/fast-pull/src/core/mod.rs +++ b/crates/fast-pull/src/core/mod.rs @@ -1,9 +1,17 @@ +//! Top-level download orchestration: session handle plus single- and +//! multi-threaded entry points. +//! +//! [`download_single`](crate::single::download_single) runs a sequential pull, +//! while [`download_multi`](crate::multi::download_multi) splits the work across +//! concurrent workers with work-stealing. Both return a [`DownloadResult`], a +//! cheaply cloneable handle that keeps the session alive until the last clone is +//! dropped (or [`DownloadResult::abort`] is called). + use crate::{Event, handle::SharedHandle}; use core::sync::atomic::{AtomicBool, Ordering}; use crossfire::{MAsyncRx, mpmc}; use fast_steal::{Executor, Handle, TaskQueue}; use std::fmt; -use std::ops::Deref; use std::sync::{Arc, OnceLock, Weak}; use std::thread::Thread; use tokio::task::{AbortHandle, JoinError, JoinHandle}; @@ -21,13 +29,13 @@ pub mod single; /// cancellation happens, giving `DownloadResult` `Arc`-style "last owner gone → /// release" semantics: the download keeps running as long as any handle is /// alive, and is cancelled only when the final one is dropped. -pub struct DownloadResultInner +struct DownloadResultInner where E: Executor + Send + Sync, PullError: Send + Unpin + 'static, PushError: Send + Unpin + 'static, { - pub event_chain: MAsyncRx>>, + event_chain: MAsyncRx>>, handle: SharedHandle<()>, abort_handles: Option>, task_queue: Option<(Weak, TaskQueue)>, @@ -135,11 +143,11 @@ where /// /// Cheaply cloneable shared handle. The underlying download keeps running as /// long as **any** clone is alive, and is cancelled only once the last clone is -/// dropped. An explicit [`DownloadResultInner::abort`] cancels immediately. +/// dropped. An explicit [`abort`](Self::abort) cancels immediately. /// -/// `DownloadResult` derefs to [`DownloadResultInner`], so all session methods -/// (`join`, `abort`, `set_threads`, `is_aborted`) and the `event_chain` field -/// are reachable directly on the handle. +/// `DownloadResult` derefs to `DownloadResultInner`, so all session methods +/// (`join`, `abort`, `set_threads`, `is_aborted`) and the [`event_chain`](Self::event_chain) +/// method are reachable directly on the handle. #[derive(Debug)] pub struct DownloadResult where @@ -163,25 +171,18 @@ where } } -impl Deref for DownloadResult -where - E: Executor + Send + Sync, - PullError: Send + Unpin + 'static, - PushError: Send + Unpin + 'static, -{ - type Target = DownloadResultInner; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - impl DownloadResult where E: Executor + Send + Sync, PullError: Send + Unpin + 'static, PushError: Send + Unpin + 'static, { + /// Construct a [`DownloadResult`] from the raw session pieces. + /// + /// This is an internal constructor used by + /// [`download_single`](crate::single::download_single) and + /// [`download_multi`](crate::multi::download_multi); prefer those entry + /// points instead of calling this directly. pub fn new( event_chain: MAsyncRx>>, handle: JoinHandle<()>, @@ -201,4 +202,42 @@ where }), } } + + /// Access the stream of [`Event`]s emitted during the session. + /// + /// The receiver closes once the last clone of this handle is dropped or the + /// session is aborted, so draining it is a natural way to observe progress. + #[must_use] + pub fn event_chain(&self) -> &MAsyncRx>> { + &self.inner.event_chain + } + + /// # Errors + /// Returns `Arc` if the writer thread exits unexpectedly + pub async fn join(&self) -> Result<(), Arc> { + self.inner.join().await + } + + /// Cancel all workers immediately. + /// + /// Safe to call multiple times and safe to call while other clones of the + /// owning [`DownloadResult`] are still alive. The implicit drop-based + /// cancellation (on the last clone) becomes a no-op once this has run. + pub fn abort(&self) { + self.inner.abort(); + } + + /// Adjust the worker thread count and minimum chunk size of a running + /// multi-threaded session. + /// + /// No-op for single-threaded sessions, which have no task queue. + pub fn set_threads(&self, threads: usize, min_chunk_size: u64) { + self.inner.set_threads(threads, min_chunk_size); + } + + /// Whether the session has been (or is being) cancelled. + #[must_use] + pub fn is_aborted(&self) -> bool { + self.inner.is_aborted() + } } diff --git a/crates/fast-pull/src/core/multi.rs b/crates/fast-pull/src/core/multi.rs index 4d77d19..ffb2320 100644 --- a/crates/fast-pull/src/core/multi.rs +++ b/crates/fast-pull/src/core/multi.rs @@ -1,3 +1,5 @@ +//! Multi-threaded concurrent download with work-stealing. + use crate::{DownloadResult, Event, ProgressEntry, Puller, PullerError, Pusher, WorkerId}; use bytes::Bytes; use core::{ @@ -302,7 +304,7 @@ mod tests { let mut pull_progress: Vec = Vec::new(); let mut push_progress: Vec = Vec::new(); let mut pull_ids = [false; 32]; - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { match e { Event::PullProgress(id, p) => { pull_ids[id] = true; @@ -447,7 +449,7 @@ mod tests { // Abort as soon as the push driver starts processing (first `Pushing`). let mut aborted = false; - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { if matches!(e, Event::Pushing(_, _)) { result.abort(); assert!(result.is_aborted()); diff --git a/crates/fast-pull/src/core/single.rs b/crates/fast-pull/src/core/single.rs index de27264..9acb512 100644 --- a/crates/fast-pull/src/core/single.rs +++ b/crates/fast-pull/src/core/single.rs @@ -1,3 +1,5 @@ +//! Single-threaded sequential download. + use crate::{ DownloadResult, Event, ProgressEntry, Puller, PullerError, Pusher, multi::TokioExecutor, }; @@ -155,7 +157,7 @@ mod tests { let mut pull_progress: Vec = Vec::new(); let mut push_progress: Vec = Vec::new(); - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { match e { Event::PullProgress(_, p) => pull_progress.merge_progress(p), Event::PushProgress(p) => push_progress.merge_progress(p), @@ -284,7 +286,7 @@ mod tests { // event). This lands reliably before completion because the puller is // slow. let mut aborted = false; - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { if matches!(e, Event::Pushing(_, _)) { result.abort(); assert!(result.is_aborted()); @@ -339,7 +341,7 @@ mod tests { ); let mut aborted = false; - while let Ok(e) = result.event_chain.recv().await { + while let Ok(e) = result.event_chain().recv().await { if matches!(e, Event::Pushing(_, _)) { result.abort(); assert!(result.is_aborted()); diff --git a/crates/fast-pull/src/file/cache_std.rs b/crates/fast-pull/src/file/cache_std.rs index 7a97a6e..fb88c21 100644 --- a/crates/fast-pull/src/file/cache_std.rs +++ b/crates/fast-pull/src/file/cache_std.rs @@ -1,3 +1,5 @@ +//! High-level file pusher combining caching, buffering, and standard file I/O. + use crate::{ BufWriterPusher, CacheSeqPusher, ProgressEntry, ProgressListener, Pusher, file::StdFilePusher, }; diff --git a/crates/fast-pull/src/file/mmap.rs b/crates/fast-pull/src/file/mmap.rs index ce7798e..906a22b 100644 --- a/crates/fast-pull/src/file/mmap.rs +++ b/crates/fast-pull/src/file/mmap.rs @@ -1,3 +1,5 @@ +//! Memory-mapped file pusher (feature `file`). + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::Bytes; use memmap2::MmapMut; diff --git a/crates/fast-pull/src/file/mod.rs b/crates/fast-pull/src/file/mod.rs index 650fbdf..85d89e6 100644 --- a/crates/fast-pull/src/file/mod.rs +++ b/crates/fast-pull/src/file/mod.rs @@ -1,3 +1,10 @@ +//! File-backed [`Pusher`](crate::Pusher) implementations (feature `file`). +//! +//! Provides [`StdFilePusher`] (raw `std::fs::File` random-access writes), +//! [`MmapFilePusher`] (memory-mapped zero-copy writes), and [`CacheFilePusher`] +//! (a ready-made `CacheSeqPusher>` stack). +//! Enabled by the `file` feature. + mod cache_std; mod mmap; mod std; diff --git a/crates/fast-pull/src/file/std.rs b/crates/fast-pull/src/file/std.rs index 945af1d..b251b41 100644 --- a/crates/fast-pull/src/file/std.rs +++ b/crates/fast-pull/src/file/std.rs @@ -1,3 +1,5 @@ +//! Raw `std::fs::File` random-access pusher (feature `file`). + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::Bytes; use std::{ diff --git a/crates/fast-pull/src/mem/mod.rs b/crates/fast-pull/src/mem/mod.rs index 0ca5edd..c8437d4 100644 --- a/crates/fast-pull/src/mem/mod.rs +++ b/crates/fast-pull/src/mem/mod.rs @@ -1,3 +1,5 @@ +//! In-memory [`Pusher`](crate::Pusher) implementation (feature `mem`). + mod pusher; pub use pusher::*; diff --git a/crates/fast-pull/src/mem/pusher.rs b/crates/fast-pull/src/mem/pusher.rs index 5ed81ee..17fab5d 100644 --- a/crates/fast-pull/src/mem/pusher.rs +++ b/crates/fast-pull/src/mem/pusher.rs @@ -1,3 +1,5 @@ +//! In-memory pusher backed by a shared `Vec` (feature `mem`). + use crate::{ProgressEntry, ProgressListener, Pusher}; use bytes::Bytes; use parking_lot::Mutex; @@ -68,3 +70,57 @@ impl Pusher for MemPusher { Ok(()) } } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use std::sync::{Arc, Mutex}; + + #[test] + fn sequential_append() { + let mut p = MemPusher::new(); + p.push(&(0..3), Bytes::copy_from_slice(b"abc")).unwrap(); + p.push(&(3..6), Bytes::copy_from_slice(b"def")).unwrap(); + assert_eq!(&p.receive.lock()[..], b"abcdef"); + } + + #[test] + fn random_access_resizes_and_writes() { + let mut p = MemPusher::new(); + p.push(&(5..8), Bytes::copy_from_slice(b"xyz")).unwrap(); + // The gap before the write is filled with zeros via resize. + assert_eq!(p.receive.lock().len(), 8); + p.push(&(0..5), Bytes::copy_from_slice(b"hello")).unwrap(); + assert_eq!(&p.receive.lock()[..], b"helloxyz"); + } + + #[test] + fn empty_push_is_noop() { + let mut p = MemPusher::new(); + p.push(&(0..0), Bytes::new()).unwrap(); + assert!(p.receive.lock().is_empty()); + } + + #[test] + fn listener_invoked_with_range() { + let mut p = MemPusher::new(); + let seen = Arc::new(Mutex::new(None::)); + let seen2 = seen.clone(); + p.set_listener(Box::new(move |r| { + *seen2.lock().unwrap() = Some(r); + })); + p.push(&(0..4), Bytes::copy_from_slice(b"data")).unwrap(); + assert_eq!(*seen.lock().unwrap(), Some(0..4)); + } + + #[test] + fn clone_shares_receive() { + let mut p = MemPusher::new(); + p.push(&(0..2), Bytes::copy_from_slice(b"hi")).unwrap(); + let mut q = p.clone(); + q.push(&(2..4), Bytes::copy_from_slice(b"ya")).unwrap(); + // Both handles observe the same underlying vec. + assert_eq!(&p.receive.lock()[..], b"hiya"); + } +} diff --git a/crates/fast-steal/README.md b/crates/fast-steal/README.md index 51fe1f0..f09639c 100644 --- a/crates/fast-steal/README.md +++ b/crates/fast-steal/README.md @@ -2,6 +2,7 @@ [![GitHub last commit](https://img.shields.io/github/last-commit/fast-down/core/main)](https://github.com/fast-down/core/commits/main) [![Test](https://github.com/fast-down/core/workflows/Test/badge.svg)](https://github.com/fast-down/core/actions) +[![codecov](https://codecov.io/gh/fast-down/core/branch/main/graph/badge.svg)](https://codecov.io/gh/fast-down/core) [![Latest version](https://img.shields.io/crates/v/fast-steal.svg)](https://crates.io/crates/fast-steal) [![Documentation](https://docs.rs/fast-steal/badge.svg)](https://docs.rs/fast-steal) [![License](https://img.shields.io/crates/l/fast-steal.svg)](https://github.com/fast-down/core/blob/main/LICENSE) diff --git a/crates/fast-steal/src/executor.rs b/crates/fast-steal/src/executor.rs index 29bd4a7..6241a6a 100644 --- a/crates/fast-steal/src/executor.rs +++ b/crates/fast-steal/src/executor.rs @@ -1,3 +1,8 @@ +//! Traits that connect a [`TaskQueue`](crate::TaskQueue) to an async runtime. +//! +//! Implement [`Executor`] to spawn work onto your runtime, and [`Handle`] to give the +//! queue a way to identify and abort the tasks it spawned. + use crate::{Task, TaskQueue}; /// User-defined executor that runs tasks on a [`TaskQueue`]. diff --git a/crates/fast-steal/src/lib.rs b/crates/fast-steal/src/lib.rs index 312a439..08db8a1 100644 --- a/crates/fast-steal/src/lib.rs +++ b/crates/fast-steal/src/lib.rs @@ -1,4 +1,17 @@ #![no_std] +//! `fast-steal` provides a `no_std`-compatible building block for work-stealing +//! schedulers. +//! +//! The crate is built around three public types: +//! - [`Task`] — a lock-free, cancellable unit of work tracking a `start..end` range. +//! - [`TaskQueue`] — a concurrent queue that hands out work and steals sub-ranges +//! from busy workers. +//! - [`Executor`] / [`Handle`] — traits you implement to plug the queue into any +//! async runtime. +//! +//! Only [`alloc`](https://doc.rust-lang.org/alloc/) is required by the core paths; +//! `std` is used exclusively in tests and doctests. For a complete runnable example, +//! see the crate-level documentation (the included README) below. #![doc = include_str!("../README.md")] mod executor; diff --git a/crates/fast-steal/src/task.rs b/crates/fast-steal/src/task.rs index bf47958..d743c39 100644 --- a/crates/fast-steal/src/task.rs +++ b/crates/fast-steal/src/task.rs @@ -1,3 +1,10 @@ +//! Cancellable, lock-free units of work. +//! +//! A [`Task`] stores its remaining work as a single atomic `u128` packing the +//! `start` (high 64 bits) and `end` (low 64 bits) bounds, which lets multiple +//! worker threads advance the same task without locking. Because the crate is +//! `no_std`, only `alloc` is required for the reference-counted state. + extern crate alloc; use alloc::sync::{Arc, Weak}; use core::{fmt, ops::Range, sync::atomic::Ordering}; @@ -8,8 +15,15 @@ use portable_atomic::AtomicU128; /// The range is stored as a single atomic `u128`, allowing lock-free reads and /// fine-grained progress updates. Multiple workers can safely steal sub-ranges /// from the same task via [`split_two`](Task::split_two). +/// +/// Two `Task`s are equal iff they point to the same underlying state (see the +/// `PartialEq` impl, which uses `Arc::ptr_eq`). #[derive(Debug, Clone)] pub struct Task { + /// Atomic state packing `start` (high 64 bits) and `end` (low 64 bits). + /// + /// Prefer the safe accessors ([`Task::start`], [`Task::end`], [`Task::get`], + /// [`Task::safe_add_start`]); this field is exposed for advanced use. pub state: Arc, } /// A weak reference to a [`Task`], obtained via [`Task::downgrade`]. @@ -18,18 +32,25 @@ pub struct Task { /// to attempt to obtain a strong [`Task`] reference. #[derive(Debug, Clone)] pub struct WeakTask { + /// Weak reference to the underlying atomic state of the originating [`Task`]. pub state: Weak, } impl WeakTask { + /// Attempts to upgrade to a strong [`Task`]. + /// + /// Returns `None` if all strong references to the underlying state have already + /// been dropped. #[must_use] pub fn upgrade(&self) -> Option { self.state.upgrade().map(|state| Task { state }) } + /// Returns the number of strong [`Task`] references to the underlying state. #[must_use] pub fn strong_count(&self) -> usize { self.state.strong_count() } + /// Returns the number of weak [`WeakTask`] references to the underlying state. #[must_use] pub fn weak_count(&self) -> usize { self.state.weak_count() @@ -71,18 +92,27 @@ impl Task { state: Arc::new(AtomicU128::new(Self::pack(range))), } } + /// Returns the current `start..end` range, loaded atomically with `Acquire` + /// ordering. #[must_use] pub fn get(&self) -> Range { let state = self.state.load(Ordering::Acquire); Self::unpack(state) } + /// Returns the current start of the work range (the high 64 bits of the atomic + /// state). #[must_use] pub fn start(&self) -> u64 { (self.state.load(Ordering::Acquire) >> 64) as u64 } + /// Atomically advances `start` to `min(start + bias, end)`, but only if that + /// makes forward progress; then returns the slice that was claimed. + /// /// # Errors - /// Returns [`RangeError`] when `start` + `bias` <= `old_start` - /// Otherwise returns `old_start..new_start.min(end)` + /// Returns [`RangeError`] when `start + bias` would not exceed the current + /// `start` (no progress, a non-positive bias, or `u64` overflow), or when the + /// task range invariant `start <= end` is violated. On success returns the + /// claimed `old_start..new_start` sub-range. pub fn safe_add_start(&self, start: u64, bias: u64) -> Result, RangeError> { let new_start = start.checked_add(bias).ok_or(RangeError)?; let mut old_state = self.state.load(Ordering::Acquire); @@ -106,6 +136,8 @@ impl Task { } } } + /// Returns the current end of the work range (the low 64 bits of the atomic + /// state). #[must_use] pub fn end(&self) -> u64 { let state = self.state.load(Ordering::Acquire); @@ -113,6 +145,7 @@ impl Task { let end = state as u64; end } + /// Returns `end - start` (saturating), i.e. how much work is left. #[must_use] pub fn remain(&self) -> u64 { let range = self.get(); @@ -144,6 +177,10 @@ impl Task { } } } + /// Atomically claims and returns the entire remaining range `start..end`, + /// emptying this task (sets `start = end`). + /// + /// Returns `None` if the task is already empty. #[must_use] pub fn take(&self) -> Option> { let mut old_state = self.state.load(Ordering::Acquire); @@ -164,21 +201,28 @@ impl Task { } } } + /// Creates a [`WeakTask`] that does not keep the task's state alive. #[must_use] pub fn downgrade(&self) -> WeakTask { WeakTask { state: Arc::downgrade(&self.state), } } + /// Returns the number of strong ([`Task`]) references to this task's state. #[must_use] pub fn strong_count(&self) -> usize { Arc::strong_count(&self.state) } + /// Returns the number of weak ([`WeakTask`]) references to this task's state. #[must_use] pub fn weak_count(&self) -> usize { Arc::weak_count(&self.state) } } +/// Creates a [`Task`] from a `start..end` range. +/// +/// # Panics +/// Panics (via [`Task::new`]) if `range.start > range.end`. impl From> for Task { fn from(value: Range) -> Self { Self::new(value) @@ -238,4 +282,65 @@ mod tests { assert_eq!(task.end(), 2); assert_eq!(range, None); } + + #[test] + fn test_safe_add_start_no_progress() { + let task = Task::new(10..20); + // bias 0 -> start does not advance + assert_eq!(task.safe_add_start(10, 0), Err(RangeError)); + // bias would not exceed current start + assert_eq!(task.safe_add_start(8, 2), Err(RangeError)); + } + + #[test] + fn test_safe_add_start_claims_span() { + let task = Task::new(10..20); + let span = task.safe_add_start(10, 5).unwrap(); + assert_eq!(span, 10..15); + assert_eq!(task.start(), 15); + assert_eq!(task.remain(), 5); + } + + #[test] + fn test_safe_add_start_capped_at_end() { + let task = Task::new(10..12); + let span = task.safe_add_start(10, 100).unwrap(); + assert_eq!(span, 10..12); + assert_eq!(task.remain(), 0); + } + + #[test] + fn test_take_empties() { + let task = Task::new(5..9); + assert_eq!(task.take(), Some(5..9)); + assert_eq!(task.take(), None); + assert_eq!(task.remain(), 0); + } + + #[test] + fn test_downgrade_upgrade() { + let task = Task::new(1..10); + let weak = task.downgrade(); + assert_eq!(weak.strong_count(), 1); + assert_eq!(weak.upgrade().unwrap().get(), 1..10); + drop(task); + assert_eq!(weak.upgrade(), None); + } + + #[test] + fn test_partial_eq_by_ptr() { + let a = Task::new(1..10); + let b = a.clone(); + assert_eq!(a, b); + let c = Task::new(1..10); + assert_ne!(a, c); + } + + #[test] + fn test_split_two_halves() { + let task = Task::new(0..100); + let range = task.split_two().unwrap().unwrap(); + assert_eq!(range, 50..100); + assert_eq!(task.get(), 0..50); + } } diff --git a/crates/fast-steal/src/task_queue.rs b/crates/fast-steal/src/task_queue.rs index c4704c7..28a96fb 100644 --- a/crates/fast-steal/src/task_queue.rs +++ b/crates/fast-steal/src/task_queue.rs @@ -1,3 +1,10 @@ +//! A concurrent work-stealing queue. +//! +//! [`TaskQueue`] holds pending and running [`Task`](crate::Task)s and lets worker +//! threads pull fresh work or steal a sub-range from a busy peer via +//! [`steal`](TaskQueue::steal). The number of running workers can be adjusted at +//! runtime with [`set_threads`](TaskQueue::set_threads). + extern crate alloc; use crate::{Executor, Handle, Task, WeakTask}; use alloc::{collections::vec_deque::VecDeque, sync::Arc, vec::Vec}; @@ -26,6 +33,8 @@ struct TaskQueueInner { waiting: VecDeque, } impl TaskQueue { + /// Creates a queue from an iterator of `start..end` ranges, each wrapped in its + /// own [`Task`]. pub fn new(tasks: impl Iterator>) -> Self { let waiting: VecDeque<_> = tasks.map(Task::from).collect(); Self { @@ -35,10 +44,25 @@ impl TaskQueue { })), } } + /// Appends a [`Task`] to the waiting queue so a future + /// [`steal`](TaskQueue::steal) or [`set_threads`](TaskQueue::set_threads) can + /// pick it up. pub fn add(&self, task: Task) { let mut guard = self.inner.lock(); guard.waiting.push_back(task); } + /// Tries to refill `task` with more work for the worker identified by `id`. + /// + /// The caller must pass its own currently-held [`Task`] plus `id` (compared via + /// [`Handle::is_self`](crate::Handle::is_self)). The function first hands out a + /// pending task from the waiting queue; if none is available it steals a half + /// range from the busiest running task via [`Task::split_two`](crate::Task::split_two) + /// (when at least `min_chunk_size * 2` work remains), or, if `max_speculative > 1` + /// and the stolen task has few enough strong references, shares that same task + /// speculatively. + /// + /// Returns `true` if `task` was refilled, or `false` if the worker is not + /// registered or no work could be found. pub fn steal( &self, id: &H::Id, @@ -143,6 +167,8 @@ impl TaskQueue { } Some(()) } + /// Provides mutable access to the handles of all running tasks, e.g. to abort + /// or inspect them. pub fn handles(&self, f: F) -> R where F: FnOnce(&mut dyn Iterator) -> R, @@ -153,6 +179,8 @@ impl TaskQueue { f(&mut iter) } + /// Aborts every running task equal to `task` that does not belong to the + /// worker `id`, reclaiming it back into the waiting queue. pub fn cancel_task(&self, task: &Task, id: &H::Id) { let mut guard = self.inner.lock(); for (weak, handle) in &mut guard.running {