From dddb2a396e099bf03812040de749652c9703b680 Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Sat, 14 Mar 2026 20:44:15 -0700 Subject: [PATCH 1/2] feat: make pcre2 an optional cargo feature with docs Add [features] section to Cargo.toml with pcre2 as a default feature. Gate all pcre2-related code in split.rs behind #[cfg(feature = pcre2)] so consumers can opt out with default-features = false. Add feature-gated tests for both pcre2-on and fallback paths. Document the feature flag in README.md and add CONTRIBUTING.md with build/test instructions. --- CONTRIBUTING.md | 40 +++++ Cargo.lock | 239 +++----------------------- Cargo.toml | 6 +- README.md | 12 ++ examples/profile_sample.rs | 4 +- examples/profile_stages.rs | 100 ++++++++--- examples/simple_bench.rs | 49 +++++- src/added_tokens.rs | 6 +- src/lib.rs | 187 +++++++++++++-------- src/models/bpe.rs | 54 ++++-- src/post_processors.rs | 19 +-- src/pre_tokenized.rs | 10 +- src/pre_tokenizers/split.rs | 327 +++++++++++++++--------------------- 13 files changed, 505 insertions(+), 548 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..9ff2a51 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,40 @@ +# Contributing to fastokens + +Thank you for your interest in contributing. The notes below cover the development workflow and testing requirements. + +## Building + +```sh +cargo build +``` + +To build without the optional `pcre2` C dependency (pure-Rust build): + +```sh +cargo build --no-default-features +``` + +## Testing +Run the test suite to catch regressions: + +```sh +# Default build — pcre2 JIT fast path enabled +cargo test + +# Pure-Rust build — fancy-regex fallback, no libpcre2-8 required +cargo test --no-default-features +``` + +## Code style + +After making any changes to Rust source files, run the formatter before committing: + +```sh +cargo fmt +``` + +## Feature flags + +| Feature | Default | Notes | +|---------|---------|-------| +| `pcre2` | on | Links to `libpcre2-8` via `pcre2-sys`. Disable with `--no-default-features` if the system library is unavailable. | diff --git a/Cargo.lock b/Cargo.lock index 71600ad..a94bc53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,7 +486,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -559,12 +559,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "foreign-types" version = "0.3.2" @@ -700,19 +694,6 @@ dependencies = [ "wasip2", ] -[[package]] -name = "getrandom" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasip2", - "wasip3", -] - [[package]] name = "h2" version = "0.4.13" @@ -732,15 +713,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", -] - [[package]] name = "hashbrown" version = "0.16.1" @@ -985,12 +957,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -1025,9 +991,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown 0.16.1", - "serde", - "serde_core", + "hashbrown", ] [[package]] @@ -1113,12 +1077,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" version = "0.2.182" @@ -1137,9 +1095,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -1232,9 +1190,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5d26952a508f321b4d3d2e80e78fc2603eaefcdf0c30783867f19586518bdc" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" dependencies = [ "libc", "log", @@ -1309,9 +1267,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ "bitflags", "cfg-if", @@ -1341,9 +1299,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" dependencies = [ "cc", "libc", @@ -1433,16 +1391,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -1626,15 +1574,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1686,18 +1634,18 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] [[package]] name = "security-framework" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", "core-foundation 0.10.1", @@ -1708,20 +1656,14 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.16.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321c8673b092a9a42605034a9879d73cb79101ed5fd117bc9a597b89b4e9e61a" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", ] -[[package]] -name = "semver" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" - [[package]] name = "serde" version = "1.0.228" @@ -1933,15 +1875,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.25.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.1", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2026,9 +1968,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", @@ -2165,12 +2107,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "unicode_categories" version = "0.1.1" @@ -2275,15 +2211,6 @@ dependencies = [ "wit-bindgen", ] -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "wasm-bindgen" version = "0.2.108" @@ -2343,28 +2270,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - [[package]] name = "wasm-streams" version = "0.4.2" @@ -2378,18 +2283,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - [[package]] name = "web-sys" version = "0.3.85" @@ -2655,88 +2548,6 @@ name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] [[package]] name = "write16" diff --git a/Cargo.toml b/Cargo.toml index 9d6188a..6e9e7eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,11 +3,15 @@ name = "fastokens" version = "0.1.0" edition = "2024" +[features] +default = ["pcre2"] +pcre2 = ["dep:pcre2"] + [dependencies] daachorse = "1.0.0" fancy-regex = "0.17.0" memchr = "2" -pcre2 = "0.2" +pcre2 = { version = "0.2", optional = true } hf-hub = "0.4.3" rayon = "1.11.0" serde = { version = "1.0.228", features = ["derive"] } diff --git a/README.md b/README.md index 348c636..ec7e46c 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,18 @@ tokenizer = Tokenizer.from_model("deepseek-ai/DeepSeek-V3.2") tokens = tokenizer.encode("A very long prompt that is now lightning fast.") ``` +## Cargo Features + +| Feature | Default | Description | +|---------|---------|-------------| +| `pcre2` | **enabled** | Uses the PCRE2 JIT engine for faster regex matching in the `Split` pre-tokenizer. Requires `libpcre2-8` on the system. When disabled, the pure-Rust `fancy-regex` engine is used as a fallback with identical output. | + +To opt out of the native C dependency (e.g. in environments where `libpcre2-8` is not available): + +```toml +fastokens = { version = "...", default-features = false } +``` + ## Acknowledgements This library builds on the well-known and widely used Hugging Face tokenizers library and uses code written for HF tokenizers in several flows. diff --git a/examples/profile_sample.rs b/examples/profile_sample.rs index c1f228c..aebac95 100644 --- a/examples/profile_sample.rs +++ b/examples/profile_sample.rs @@ -113,8 +113,8 @@ fn main() -> Result<()> { println!("Warm encode: {:.2} ms", warm_time.as_secs_f64() * 1000.0); // Step 4: HF comparison - let hf = tokenizers::Tokenizer::from_pretrained(model_name, None) - .map_err(|e| anyhow::anyhow!(e))?; + let hf = + tokenizers::Tokenizer::from_pretrained(model_name, None).map_err(|e| anyhow::anyhow!(e))?; let t0 = Instant::now(); let _ = hf .encode_fast(input, true) diff --git a/examples/profile_stages.rs b/examples/profile_stages.rs index cba4cdd..b85f4d4 100644 --- a/examples/profile_stages.rs +++ b/examples/profile_stages.rs @@ -27,7 +27,10 @@ fn main() -> Result<()> { let ids = tokenizer.encode(&input).unwrap(); std::hint::black_box(ids); } - println!("Full encode: {:.2} ms avg ({n} iters)", t.elapsed().as_secs_f64() * 1000.0 / n as f64); + println!( + "Full encode: {:.2} ms avg ({n} iters)", + t.elapsed().as_secs_f64() * 1000.0 / n as f64 + ); // Build the initial pre-tokenized string use fastokens::pre_tokenized::{PreTokenizedString, Split as PtSplit}; @@ -46,12 +49,8 @@ fn main() -> Result<()> { // Create Split pre-tokenizer directly let split_pattern = r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+"; - let split = fastokens::Split::from_config( - &json!({"Regex": split_pattern}), - "Isolated", - false, - ) - .unwrap(); + let split = + fastokens::Split::from_config(&json!({"Regex": split_pattern}), "Isolated", false).unwrap(); // Create ByteLevel pre-tokenizer directly let byte_level = fastokens::ByteLevel::from_config(false, true, false).unwrap(); @@ -66,7 +65,10 @@ fn main() -> Result<()> { let mut pts = base_pts.clone(); split.pre_tokenize(&mut pts).unwrap(); } - println!("Split only: {:.2} ms avg", t.elapsed().as_secs_f64() * 1000.0 / n as f64); + println!( + "Split only: {:.2} ms avg", + t.elapsed().as_secs_f64() * 1000.0 / n as f64 + ); // Profile ByteLevel only (after Split) let mut after_split = base_pts.clone(); @@ -82,7 +84,10 @@ fn main() -> Result<()> { let mut pts = after_split.clone(); byte_level.pre_tokenize(&mut pts).unwrap(); } - println!("ByteLevel only: {:.2} ms avg", t.elapsed().as_secs_f64() * 1000.0 / n as f64); + println!( + "ByteLevel only: {:.2} ms avg", + t.elapsed().as_secs_f64() * 1000.0 / n as f64 + ); // Profile BPE tokenize (non-fused, via full pre-tokenizer) let model = tokenizer.model(); @@ -98,9 +103,15 @@ fn main() -> Result<()> { for s in after_pretok.splits() { let t = after_pretok.split_text(s); unique.insert(t); - if t.len() <= 4 { single_token += 1; } + if t.len() <= 4 { + single_token += 1; + } } - println!("Unique splits: {} (short <=4 bytes: {})", unique.len(), single_token); + println!( + "Unique splits: {} (short <=4 bytes: {})", + unique.len(), + single_token + ); } { @@ -108,10 +119,15 @@ fn main() -> Result<()> { } let t = Instant::now(); for _ in 0..n { - let ids = after_pretok.tokenize(|t, out| model.tokenize_into(t, out)).unwrap(); + let ids = after_pretok + .tokenize(|t, out| model.tokenize_into(t, out)) + .unwrap(); std::hint::black_box(ids); } - println!("BPE tokenize: {:.2} ms avg", t.elapsed().as_secs_f64() * 1000.0 / n as f64); + println!( + "BPE tokenize: {:.2} ms avg", + t.elapsed().as_secs_f64() * 1000.0 / n as f64 + ); // Profile fused BPE tokenize (Split-only + fused BPE) { @@ -123,10 +139,15 @@ fn main() -> Result<()> { for _ in 0..n { let mut pts = base_pts.clone(); split.pre_tokenize(&mut pts).unwrap(); - let ids = pts.tokenize(|t, out| model.tokenize_into_fused(t, out)).unwrap(); + let ids = pts + .tokenize(|t, out| model.tokenize_into_fused(t, out)) + .unwrap(); std::hint::black_box(ids); } - println!("Fused tok: {:.2} ms avg (Split + fused BPE)", t.elapsed().as_secs_f64() * 1000.0 / n as f64); + println!( + "Fused tok: {:.2} ms avg (Split + fused BPE)", + t.elapsed().as_secs_f64() * 1000.0 / n as f64 + ); // Profile BPE tokenize sequential (no rayon) { @@ -134,10 +155,15 @@ fn main() -> Result<()> { } let t = Instant::now(); for _ in 0..n { - let ids = after_pretok.tokenize_sequential_pub(|t, out| model.tokenize_into(t, out)).unwrap(); + let ids = after_pretok + .tokenize_sequential_pub(|t, out| model.tokenize_into(t, out)) + .unwrap(); std::hint::black_box(ids); } - println!("BPE seq: {:.2} ms avg (sequential, no rayon)", t.elapsed().as_secs_f64() * 1000.0 / n as f64); + println!( + "BPE seq: {:.2} ms avg (sequential, no rayon)", + t.elapsed().as_secs_f64() * 1000.0 / n as f64 + ); // Profile iteration overhead only (no BPE, just iterate splits) let t = Instant::now(); @@ -148,7 +174,10 @@ fn main() -> Result<()> { } std::hint::black_box(count); } - println!("Iter only: {:.2} ms avg (iterate splits, no BPE)", t.elapsed().as_secs_f64() * 1000.0 / n as f64); + println!( + "Iter only: {:.2} ms avg (iterate splits, no BPE)", + t.elapsed().as_secs_f64() * 1000.0 / n as f64 + ); // Profile build_pre_tokenized (added tokens + NFC + buffer copy) { @@ -159,7 +188,10 @@ fn main() -> Result<()> { let pts = tokenizer.build_pre_tokenized(&input); std::hint::black_box(pts); } - println!("build_pre_tok: {:.2} ms avg", t.elapsed().as_secs_f64() * 1000.0 / n as f64); + println!( + "build_pre_tok: {:.2} ms avg", + t.elapsed().as_secs_f64() * 1000.0 / n as f64 + ); // Profile NFC normalization alone let normalizer = tokenizer.normalizer(); @@ -175,7 +207,10 @@ fn main() -> Result<()> { std::hint::black_box(r); } } - println!("NFC normalize: {:.2} ms avg", t.elapsed().as_secs_f64() * 1000.0 / n as f64); + println!( + "NFC normalize: {:.2} ms avg", + t.elapsed().as_secs_f64() * 1000.0 / n as f64 + ); // Profile just buffer copy (to measure memcpy cost) let t = Instant::now(); @@ -183,7 +218,10 @@ fn main() -> Result<()> { let s = input.to_string(); std::hint::black_box(s); } - println!("String copy: {:.2} ms avg", t.elapsed().as_secs_f64() * 1000.0 / n as f64); + println!( + "String copy: {:.2} ms avg", + t.elapsed().as_secs_f64() * 1000.0 / n as f64 + ); // Profile contains("<|") prefilter let t = Instant::now(); @@ -191,7 +229,10 @@ fn main() -> Result<()> { let r = input.contains("<|"); std::hint::black_box(r); } - println!("contains <|: {:.2} ms avg (100 iters)", t.elapsed().as_secs_f64() * 1000.0 / 100.0); + println!( + "contains <|: {:.2} ms avg (100 iters)", + t.elapsed().as_secs_f64() * 1000.0 / 100.0 + ); // Profile NFC on the actual input with 100 iters for accuracy let t = Instant::now(); @@ -201,7 +242,10 @@ fn main() -> Result<()> { std::hint::black_box(r); } } - println!("NFC (100 iter): {:.2} ms avg", t.elapsed().as_secs_f64() * 1000.0 / 100.0); + println!( + "NFC (100 iter): {:.2} ms avg", + t.elapsed().as_secs_f64() * 1000.0 / 100.0 + ); // Profile from_text (string allocation + copy) let t = Instant::now(); @@ -209,7 +253,10 @@ fn main() -> Result<()> { let pts = PreTokenizedString::from_text(&input); std::hint::black_box(pts); } - println!("from_text: {:.2} ms avg (100 iters)", t.elapsed().as_secs_f64() * 1000.0 / 100.0); + println!( + "from_text: {:.2} ms avg (100 iters)", + t.elapsed().as_secs_f64() * 1000.0 / 100.0 + ); // Profile memchr for '<' let t = Instant::now(); @@ -217,7 +264,10 @@ fn main() -> Result<()> { let r = input.as_bytes().iter().position(|&b| b == b'<'); std::hint::black_box(r); } - println!("find '<': {:.2} ms avg (100 iters)", t.elapsed().as_secs_f64() * 1000.0 / 100.0); + println!( + "find '<': {:.2} ms avg (100 iters)", + t.elapsed().as_secs_f64() * 1000.0 / 100.0 + ); println!("\nDone."); Ok(()) diff --git a/examples/simple_bench.rs b/examples/simple_bench.rs index 279285e..1b1b832 100644 --- a/examples/simple_bench.rs +++ b/examples/simple_bench.rs @@ -115,7 +115,14 @@ fn load_dataset(dataset: &str, max_items: Option, verbose: bool) -> Resul Ok(samples) } -fn print_summary(label: &str, n: usize, total_chars: u64, total_tokens: u64, total_hf: Duration, total_ft: Duration) { +fn print_summary( + label: &str, + n: usize, + total_chars: u64, + total_tokens: u64, + total_hf: Duration, + total_ft: Duration, +) { let nf = n as f64; let hf_ms = total_hf.as_secs_f64() * 1000.0; let ft_ms = total_ft.as_secs_f64() * 1000.0; @@ -179,7 +186,9 @@ fn bench_sequential( .context("HF tokenizer encode failed")?; let enc_hf = enc_hf.get_ids(); let t1 = Instant::now(); - let enc = tokenizer.encode_with_special_tokens(chunk, true).context("fastokens encode failed")?; + let enc = tokenizer + .encode_with_special_tokens(chunk, true) + .context("fastokens encode failed")?; let t2 = Instant::now(); if enc_hf != enc { @@ -233,7 +242,14 @@ fn bench_sequential( } pb.finish(); - print_summary("Benchmark Summary", chunks.len(), total_chars, total_tokens, total_hf, total_ft); + print_summary( + "Benchmark Summary", + chunks.len(), + total_chars, + total_tokens, + total_hf, + total_ft, + ); Ok(()) } @@ -254,9 +270,11 @@ fn bench_batched( } else { let pb = ProgressBar::new(num_batches as u64); pb.set_style( - ProgressStyle::with_template("[{elapsed_precise}] [{bar:40}] {pos}/{len} batches ({eta})") - .expect("valid template") - .progress_chars("=> "), + ProgressStyle::with_template( + "[{elapsed_precise}] [{bar:40}] {pos}/{len} batches ({eta})", + ) + .expect("valid template") + .progress_chars("=> "), ); pb }; @@ -395,10 +413,23 @@ fn main() -> Result<()> { } if let Some(batch_size) = args.batch_size { - bench_batched(&chunks, &hf_tokenizer, &tokenizer, batch_size, args.verbose, csv_writer.as_mut())?; + bench_batched( + &chunks, + &hf_tokenizer, + &tokenizer, + batch_size, + args.verbose, + csv_writer.as_mut(), + )?; } else { - bench_sequential(&chunks, &hf_tokenizer, &tokenizer, args.verbose, csv_writer.as_mut())?; + bench_sequential( + &chunks, + &hf_tokenizer, + &tokenizer, + args.verbose, + csv_writer.as_mut(), + )?; } Ok(()) -} \ No newline at end of file +} diff --git a/src/added_tokens.rs b/src/added_tokens.rs index 7ad525c..b69edcd 100644 --- a/src/added_tokens.rs +++ b/src/added_tokens.rs @@ -128,11 +128,7 @@ impl AddedTokens { ), 2 => self.split_prefilter( input, - memchr::memchr2_iter( - self.start_bytes[0], - self.start_bytes[1], - input.as_bytes(), - ), + memchr::memchr2_iter(self.start_bytes[0], self.start_bytes[1], input.as_bytes()), ), 3 => self.split_prefilter( input, diff --git a/src/lib.rs b/src/lib.rs index bf8bf00..b195329 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -383,7 +383,6 @@ impl Tokenizer { PreTokenizedString::new(buffer, splits) } - } #[cfg(test)] @@ -435,7 +434,10 @@ mod tests { for (input, batch_result) in inputs.iter().zip(&batch_results) { let sequential_result = ours.encode(input).unwrap(); - assert_eq!(batch_result, &sequential_result, "batch mismatch for {input:?}"); + assert_eq!( + batch_result, &sequential_result, + "batch mismatch for {input:?}" + ); } } @@ -448,7 +450,9 @@ mod tests { assert!(ours.vocab_size() > 0); let token_str = ours.id_to_token(0).expect("token 0 should exist"); - let id = ours.token_to_id(token_str).expect("reverse lookup should work"); + let id = ours + .token_to_id(token_str) + .expect("reverse lookup should work"); assert_eq!(id, 0); } @@ -469,8 +473,8 @@ mod tests { "Z", "0", "!", - "\u{00e9}", // é (precomposed) - "\u{4e2d}", // 中 + "\u{00e9}", // é (precomposed) + "\u{4e2d}", // 中 // ── basic text ── "Hello, world!", "The quick brown fox jumps over the lazy dog.", @@ -505,9 +509,9 @@ mod tests { "\u{00fc}ber stra\u{00df}e gr\u{00f6}\u{00df}e", "se\u{00f1}or ni\u{00f1}o a\u{00f1}o", // ── Unicode: CJK ── - "\u{4f60}\u{597d}\u{4e16}\u{754c}", // 你好世界 - "\u{3053}\u{3093}\u{306b}\u{3061}\u{306f}", // こんにちは - "\u{c548}\u{b155}\u{d558}\u{c138}\u{c694}", // 안녕하세요 + "\u{4f60}\u{597d}\u{4e16}\u{754c}", // 你好世界 + "\u{3053}\u{3093}\u{306b}\u{3061}\u{306f}", // こんにちは + "\u{c548}\u{b155}\u{d558}\u{c138}\u{c694}", // 안녕하세요 // ── Unicode: Cyrillic ── "\u{041f}\u{0440}\u{0438}\u{0432}\u{0435}\u{0442} \u{043c}\u{0438}\u{0440}", // ── Unicode: Arabic ── @@ -517,11 +521,11 @@ mod tests { // ── Unicode: Emoji ── "\u{1f600}\u{1f680}\u{2764}\u{fe0f}", "\u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467}\u{200d}\u{1f466}", - "\u{1f1fa}\u{1f1f8}", // 🇺🇸 + "\u{1f1fa}\u{1f1f8}", // 🇺🇸 // ── Unicode: combining marks (NFD forms) ── - "e\u{0301}", // e + combining acute - "n\u{0303}", // n + combining tilde - "a\u{0308}", // a + combining diaeresis + "e\u{0301}", // e + combining acute + "n\u{0303}", // n + combining tilde + "a\u{0308}", // a + combining diaeresis // ── mixed scripts ── "Hello \u{4e16}\u{754c} \u{041c}\u{0438}\u{0440}!", "User123 wrote: \u{4f60}\u{597d}!", @@ -557,28 +561,28 @@ mod tests { // ── boundary / edge cases ── "a\nb\nc\n", "# Heading\n\n- item 1\n- item 2\n\n```code```", - "\u{ffff}", // max BMP non-character - "\u{0080}", // first non-ASCII - "\u{07ff}", // max 2-byte UTF-8 - "\u{0800}", // first 3-byte UTF-8 - "\u{10000}", // first surrogate-pair range + "\u{ffff}", // max BMP non-character + "\u{0080}", // first non-ASCII + "\u{07ff}", // max 2-byte UTF-8 + "\u{0800}", // first 3-byte UTF-8 + "\u{10000}", // first surrogate-pair range // ── unusual / invalid-ish Unicode ── - "\u{fffd}", // replacement character - "\u{feff}Hello", // BOM prefix - "\u{0000}", // null - "abc\u{0000}def", // embedded null - "\u{fffe}", // non-character - "\u{fdd0}", // non-character (FDD0 block) - "\u{200b}\u{200c}\u{200d}", // zero-width space / ZWNJ / ZWJ - "\u{202e}Hello\u{202c}", // RTL override + pop directional - "\u{0001}\u{0002}\u{001f}\u{007f}", // C0 controls + DEL - "\u{0300}", // lone combining grave (no base) - "a\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}", // 5 combining marks on one base - "\u{e000}\u{f8ff}", // private use area - "\u{01c5}\u{01c8}\u{01cb}", // titlecase letters (Dž Lj Nj) - "\u{2028}\u{2029}", // line / paragraph separators - "\u{fff9}\u{fffa}\u{fffb}", // interlinear annotation - "\u{d7ff}\u{10ffff}", // last before surrogates + max codepoint + "\u{fffd}", // replacement character + "\u{feff}Hello", // BOM prefix + "\u{0000}", // null + "abc\u{0000}def", // embedded null + "\u{fffe}", // non-character + "\u{fdd0}", // non-character (FDD0 block) + "\u{200b}\u{200c}\u{200d}", // zero-width space / ZWNJ / ZWJ + "\u{202e}Hello\u{202c}", // RTL override + pop directional + "\u{0001}\u{0002}\u{001f}\u{007f}", // C0 controls + DEL + "\u{0300}", // lone combining grave (no base) + "a\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}", // 5 combining marks on one base + "\u{e000}\u{f8ff}", // private use area + "\u{01c5}\u{01c8}\u{01cb}", // titlecase letters (Dž Lj Nj) + "\u{2028}\u{2029}", // line / paragraph separators + "\u{fff9}\u{fffa}\u{fffb}", // interlinear annotation + "\u{d7ff}\u{10ffff}", // last before surrogates + max codepoint // ── potential BPE merge edge cases ── "ab", "abc", @@ -596,10 +600,7 @@ mod tests { /// Helper: compare both encoding and decoding of every input in `corpus` /// between our tokenizer and the HuggingFace tokenizer for a given model. /// Returns a list of failure descriptions (empty = all passed). - fn compare_encode_decode( - model_name: &str, - corpus: &[&str], - ) -> Vec { + fn compare_encode_decode(model_name: &str, corpus: &[&str]) -> Vec { let hf = tokenizers::Tokenizer::from_pretrained(model_name, None) .unwrap_or_else(|e| panic!("{model_name}: HF load failed: {e}")); let ours = Tokenizer::from_model(model_name) @@ -614,9 +615,7 @@ mod tests { let our_ids = match ours.encode(input) { Ok(ids) => ids, Err(e) => { - failures.push(format!( - " encode error on {input:?}: {e}" - )); + failures.push(format!(" encode error on {input:?}: {e}")); continue; } }; @@ -643,9 +642,7 @@ mod tests { let our_decoded = match ours.decode(&hf_ids, false) { Ok(d) => d, Err(e) => { - failures.push(format!( - " decode error on {input:?}: {e}" - )); + failures.push(format!(" decode error on {input:?}: {e}")); continue; } }; @@ -673,7 +670,11 @@ mod tests { #[test] fn correctness_nemotron() { let f = compare_encode_decode("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", CORPUS); - assert!(f.is_empty(), "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16:\n{}", f.join("\n")); + assert!( + f.is_empty(), + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16:\n{}", + f.join("\n") + ); } #[test] @@ -697,13 +698,21 @@ mod tests { #[test] fn correctness_mistral_nemo() { let f = compare_encode_decode("mistralai/Mistral-Nemo-Instruct-2407", CORPUS); - assert!(f.is_empty(), "mistralai/Mistral-Nemo-Instruct-2407:\n{}", f.join("\n")); + assert!( + f.is_empty(), + "mistralai/Mistral-Nemo-Instruct-2407:\n{}", + f.join("\n") + ); } #[test] fn correctness_qwen3_nemotron() { let f = compare_encode_decode("nvidia/Qwen3-Nemotron-235B-A22B-GenRM", CORPUS); - assert!(f.is_empty(), "nvidia/Qwen3-Nemotron-235B-A22B-GenRM:\n{}", f.join("\n")); + assert!( + f.is_empty(), + "nvidia/Qwen3-Nemotron-235B-A22B-GenRM:\n{}", + f.join("\n") + ); } // ── Cache consistency ──────────────────────────────────────────── @@ -743,10 +752,7 @@ mod tests { let ours = Tokenizer::from_model(model).unwrap(); // Verify the fused path is active. - assert!( - ours.split_only.is_some(), - "expected fused path for {model}", - ); + assert!(ours.split_only.is_some(), "expected fused path for {model}",); // Run the same input many times to stress the fused cache. let input = "The year 2024 was notable for advances in AI. Models like \ @@ -775,7 +781,11 @@ mod tests { "code heremore code", ]; let f = compare_encode_decode("MiniMaxAI/MiniMax-M2.1", corpus); - assert!(f.is_empty(), "MiniMaxAI/MiniMax-M2.1 added tokens:\n{}", f.join("\n")); + assert!( + f.is_empty(), + "MiniMaxAI/MiniMax-M2.1 added tokens:\n{}", + f.join("\n") + ); } /// DeepSeek-V3.2 added tokens. @@ -789,7 +799,11 @@ mod tests { "<|tool▁calls▁begin|>call<|tool▁calls▁end|>", ]; let f = compare_encode_decode("deepseek-ai/DeepSeek-V3.2", corpus); - assert!(f.is_empty(), "deepseek-ai/DeepSeek-V3.2 added tokens:\n{}", f.join("\n")); + assert!( + f.is_empty(), + "deepseek-ai/DeepSeek-V3.2 added tokens:\n{}", + f.join("\n") + ); } /// Qwen3 added tokens. @@ -802,7 +816,11 @@ mod tests { "Plain text with no special tokens at all.", ]; let f = compare_encode_decode("Qwen/Qwen3-0.6B", corpus); - assert!(f.is_empty(), "Qwen/Qwen3-0.6B added tokens:\n{}", f.join("\n")); + assert!( + f.is_empty(), + "Qwen/Qwen3-0.6B added tokens:\n{}", + f.join("\n") + ); } /// Nemotron added tokens. @@ -816,7 +834,11 @@ mod tests { "No special tokens here.", ]; let f = compare_encode_decode("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", corpus); - assert!(f.is_empty(), "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 added tokens:\n{}", f.join("\n")); + assert!( + f.is_empty(), + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 added tokens:\n{}", + f.join("\n") + ); } // ── Long input stress test ─────────────────────────────────────── @@ -840,7 +862,8 @@ mod tests { let hf_ids = hf.encode(input.as_str(), false).unwrap().get_ids().to_vec(); let our_ids = ours.encode(&input).unwrap(); assert_eq!( - our_ids, hf_ids, + our_ids, + hf_ids, "long input mismatch: {} vs {} tokens", our_ids.len(), hf_ids.len(), @@ -863,7 +886,8 @@ mod tests { let hf_ids = hf.encode(input.as_str(), false).unwrap().get_ids().to_vec(); let our_ids = ours.encode(&input).unwrap(); assert_eq!( - our_ids, hf_ids, + our_ids, + hf_ids, "long input mismatch: {} vs {} tokens", our_ids.len(), hf_ids.len(), @@ -893,7 +917,11 @@ mod tests { .iter() .filter_map(|item| { let ctx = item.get("context")?.as_str()?; - if ctx.is_empty() { None } else { Some(ctx.to_string()) } + if ctx.is_empty() { + None + } else { + Some(ctx.to_string()) + } }) .collect(); @@ -909,17 +937,29 @@ mod tests { let parts: Vec = messages .iter() .filter_map(|msg| { - let role = msg.get("from").and_then(|v| v.as_str()).unwrap_or("unknown"); + let role = msg + .get("from") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); let value = msg.get("value").and_then(|v| v.as_str())?; - if value.is_empty() { return None; } + if value.is_empty() { + return None; + } Some(format!("[{role}]: {value}")) }) .collect(); - if parts.is_empty() { None } else { Some(parts.join("\n\n")) } + if parts.is_empty() { + None + } else { + Some(parts.join("\n\n")) + } }) .collect(); - ExtendedCorpus { longbench, sharegpt } + ExtendedCorpus { + longbench, + sharegpt, + } }) } @@ -991,9 +1031,7 @@ mod tests { let our_decoded = match ours.decode(hf_ids, false) { Ok(d) => d, Err(e) => { - failures.push(format!( - " decode error on {input_preview:?}: {e}" - )); + failures.push(format!(" decode error on {input_preview:?}: {e}")); continue; } }; @@ -1025,13 +1063,25 @@ mod tests { let progress = std::env::var("EXTENDED_PROGRESS").is_ok(); let corpus = extended_corpus(); if progress { - eprintln!(" {model_name}: longbench ({} samples)", corpus.longbench.len()); + eprintln!( + " {model_name}: longbench ({} samples)", + corpus.longbench.len() + ); } - let mut failures = compare_encode_decode_batched(model_name, &corpus.longbench, 10, progress); + let mut failures = + compare_encode_decode_batched(model_name, &corpus.longbench, 10, progress); if progress { - eprintln!(" {model_name}: sharegpt ({} samples)", corpus.sharegpt.len()); + eprintln!( + " {model_name}: sharegpt ({} samples)", + corpus.sharegpt.len() + ); } - failures.extend(compare_encode_decode_batched(model_name, &corpus.sharegpt, 10, progress)); + failures.extend(compare_encode_decode_batched( + model_name, + &corpus.sharegpt, + 10, + progress, + )); assert!( failures.is_empty(), "{model_name} extended ({} failures):\n{}", @@ -1093,5 +1143,4 @@ mod tests { fn extended_qwen_small() { run_extended("Qwen/Qwen3-0.6B"); } - } diff --git a/src/models/bpe.rs b/src/models/bpe.rs index 902c143..36919dd 100644 --- a/src/models/bpe.rs +++ b/src/models/bpe.rs @@ -4,8 +4,8 @@ use std::{ collections::{BinaryHeap, HashMap}, fmt, sync::{ - atomic::{AtomicUsize, Ordering}, Mutex, + atomic::{AtomicUsize, Ordering}, }, }; @@ -254,8 +254,12 @@ impl FlatCache { let slot = unsafe { self.slots.get_unchecked(idx) }; let h = slot.hash; if h == EMPTY_SLOT { - let Ok(len) = u16::try_from(ids.len()) else { return }; - let Ok(key_len) = u16::try_from(key_bytes.len()) else { return }; + let Ok(len) = u16::try_from(ids.len()) else { + return; + }; + let Ok(key_len) = u16::try_from(key_bytes.len()) else { + return; + }; self.count += 1; let offset = self.pool.len() as u32; self.pool.extend_from_slice(ids); @@ -273,7 +277,9 @@ impl FlatCache { let ks = slot.key_offset as usize; let ke = ks + slot.key_len as usize; if unsafe { self.key_pool.get_unchecked(ks..ke) } == key_bytes { - let Ok(len) = u16::try_from(ids.len()) else { return }; + let Ok(len) = u16::try_from(ids.len()) else { + return; + }; let offset = self.pool.len() as u32; self.pool.extend_from_slice(ids); let slot = unsafe { self.slots.get_unchecked_mut(idx) }; @@ -312,9 +318,7 @@ impl SharedCache { let bytes = key.as_bytes(); let mut h: u64 = bytes.len() as u64; for &b in &bytes[..bytes.len().min(8)] { - h = h - .wrapping_add(b as u64) - .wrapping_mul(0x9E3779B97F4A7C15); + h = h.wrapping_add(b as u64).wrapping_mul(0x9E3779B97F4A7C15); } h as usize & (CACHE_SHARDS - 1) } @@ -466,14 +470,25 @@ impl RankedMergeMap { } let capacity = (parsed.len() * 2).next_power_of_two(); let mask = capacity - 1; - let mut slots = vec![RankedMergeSlot { key: EMPTY_KEY, rank: 0, id: 0 }; capacity]; + let mut slots = vec![ + RankedMergeSlot { + key: EMPTY_KEY, + rank: 0, + id: 0 + }; + capacity + ]; for (&(t1, t2), &(rank, merged_id)) in parsed { let key = pack_pair(t1, t2); let mut idx = fx_hash(key) as usize & mask; loop { if slots[idx].key == EMPTY_KEY { - slots[idx] = RankedMergeSlot { key, rank, id: merged_id }; + slots[idx] = RankedMergeSlot { + key, + rank, + id: merged_id, + }; break; } idx = (idx + 1) & mask; @@ -718,9 +733,8 @@ impl Bpe { let token_lens: Vec = (0..=max_token) .map(|t| { - u16::try_from(vocab_r[&t].len()).map_err(|_| { - format!("token {t} length {} exceeds u16::MAX", vocab_r[&t].len()) - }) + u16::try_from(vocab_r[&t].len()) + .map_err(|_| format!("token {t} length {} exceeds u16::MAX", vocab_r[&t].len())) }) .collect::, _>>()?; @@ -962,7 +976,8 @@ impl Bpe { }); // Check pair with previous byte via pre-computed table. if i > 0 { - let (rank, _new_id) = self.byte_pair_initial[prev_byte as usize * 256 + byte as usize]; + let (rank, _new_id) = + self.byte_pair_initial[prev_byte as usize * 256 + byte as usize]; if rank != u32::MAX { scratch.heap_buf.push(Reverse(MergeEntry::new( rank, @@ -998,9 +1013,9 @@ impl Bpe { scratch.heap.extend((0..n - 1).filter_map(|i| { let left = symbols[i].c; let right = symbols[i + 1].c; - self.merge_adj.get(left, right).map(|(rank, _new_id)| { - Reverse(MergeEntry::new(rank, i as u32, left, right)) - }) + self.merge_adj + .get(left, right) + .map(|(rank, _new_id)| Reverse(MergeEntry::new(rank, i as u32, left, right))) })); } @@ -1047,7 +1062,12 @@ impl Bpe { if sym.prev >= 0 { let prev_c = symbols[sym.prev as usize].c; if let Some((rank, _)) = self.merge_adj.get(prev_c, new_id) { - heap.push(Reverse(MergeEntry::new(rank, sym.prev as u32, prev_c, new_id))); + heap.push(Reverse(MergeEntry::new( + rank, + sym.prev as u32, + prev_c, + new_id, + ))); } } let new_next = symbols[pos].next; diff --git a/src/post_processors.rs b/src/post_processors.rs index 7a1fb7d..238e111 100644 --- a/src/post_processors.rs +++ b/src/post_processors.rs @@ -139,9 +139,11 @@ impl PostProcessor { single, pair, special_tokens, - } => Ok(Self::TemplateProcessing( - TemplateProcessing::from_config(single, pair, special_tokens)?, - )), + } => Ok(Self::TemplateProcessing(TemplateProcessing::from_config( + single, + pair, + special_tokens, + )?)), PostProcessorConfig::Sequence { processors } => { let steps = processors .into_iter() @@ -164,11 +166,7 @@ impl PostProcessor { /// /// Only has an effect when `add_special_tokens` is true and the processor /// adds special tokens (e.g. `TemplateProcessing`). - pub fn post_process_single( - &self, - encoded: Vec, - add_special_tokens: bool, - ) -> Vec { + pub fn post_process_single(&self, encoded: Vec, add_special_tokens: bool) -> Vec { if !add_special_tokens { return encoded; } @@ -282,10 +280,7 @@ mod tests { }); // With special tokens - assert_eq!( - pp.post_process_single(vec![10, 20], true), - vec![1, 10, 20] - ); + assert_eq!(pp.post_process_single(vec![10, 20], true), vec![1, 10, 20]); // Without special tokens assert_eq!(pp.post_process_single(vec![10, 20], false), vec![10, 20]); } diff --git a/src/pre_tokenized.rs b/src/pre_tokenized.rs index 5cf272a..45422d6 100644 --- a/src/pre_tokenized.rs +++ b/src/pre_tokenized.rs @@ -119,7 +119,8 @@ impl PreTokenizedString { } let pool = bpe_pool(); - let chunk_size = (self.splits.len() + pool.current_num_threads() - 1) / pool.current_num_threads(); + let chunk_size = + (self.splits.len() + pool.current_num_threads() - 1) / pool.current_num_threads(); pool.install(|| { let chunk_results: Result>, String> = self @@ -163,7 +164,8 @@ impl PreTokenizedString { } let pool = bpe_pool(); - let chunk_size = (self.splits.len() + pool.current_num_threads() - 1) / pool.current_num_threads(); + let chunk_size = + (self.splits.len() + pool.current_num_threads() - 1) / pool.current_num_threads(); pool.install(|| { let chunk_results: Result>, String> = self @@ -344,9 +346,7 @@ mod tests { #[test] fn tokenize_propagates_error() { let pts = PreTokenizedString::from_text("x"); - let err = pts - .tokenize(|_, _out| Err("boom".to_string())) - .unwrap_err(); + let err = pts.tokenize(|_, _out| Err("boom".to_string())).unwrap_err(); assert_eq!(err, "boom"); } } diff --git a/src/pre_tokenizers/split.rs b/src/pre_tokenizers/split.rs index 3fb731c..7327533 100644 --- a/src/pre_tokenizers/split.rs +++ b/src/pre_tokenizers/split.rs @@ -1,4 +1,6 @@ +#[cfg(feature = "pcre2")] use std::cell::RefCell; +#[cfg(feature = "pcre2")] use std::sync::atomic::{AtomicUsize, Ordering}; use fancy_regex::Regex; @@ -11,10 +13,12 @@ use crate::pre_tokenized::{PreTokenizedString, Split as PtSplit}; use super::Error; // Thread-local cache of previous Split results for incremental re-use. +#[cfg(feature = "pcre2")] thread_local! { static SPLIT_CACHE: RefCell = RefCell::new(SplitCache::default()); } +#[cfg(feature = "pcre2")] struct SplitCache { /// Identity of the Split pre-tokenizer that populated this cache. /// Prevents cross-model contamination when different models share a thread. @@ -23,6 +27,7 @@ struct SplitCache { prev_matches: Vec<(usize, usize)>, } +#[cfg(feature = "pcre2")] impl Default for SplitCache { fn default() -> Self { Self { @@ -34,22 +39,28 @@ impl Default for SplitCache { } /// Minimum shared prefix length (bytes) before incremental re-use kicks in. +#[cfg(feature = "pcre2")] const INCREMENTAL_MIN_PREFIX: usize = 4096; /// Wrapper around a JIT-compiled PCRE2 regex for the Llama-3 pattern. +#[cfg(feature = "pcre2")] struct Pcre2Regex(pcre2::bytes::Regex); // Safety: PCRE2 JIT-compiled regexes are thread-safe for matching. // Each thread uses independent match data internally via pcre2 crate. +#[cfg(feature = "pcre2")] unsafe impl Send for Pcre2Regex {} +#[cfg(feature = "pcre2")] unsafe impl Sync for Pcre2Regex {} +#[cfg(feature = "pcre2")] impl std::fmt::Debug for Pcre2Regex { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("Pcre2Regex(...)") } } +#[cfg(feature = "pcre2")] impl Clone for Pcre2Regex { fn clone(&self) -> Self { // Re-compile for independent match state. @@ -137,6 +148,7 @@ struct SplitRaw { } /// Monotonic counter for unique Split instance IDs. +#[cfg(feature = "pcre2")] static SPLIT_ID_COUNTER: AtomicUsize = AtomicUsize::new(1); /// A compiled Split pre-tokenizer. @@ -151,6 +163,7 @@ static SPLIT_ID_COUNTER: AtomicUsize = AtomicUsize::new(1); pub struct Split { /// Unique identity for cache invalidation across different Split instances. #[serde(skip)] + #[cfg(feature = "pcre2")] id: usize, /// Pre-compiled regex copies for parallel matching. Index 0 is the /// "primary" used for sequential matching; the rest are independent @@ -160,12 +173,14 @@ pub struct Split { invert: bool, /// PCRE2 JIT-compiled regex copies for parallel matching (one per thread). /// Compiled opportunistically for all patterns; `None` only if PCRE2 - /// cannot handle the pattern syntax. + /// cannot handle the pattern syntax or the feature is disabled. + #[cfg(feature = "pcre2")] pcre2_regexes: Option>, } /// Compile PCRE2 JIT regexes from `source`, returning `None` if PCRE2 cannot /// handle the pattern (e.g. unsupported syntax). +#[cfg(feature = "pcre2")] fn try_compile_pcre2_regexes(source: &str, n: usize) -> Option> { let mut regexes = Vec::with_capacity(n); for _ in 0..n { @@ -180,7 +195,6 @@ fn try_compile_pcre2_regexes(source: &str, n: usize) -> Option> Some(regexes) } - /// Compile `n` independent copies of a regex from `source`. fn compile_regexes(source: &str, n: usize) -> Result, Error> { let mut regexes = Vec::with_capacity(n); @@ -195,14 +209,15 @@ impl TryFrom for Split { fn try_from(raw: SplitRaw) -> Result { let source = raw.pattern.source(); - let pcre2_regexes = try_compile_pcre2_regexes(&source, max_parallel()); let regexes = compile_regexes(&source, max_parallel())?; Ok(Self { + #[cfg(feature = "pcre2")] id: SPLIT_ID_COUNTER.fetch_add(1, Ordering::Relaxed), regexes, behavior: raw.behavior, invert: raw.invert, - pcre2_regexes, + #[cfg(feature = "pcre2")] + pcre2_regexes: try_compile_pcre2_regexes(&source, max_parallel()), }) } } @@ -217,13 +232,14 @@ impl Split { let source = pattern.source(); let regexes = compile_regexes(&source, max_parallel())?; let behavior: SplitBehavior = serde_json::from_value(Value::String(behavior.to_string()))?; - let pcre2_regexes = try_compile_pcre2_regexes(&source, max_parallel()); Ok(Self { + #[cfg(feature = "pcre2")] id: SPLIT_ID_COUNTER.fetch_add(1, Ordering::Relaxed), regexes, behavior, invert, - pcre2_regexes, + #[cfg(feature = "pcre2")] + pcre2_regexes: try_compile_pcre2_regexes(&source, max_parallel()), }) } @@ -240,6 +256,7 @@ impl Split { // step). Multi-split inputs (e.g. from an earlier Sequence step) // go through the generic per-split path which still uses PCRE2 via // find_segments. + #[cfg(feature = "pcre2")] if self.pcre2_regexes.is_some() && self.behavior == SplitBehavior::Isolated && !self.invert @@ -284,6 +301,7 @@ impl Split { /// Uses PCRE2 JIT-compiled regex with parallel matching and incremental /// caching. Since behavior is Isolated, every match and every gap between /// matches becomes its own split. + #[cfg(feature = "pcre2")] fn pre_tokenize_pcre2_isolated(&self, pts: &mut PreTokenizedString) -> Result<(), Error> { let buffer = pts.buffer(); let bytes = buffer.as_bytes(); @@ -328,12 +346,10 @@ impl Split { // Run PCRE2 on the portion after the reusable prefix. let suffix = &text[restart_pos..]; if suffix.len() >= MIN_CHUNK_SIZE * 2 { - let suffix_matches = - self.find_matches_pcre2_parallel(suffix, base + restart_pos)?; + let suffix_matches = self.find_matches_pcre2_parallel(suffix, base + restart_pos)?; matches.extend(suffix_matches); } else if !suffix.is_empty() { - let suffix_matches = - find_matches_pcre2(suffix, base + restart_pos, &pcre2[0])?; + let suffix_matches = find_matches_pcre2(suffix, base + restart_pos, &pcre2[0])?; matches.extend(suffix_matches); } @@ -342,13 +358,22 @@ impl Split { let mut prev = base; for &(s, e) in &matches { if s > prev { - new_splits.push(PtSplit { range: prev..s, token_id: None }); + new_splits.push(PtSplit { + range: prev..s, + token_id: None, + }); } - new_splits.push(PtSplit { range: s..e, token_id: None }); + new_splits.push(PtSplit { + range: s..e, + token_id: None, + }); prev = e; } if prev < base + text.len() { - new_splits.push(PtSplit { range: prev..(base + text.len()), token_id: None }); + new_splits.push(PtSplit { + range: prev..(base + text.len()), + token_id: None, + }); } // Update the cache for next call: move matches (no clone). @@ -375,6 +400,7 @@ impl Split { /// starting near a boundary are still found in full. Only matches whose /// start falls within a thread's authority zone are kept; the rest are /// discarded (the adjacent chunk owns them). + #[cfg(feature = "pcre2")] fn find_matches_pcre2_parallel( &self, text: &str, @@ -414,10 +440,7 @@ impl Split { let auth_end = auth[i + 1]; // Extend past authority zone so cross-boundary matches are // found in full. - let chunk_end = snap_char_ceil( - text, - (auth_end + CHUNK_OVERLAP).min(text.len()), - ); + let chunk_end = snap_char_ceil(text, (auth_end + CHUNK_OVERLAP).min(text.len())); let chunk = &text[auth_start..chunk_end]; let all = find_matches_pcre2(chunk, base + auth_start, &pcre2[i])?; // Keep only matches whose start is in our authority zone. @@ -473,6 +496,7 @@ impl Split { /// `(start, end, is_match)` segments. fn find_segments(&self, input: &str) -> Result, Error> { // Prefer PCRE2 JIT when available. + #[cfg(feature = "pcre2")] if let Some(pcre2) = &self.pcre2_regexes { let matches = if input.len() >= MIN_CHUNK_SIZE * 2 && pcre2.len() >= 2 { self.find_matches_pcre2_parallel(input, 0)? @@ -558,10 +582,7 @@ impl Split { .map(|i| { let auth_start = auth[i]; let auth_end = auth[i + 1]; - let chunk_end = snap_char_ceil( - input, - (auth_end + CHUNK_OVERLAP).min(input.len()), - ); + let chunk_end = snap_char_ceil(input, (auth_end + CHUNK_OVERLAP).min(input.len())); let regex = ®exes[i]; let chunk = &input[auth_start..chunk_end]; let mut matches = Vec::new(); @@ -755,8 +776,7 @@ fn merge_chunk_matches( Some((ms, me)) => { // Check convergence with remaining parallel matches. let limit = remaining.len().min(64); - if let Some(j) = - remaining[..limit].iter().position(|&m| m == (ms, me)) + if let Some(j) = remaining[..limit].iter().position(|&m| m == (ms, me)) { // Converged — resume from this point in flat. idx += j; @@ -806,6 +826,7 @@ fn matches_to_segments( /// Find the length of the common prefix between two byte slices. /// /// Compares 8 bytes at a time for speed on large inputs. +#[cfg(feature = "pcre2")] fn common_prefix_len(a: &[u8], b: &[u8]) -> usize { let min_len = a.len().min(b.len()); let chunks = min_len / 8; @@ -828,6 +849,7 @@ fn common_prefix_len(a: &[u8], b: &[u8]) -> usize { } /// Find all pattern matches using PCRE2 JIT. +#[cfg(feature = "pcre2")] fn find_matches_pcre2( input: &str, base: usize, @@ -1059,12 +1081,7 @@ mod tests { #[test] fn llama3_full_coverage() { - let s = Split::from_config( - &json!({"Regex": LLAMA3_PATTERN}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": LLAMA3_PATTERN}), "Isolated", false).unwrap(); let inputs = [ "Hello, world!", "fn main() { println!(\"hello\"); }", @@ -1090,12 +1107,7 @@ mod tests { #[test] fn llama3_digits_are_individual() { - let s = Split::from_config( - &json!({"Regex": LLAMA3_PATTERN}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": LLAMA3_PATTERN}), "Isolated", false).unwrap(); // \p{N} matches one digit at a time, not runs. let result = s.split("12345").unwrap(); assert_eq!(result, vec!["1", "2", "3", "4", "5"]); @@ -1103,12 +1115,7 @@ mod tests { #[test] fn gpt2_contractions() { - let s = Split::from_config( - &json!({"Regex": GPT2_PATTERN}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": GPT2_PATTERN}), "Isolated", false).unwrap(); let input = "I'm don't we're they've I'll he'd"; let result = s.split(input).unwrap(); assert!(result.contains(&"'m")); @@ -1122,12 +1129,7 @@ mod tests { #[test] fn gpt2_full_coverage() { - let s = Split::from_config( - &json!({"Regex": GPT2_PATTERN}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": GPT2_PATTERN}), "Isolated", false).unwrap(); let inputs = [ "Hello, world!", " multiple spaces ", @@ -1145,48 +1147,28 @@ mod tests { #[test] fn digits_only_leaves_gaps() { - let s = Split::from_config( - &json!({"Regex": "\\d+"}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "\\d+"}), "Isolated", false).unwrap(); let result = s.split("abc123def456ghi").unwrap(); assert_eq!(result, vec!["abc", "123", "def", "456", "ghi"]); } #[test] fn letters_only_with_gaps() { - let s = Split::from_config( - &json!({"Regex": "\\p{L}+"}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "\\p{L}+"}), "Isolated", false).unwrap(); let result = s.split("hello---world...test").unwrap(); assert_eq!(result, vec!["hello", "---", "world", "...", "test"]); } #[test] fn alternation_with_gaps() { - let s = Split::from_config( - &json!({"Regex": "\\p{L}+|\\d+"}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "\\p{L}+|\\d+"}), "Isolated", false).unwrap(); let result = s.split("abc@123#def!456").unwrap(); assert_eq!(result, vec!["abc", "@", "123", "#", "def", "!", "456"]); } #[test] fn gaps_with_removed_behavior() { - let s = Split::from_config( - &json!({"Regex": "\\d+"}), - "Removed", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "\\d+"}), "Removed", false).unwrap(); // Matches (digits) are removed; only gaps remain. let result = s.split("abc123def456ghi").unwrap(); assert_eq!(result, vec!["abc", "def", "ghi"]); @@ -1194,24 +1176,14 @@ mod tests { #[test] fn gaps_merged_with_next() { - let s = Split::from_config( - &json!({"Regex": "\\s+"}), - "MergedWithNext", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "\\s+"}), "MergedWithNext", false).unwrap(); let result = s.split("hello world test").unwrap(); assert_eq!(result, vec!["hello", " world", " test"]); } #[test] fn contiguous_adjacent_single_matches() { - let s = Split::from_config( - &json!({"Regex": "[a-c]"}), - "Contiguous", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "[a-c]"}), "Contiguous", false).unwrap(); // Each letter is a separate match; Contiguous merges adjacent matches. let result = s.split("abcxabc").unwrap(); assert_eq!(result, vec!["abc", "x", "abc"]); @@ -1221,12 +1193,7 @@ mod tests { #[test] fn titlecase_letter_pattern() { - let s = Split::from_config( - &json!({"Regex": "\\p{Lt}"}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "\\p{Lt}"}), "Isolated", false).unwrap(); // U+01C5 (Dž), U+01C8 (Lj), U+01CB (Nj) are titlecase letters. let input = "a\u{01c5}b\u{01c8}c\u{01cb}d"; let result = s.split(input).unwrap(); @@ -1238,12 +1205,8 @@ mod tests { #[test] fn camelcase_pattern() { - let s = Split::from_config( - &json!({"Regex": "[\\p{Lu}][\\p{Ll}]*"}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "[\\p{Lu}][\\p{Ll}]*"}), "Isolated", false) + .unwrap(); let result = s.split("CamelCaseHTTPServer").unwrap(); assert_eq!(result.join(""), "CamelCaseHTTPServer"); assert!(result.contains(&"Camel")); @@ -1272,12 +1235,7 @@ mod tests { #[test] fn pattern_matches_nothing_in_input() { - let s = Split::from_config( - &json!({"Regex": "ZZZZZ"}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "ZZZZZ"}), "Isolated", false).unwrap(); // No matches → entire input is one gap segment. let result = s.split("hello world").unwrap(); assert_eq!(result, vec!["hello world"]); @@ -1285,12 +1243,7 @@ mod tests { #[test] fn pattern_single_char_matches() { - let s = Split::from_config( - &json!({"Regex": "."}), - "Isolated", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "."}), "Isolated", false).unwrap(); // Every char is a match, no gaps at all. let result = s.split("abc").unwrap(); assert_eq!(result, vec!["a", "b", "c"]); @@ -1300,12 +1253,7 @@ mod tests { #[test] fn invert_removes_non_matching() { - let s = Split::from_config( - &json!({"Regex": "\\p{L}+"}), - "Removed", - true, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "\\p{L}+"}), "Removed", true).unwrap(); // Invert flips match/gap, Removed drops the new "matches" (which // are the original gaps). Only letter runs survive. let result = s.split("hello 123 world 456").unwrap(); @@ -1314,12 +1262,7 @@ mod tests { #[test] fn invert_contiguous_vowels() { - let s = Split::from_config( - &json!({"Regex": "[aeiou]+"}), - "Contiguous", - true, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "[aeiou]+"}), "Contiguous", true).unwrap(); // Inverted: vowels become gaps, consonant runs become matches. // Contiguous merges adjacent same-type segments. let result = s.split("hello").unwrap(); @@ -1342,18 +1285,8 @@ mod tests { #[test] fn sequential_two_splits() { // First: remove whitespace. Then: isolate digits. - let s1 = Split::from_config( - &json!({"Regex": "\\s+"}), - "Removed", - false, - ) - .unwrap(); - let s2 = Split::from_config( - &json!({"Regex": "\\d+"}), - "Isolated", - false, - ) - .unwrap(); + let s1 = Split::from_config(&json!({"Regex": "\\s+"}), "Removed", false).unwrap(); + let s2 = Split::from_config(&json!({"Regex": "\\d+"}), "Isolated", false).unwrap(); let mut pts = PreTokenizedString::from_text("hello 123world 456test"); s1.pre_tokenize(&mut pts).unwrap(); @@ -1368,50 +1301,36 @@ mod tests { #[test] fn sequential_three_splits() { // Whitespace → digits → uppercase letters. - let s1 = Split::from_config( - &json!({"Regex": "\\s+"}), - "Removed", - false, - ) - .unwrap(); - let s2 = Split::from_config( - &json!({"Regex": "\\d+"}), - "Isolated", - false, - ) - .unwrap(); - let s3 = Split::from_config( - &json!({"Regex": "[\\p{Lu}]+"}), - "Isolated", - false, - ) - .unwrap(); + let s1 = Split::from_config(&json!({"Regex": "\\s+"}), "Removed", false).unwrap(); + let s2 = Split::from_config(&json!({"Regex": "\\d+"}), "Isolated", false).unwrap(); + let s3 = Split::from_config(&json!({"Regex": "[\\p{Lu}]+"}), "Isolated", false).unwrap(); let mut pts = PreTokenizedString::from_text("helloWORLD 123ABCdef"); s1.pre_tokenize(&mut pts).unwrap(); s2.pre_tokenize(&mut pts).unwrap(); s3.pre_tokenize(&mut pts).unwrap(); - assert_eq!( - pts_texts(&pts), - vec!["hello", "WORLD", "123", "ABC", "def"], - ); + assert_eq!(pts_texts(&pts), vec!["hello", "WORLD", "123", "ABC", "def"],); } #[test] fn sequential_preserves_added_tokens() { - let s = Split::from_config( - &json!({"Regex": "\\s+"}), - "Removed", - false, - ) - .unwrap(); + let s = Split::from_config(&json!({"Regex": "\\s+"}), "Removed", false).unwrap(); let buffer = "hello world".to_string(); let splits = vec![ - PtSplit { range: 0..5, token_id: None }, - PtSplit { range: 5..5, token_id: Some(42) }, - PtSplit { range: 5..11, token_id: None }, + PtSplit { + range: 0..5, + token_id: None, + }, + PtSplit { + range: 5..5, + token_id: Some(42), + }, + PtSplit { + range: 5..11, + token_id: None, + }, ]; let mut pts = PreTokenizedString::new(buffer, splits); s.pre_tokenize(&mut pts).unwrap(); @@ -1423,18 +1342,8 @@ mod tests { #[test] fn sequential_mixed_behaviors() { // First split: isolate punctuation. Second split: merge whitespace with next. - let s1 = Split::from_config( - &json!({"Regex": "[,.!?]+"}), - "Isolated", - false, - ) - .unwrap(); - let s2 = Split::from_config( - &json!({"Regex": "\\s+"}), - "MergedWithNext", - false, - ) - .unwrap(); + let s1 = Split::from_config(&json!({"Regex": "[,.!?]+"}), "Isolated", false).unwrap(); + let s2 = Split::from_config(&json!({"Regex": "\\s+"}), "MergedWithNext", false).unwrap(); let mut pts = PreTokenizedString::from_text("hello, world! test"); s1.pre_tokenize(&mut pts).unwrap(); @@ -1491,11 +1400,7 @@ mod tests { // A single match spans three chunks. // Chunk 0: truncated (0, 10). Chunk 1: ghost (8, 20). Chunk 2: ghost (18, 30). // True match: (0, 30). Then a normal match at (35, 40). - let chunks = vec![ - vec![(0, 10)], - vec![(8, 20)], - vec![(18, 30), (35, 40)], - ]; + let chunks = vec![vec![(0, 10)], vec![(8, 20)], vec![(18, 30), (35, 40)]]; let full = [(0, 30), (35, 40)]; let result = merge_with_oracle(chunks, &full); assert_eq!(result, vec![(0, 30), (35, 40)]); @@ -1519,6 +1424,49 @@ mod tests { assert_eq!(result, vec![(0, 10), (15, 20)]); } + // ── pcre2 feature gate ──────────────────────────────────────────── + + /// When the `pcre2` feature is on, the Split struct should have a + /// `pcre2_regexes` field that is `Some` for compilable patterns. + #[test] + #[cfg(feature = "pcre2")] + fn pcre2_regexes_populated_when_feature_on() { + let s = Split::from_config(&json!({"Regex": "\\p{L}+"}), "Isolated", false).unwrap(); + assert!( + s.pcre2_regexes.is_some(), + "pcre2_regexes should be Some when pcre2 feature is enabled", + ); + } + + /// When the `pcre2` feature is off, the fancy-regex fallback must + /// produce identical results to the pcre2 path. This test uses + /// a complex real-world pattern (Llama-3) on non-trivial Unicode + /// input to verify correctness of the fallback. + #[test] + #[cfg(not(feature = "pcre2"))] + fn fancy_regex_fallback_matches_expected_output() { + let s = Split::from_config(&json!({"Regex": LLAMA3_PATTERN}), "Isolated", false).unwrap(); + // Verify on a mix of scripts, emoji, whitespace, digits, and + // contractions -- the same inputs that the pcre2 path handles. + let inputs = [ + "Hello, world!", + "café résumé naïve", + "你好世界 こんにちは 안녕하세요", + "\u{1f680}\u{1f4a1}\u{2728} emoji text \u{1f389}", + "CamelCase snake_case kebab-case", + "I'm don't we're they've I'll he'd", + "100,000.50 + 3.14e-10 = ?", + ]; + for &input in &inputs { + let pieces = s.split(input).unwrap(); + assert_eq!( + pieces.join(""), + input, + "fancy-regex fallback lost bytes on {input:?}: pieces={pieces:?}", + ); + } + } + // ── integration: long match crossing parallel chunk boundary ───── #[test] @@ -1550,12 +1498,8 @@ mod tests { } assert!(input.len() >= 2 * chunk, "input must trigger parallel path"); - let split = Split::from_config( - &serde_json::json!({"Regex": "[a-z]+"}), - "Isolated", - false, - ) - .unwrap(); + let split = + Split::from_config(&serde_json::json!({"Regex": "[a-z]+"}), "Isolated", false).unwrap(); let pieces = split.split(&input).unwrap(); // There should be exactly 3 pieces: digits, long 'a' run, digits. @@ -1569,11 +1513,16 @@ mod tests { long_piece.len(), ); // Also verify via byte offsets - let a_pieces: Vec<&str> = pieces.iter().copied().filter(|p| p.starts_with('a')).collect(); - assert_eq!(a_pieces.len(), 1, "should be exactly one 'a' piece, got {a_pieces:?}"); + let a_pieces: Vec<&str> = pieces + .iter() + .copied() + .filter(|p| p.starts_with('a')) + .collect(); assert_eq!( - &input[match_start..match_end], - *long_piece, + a_pieces.len(), + 1, + "should be exactly one 'a' piece, got {a_pieces:?}" ); + assert_eq!(&input[match_start..match_end], *long_piece,); } } From e79e7bd2a4b2e1bc3f372aa29a268954c95f43c7 Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Sat, 14 Mar 2026 15:46:32 -0700 Subject: [PATCH 2/2] deps: switch hf-hub to rustls-tls and ureq, drop native-tls Replace the default native-tls/OpenSSL backend for hf-hub with the pure-Rust rustls-tls + ureq feature set, removing a large number of platform-specific and OpenSSL-related transitive dependencies. --- Cargo.lock | 620 ++++++++++++++--------------------------------------- Cargo.toml | 2 +- 2 files changed, 164 insertions(+), 458 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a94bc53..a95ad54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,9 +33,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -48,15 +48,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -83,9 +83,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.101" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "atomic-waker" @@ -128,9 +128,9 @@ checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "byteorder" @@ -155,9 +155,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.56" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "jobserver", @@ -171,11 +171,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "clap" -version = "4.5.58" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63be97961acde393029492ce0be7a1af7e323e6bae9511ebfac33751be5e6806" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" dependencies = [ "clap_builder", "clap_derive", @@ -183,9 +189,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.58" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f13174bda5dfd69d7e947827e5af4b0f2f94a4a3ee92912fba07a66150f21e2" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -195,9 +201,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.55" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" dependencies = [ "heck", "proc-macro2", @@ -207,15 +213,15 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "compact_str" @@ -247,43 +253,16 @@ dependencies = [ [[package]] name = "console" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e45a4a8926227e4197636ba97a9fc9b00477e9f4bd711395687c5f0734bec4" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" dependencies = [ "encode_unicode", "libc", - "once_cell", "unicode-width", "windows-sys 0.61.2", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "crc32fast" version = "1.5.0" @@ -464,31 +443,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "esaxx-rs" version = "0.1.10" @@ -531,12 +485,6 @@ dependencies = [ "tokenizers", ] -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -559,21 +507,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -583,21 +516,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "futures" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - [[package]] name = "futures-channel" version = "0.3.32" @@ -605,7 +523,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", - "futures-sink", ] [[package]] @@ -614,17 +531,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" -[[package]] -name = "futures-executor" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - [[package]] name = "futures-io" version = "0.3.32" @@ -660,7 +566,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ - "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -678,8 +583,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -689,48 +596,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasip2", + "wasm-bindgen", ] -[[package]] -name = "h2" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" - [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - [[package]] name = "hf-hub" version = "0.4.3" @@ -738,19 +616,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97" dependencies = [ "dirs", - "futures", "http", "indicatif 0.17.11", "libc", "log", - "native-tls", - "num_cpus", "rand", "reqwest", "serde", "serde_json", "thiserror", - "tokio", "ureq", "windows-sys 0.60.2", ] @@ -804,7 +678,6 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", "http", "http-body", "httparse", @@ -830,22 +703,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", -] - -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", + "webpki-roots 1.0.6", ] [[package]] @@ -866,11 +724,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", - "system-configuration", "tokio", "tower-service", "tracing", - "windows-registry", ] [[package]] @@ -984,16 +840,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "indexmap" -version = "2.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" -dependencies = [ - "equivalent", - "hashbrown", -] - [[package]] name = "indicatif" version = "0.17.11" @@ -1013,7 +859,7 @@ version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" dependencies = [ - "console 0.16.2", + "console 0.16.3", "portable-atomic", "unicode-width", "unit-prefix", @@ -1022,9 +868,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" @@ -1069,9 +915,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.85" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" dependencies = [ "once_cell", "wasm-bindgen", @@ -1079,26 +925,19 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.182" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libredox" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ - "bitflags", "libc", ] -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - [[package]] name = "litemap" version = "0.8.1" @@ -1111,6 +950,12 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "macro_rules_attribute" version = "0.2.2" @@ -1133,12 +978,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1188,23 +1027,6 @@ dependencies = [ "syn", ] -[[package]] -name = "native-tls" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nom" version = "7.1.3" @@ -1215,16 +1037,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "num_cpus" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "number_prefix" version = "0.4.0" @@ -1233,9 +1045,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -1265,50 +1077,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "openssl" -version = "0.10.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" - -[[package]] -name = "openssl-sys" -version = "0.9.112" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "option-ext" version = "0.2.0" @@ -1351,9 +1119,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -1400,11 +1168,66 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -1511,9 +1334,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "reqwest" @@ -1523,30 +1346,27 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", - "encoding_rs", "futures-core", "futures-util", - "h2", "http", "http-body", "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", - "mime", - "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-native-tls", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -1556,6 +1376,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots 1.0.6", ] [[package]] @@ -1573,23 +1394,16 @@ dependencies = [ ] [[package]] -name = "rustix" -version = "1.1.4" +name = "rustc-hash" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.52.0", -] +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "log", "once_cell", @@ -1606,6 +1420,7 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ + "web-time", "zeroize", ] @@ -1632,38 +1447,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "schannel" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "security-framework" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" -dependencies = [ - "bitflags", - "core-foundation 0.10.1", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "serde" version = "1.0.228" @@ -1745,12 +1528,12 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1823,9 +1606,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.115" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e614ed320ac28113fa64972c4262d5dbc89deacdfd00c34a3e4cea073243c12" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -1852,40 +1635,6 @@ dependencies = [ "syn", ] -[[package]] -name = "system-configuration" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.3.4", - "once_cell", - "rustix", - "windows-sys 0.52.0", -] - [[package]] name = "thiserror" version = "2.0.18" @@ -1916,6 +1665,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokenizers" version = "0.22.2" @@ -1953,40 +1717,18 @@ dependencies = [ [[package]] name = "tokio" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", "mio", "pin-project-lite", "socket2", - "tokio-macros", "windows-sys 0.61.2", ] -[[package]] -name = "tokio-macros" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" @@ -2082,9 +1824,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "unicode-ident" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537dd038a89878be9b64dd4bd1b260315c1bb94f4d784956b81e27a088d9a09e" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-normalization-alignments" @@ -2134,7 +1876,6 @@ dependencies = [ "base64 0.22.1", "flate2", "log", - "native-tls", "once_cell", "rustls", "rustls-pki-types", @@ -2175,12 +1916,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.5" @@ -2213,9 +1948,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ "cfg-if", "once_cell", @@ -2226,9 +1961,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.58" +version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ "cfg-if", "futures-util", @@ -2240,9 +1975,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2250,9 +1985,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" dependencies = [ "bumpalo", "proc-macro2", @@ -2263,9 +1998,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.108" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" dependencies = [ "unicode-ident", ] @@ -2285,9 +2020,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.85" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" dependencies = [ "js-sys", "wasm-bindgen", @@ -2349,35 +2084,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -2586,18 +2292,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.39" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.39" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 6e9e7eb..ea6e991 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ daachorse = "1.0.0" fancy-regex = "0.17.0" memchr = "2" pcre2 = { version = "0.2", optional = true } -hf-hub = "0.4.3" +hf-hub = { version = "0.4.3", default-features = false, features = ["rustls-tls", "ureq"] } rayon = "1.11.0" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149"