From 608a6a6a411f86ebb38d61ed4fbb6792f4947f14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:35:21 +0000 Subject: [PATCH 1/2] chore(deps): update tokenizers requirement from 0.22.2 to 0.23.2 Updates the requirements on [tokenizers](https://github.com/huggingface/tokenizers) to permit the latest version. - [Release notes](https://github.com/huggingface/tokenizers/releases) - [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md) - [Commits](https://github.com/huggingface/tokenizers/compare/v0.22.2...v0.23.2) --- updated-dependencies: - dependency-name: tokenizers dependency-version: 0.23.2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index eea3992..207f8ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ ort = { version = "=2.0.0-rc.13", default-features = false, features = [ safetensors = { version = "0.8.0" } serde_json = { version = "1" } serde = { version = "1", features = ["derive"] } -tokenizers = { version = "0.22.2", default-features = false, features = ["onig"] } +tokenizers = { version = "0.23.2", default-features = false, features = ["onig"] } candle-nn = { version = "0.11.0", optional = true } candle-core = { version = "0.11.0", optional = true } From 433aacb15e0b91be883341e1ed00f52a1c9f3f64 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 7 Sep 2026 12:41:46 +0530 Subject: [PATCH 2/2] Fix CI failure with tokenizers 0.23.2 (#286) * Fix tokenizers 0.23 special token iteration Co-authored-by: Anush008 <46051506+Anush008@users.noreply.github.com> * Handle tokenizers special token errors Co-authored-by: Anush008 <46051506+Anush008@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Anush008 <46051506+Anush008@users.noreply.github.com> --- src/common.rs | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/common.rs b/src/common.rs index 284ef6b..2cad637 100644 --- a/src/common.rs +++ b/src/common.rs @@ -188,11 +188,13 @@ pub fn load_tokenizer(tokenizer_files: TokenizerFiles, max_length: usize) -> Res for (_, value) in root_object.iter() { if value.is_string() { if let Some(content) = value.as_str() { - tokenizer.add_special_tokens(&[AddedToken { - content: content.into(), - special: true, - ..Default::default() - }]); + tokenizer + .add_special_tokens([AddedToken { + content: content.into(), + special: true, + ..Default::default() + }]) + .map_err(|e| Error::TokenizerConfig(e.to_string()))?; } } else if value.is_object() { if let ( @@ -208,14 +210,16 @@ pub fn load_tokenizer(tokenizer_files: TokenizerFiles, max_length: usize) -> Res value["rstrip"].as_bool(), value["normalized"].as_bool(), ) { - tokenizer.add_special_tokens(&[AddedToken { - content: content.into(), - special: true, - single_word, - lstrip, - rstrip, - normalized, - }]); + tokenizer + .add_special_tokens([AddedToken { + content: content.into(), + special: true, + single_word, + lstrip, + rstrip, + normalized, + }]) + .map_err(|e| Error::TokenizerConfig(e.to_string()))?; } } }