From 116128178fdb398dced77b0c2ffd3b1285f79d62 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 7 Sep 2026 04:04:30 +0000
Subject: [PATCH 1/2] Fix tokenizers 0.23 special token iteration

Co-authored-by: Anush008 <46051506+Anush008@users.noreply.github.com>
---
 src/common.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/common.rs b/src/common.rs
index 284ef6b..d3a4104 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -188,7 +188,7 @@ pub fn load_tokenizer(tokenizer_files: TokenizerFiles, max_length: usize) -> Res
         for (_, value) in root_object.iter() {
             if value.is_string() {
                 if let Some(content) = value.as_str() {
-                    tokenizer.add_special_tokens(&[AddedToken {
+                    tokenizer.add_special_tokens([AddedToken {
                         content: content.into(),
                         special: true,
                         ..Default::default()
@@ -208,7 +208,7 @@ pub fn load_tokenizer(tokenizer_files: TokenizerFiles, max_length: usize) -> Res
                     value["rstrip"].as_bool(),
                     value["normalized"].as_bool(),
                 ) {
-                    tokenizer.add_special_tokens(&[AddedToken {
+                    tokenizer.add_special_tokens([AddedToken {
                         content: content.into(),
                         special: true,
                         single_word,

From feddf7c657e9c078c59ab9f5cdca646e04978dba Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 7 Sep 2026 04:04:57 +0000
Subject: [PATCH 2/2] Handle tokenizers special token errors

Co-authored-by: Anush008 <46051506+Anush008@users.noreply.github.com>
---
 src/common.rs | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/common.rs b/src/common.rs
index d3a4104..2cad637 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -188,11 +188,13 @@ pub fn load_tokenizer(tokenizer_files: TokenizerFiles, max_length: usize) -> Res
         for (_, value) in root_object.iter() {
             if value.is_string() {
                 if let Some(content) = value.as_str() {
-                    tokenizer.add_special_tokens([AddedToken {
-                        content: content.into(),
-                        special: true,
-                        ..Default::default()
-                    }]);
+                    tokenizer
+                        .add_special_tokens([AddedToken {
+                            content: content.into(),
+                            special: true,
+                            ..Default::default()
+                        }])
+                        .map_err(|e| Error::TokenizerConfig(e.to_string()))?;
                 }
             } else if value.is_object() {
                 if let (
@@ -208,14 +210,16 @@ pub fn load_tokenizer(tokenizer_files: TokenizerFiles, max_length: usize) -> Res
                     value["rstrip"].as_bool(),
                     value["normalized"].as_bool(),
                 ) {
-                    tokenizer.add_special_tokens([AddedToken {
-                        content: content.into(),
-                        special: true,
-                        single_word,
-                        lstrip,
-                        rstrip,
-                        normalized,
-                    }]);
+                    tokenizer
+                        .add_special_tokens([AddedToken {
+                            content: content.into(),
+                            special: true,
+                            single_word,
+                            lstrip,
+                            rstrip,
+                            normalized,
+                        }])
+                        .map_err(|e| Error::TokenizerConfig(e.to_string()))?;
                 }
             }
         }