Itertools upgrade

tokenizers/Cargo.toml

@@ -47,14 +47,14 @@ onig = { version = "6.4", default-features = false, optional = true }
 regex = "1.10"
 regex-syntax = "0.8"
 rayon = "1.10"
-rayon-cond = "0.3"
+rayon-cond = "0.4"
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = "1.0"
 unicode-normalization-alignments = "0.1"
 unicode_categories = "0.1"
 unicode-segmentation = "1.11"
 indicatif = {version = "0.17", optional = true}
-itertools = "0.13"
+itertools = "0.14"
 log = "0.4"
 derive_builder = "0.20"
 spm_precompiled = "0.1.3"

tokenizers/src/decoders/byte_fallback.rs

@@ -28,11 +28,7 @@ impl Decoder for ByteFallback {
 
         for token in tokens {
             let bytes = if token.len() == 6 && token.starts_with("<0x") && token.ends_with('>') {
-                if let Ok(byte) = u8::from_str_radix(&token[3..5], 16) {
-                    Some(byte)
-                } else {
-                    None
-                }
+                u8::from_str_radix(&token[3..5], 16).ok()
             } else {
                 None
             };

tokenizers/src/normalizers/byte_level.rs

@@ -32,13 +32,10 @@ impl Normalizer for ByteLevel {
         if !normalized.is_empty() {
             let s = normalized.get();
             let mut transformations: Vec<(char, isize)> = Vec::with_capacity(s.len());
-            let mut i = 0;
-            for cur_char in s.chars() {
+            for (i, cur_char) in s.char_indices() {
                 let size = cur_char.len_utf8();
-                let bytes = s[i..i + size].as_bytes();
-                i += size;
                 transformations.extend(
-                    bytes
+                    s.as_bytes()[i..i + size]
                         .iter()
                         .enumerate()
                         .map(|(i, b)| (BYTES_CHAR[b], isize::from(i > 0))),

tokenizers/src/pre_tokenizers/byte_level.rs

@@ -132,13 +132,10 @@ impl PreTokenizer for ByteLevel {
         pretokenized.normalize(|normalized| {
             let s = normalized.get();
             let mut transformations: Vec<(char, isize)> = Vec::with_capacity(s.len());
-            let mut i = 0;
-            for cur_char in s.chars() {
+            for (i, cur_char) in s.char_indices() {
                 let size = cur_char.len_utf8();
-                let bytes = s[i..i + size].as_bytes();
-                i += size;
                 transformations.extend(
-                    bytes
+                    s.as_bytes()[i..i + size]
                         .iter()
                         .enumerate()
                         .map(|(i, b)| (BYTES_CHAR[b], isize::from(i > 0))),

tokenizers/src/processors/template.rs

@@ -565,16 +565,16 @@ impl TemplateProcessing {
 
                             let encoding = Encoding::new(
                                 tok.ids.clone(),
-                                std::iter::repeat(*type_id).take(len).collect(),
+                                std::iter::repeat_n(*type_id, len).collect(),
                                 tok.tokens.clone(),
                                 // words
-                                std::iter::repeat(None).take(len).collect(),
+                                std::iter::repeat_n(None, len).collect(),
                                 // offsets
-                                std::iter::repeat((0, 0)).take(len).collect(),
+                                std::iter::repeat_n((0, 0), len).collect(),
                                 // special_tokens_mask
-                                std::iter::repeat(1).take(len).collect(),
+                                std::iter::repeat_n(1, len).collect(),
                                 // attention_mask
-                                std::iter::repeat(1).take(len).collect(),
+                                std::iter::repeat_n(1, len).collect(),
                                 // overflowing
                                 vec![],
                                 // sequence_range

tokenizers/src/tokenizer/encoding.rs

@@ -139,7 +139,7 @@ impl Encoding {
         for seq_id in 0..self.n_sequences() {
             let range = self.sequence_range(seq_id);
             let seq_len = range.len();
-            sequences.splice(range, std::iter::repeat(Some(seq_id)).take(seq_len));
+            sequences.splice(range, std::iter::repeat_n(Some(seq_id), seq_len));
         }
         sequences
     }