Add Qwen2Tokenizer (#146)

huggingface · Dec 24, 2024 · bda7f90 · bda7f90
1 parent d656ad4
commit bda7f90
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/Sources/Tokenizers/Tokenizer.swift b/Sources/Tokenizers/Tokenizer.swift
@@ -72,6 +72,7 @@ struct TokenizerModel {
         "T5Tokenizer"        : T5Tokenizer.self,
         "WhisperTokenizer"   : WhisperTokenizer.self,
         "CohereTokenizer"    : CohereTokenizer.self,
+        "Qwen2Tokenizer"     : Qwen2Tokenizer.self,
         "PreTrainedTokenizer": BPETokenizer.self
     ]
 
@@ -488,6 +489,7 @@ class WhisperTokenizer  : BPETokenizer {}
 class GemmaTokenizer    : BPETokenizer {}
 class CodeLlamaTokenizer: BPETokenizer {}
 class CohereTokenizer   : BPETokenizer {}
+class Qwen2Tokenizer    : BPETokenizer {}
 
 class T5Tokenizer       : UnigramTokenizer {}