From bda7f90b88a0add678e43c9d121392f66f6b87b7 Mon Sep 17 00:00:00 2001
From: Finn Voorhees <finnvoorhees@gmail.com>
Date: Tue, 24 Dec 2024 13:05:39 -0500
Subject: [PATCH] Add Qwen2Tokenizer (#146)

---
 Sources/Tokenizers/Tokenizer.swift | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Sources/Tokenizers/Tokenizer.swift b/Sources/Tokenizers/Tokenizer.swift
index 9c8c381..1319ed0 100644
--- a/Sources/Tokenizers/Tokenizer.swift
+++ b/Sources/Tokenizers/Tokenizer.swift
@@ -72,6 +72,7 @@ struct TokenizerModel {
         "T5Tokenizer"        : T5Tokenizer.self,
         "WhisperTokenizer"   : WhisperTokenizer.self,
         "CohereTokenizer"    : CohereTokenizer.self,
+        "Qwen2Tokenizer"     : Qwen2Tokenizer.self,
         "PreTrainedTokenizer": BPETokenizer.self
     ]
 
@@ -488,6 +489,7 @@ class WhisperTokenizer  : BPETokenizer {}
 class GemmaTokenizer    : BPETokenizer {}
 class CodeLlamaTokenizer: BPETokenizer {}
 class CohereTokenizer   : BPETokenizer {}
+class Qwen2Tokenizer    : BPETokenizer {}
 
 class T5Tokenizer       : UnigramTokenizer {}