add zephyr

vocab/__init__.py

@@ -70,7 +70,7 @@ uniq_tokenizers = [
     ""
 ]
 
-# TODO: alias/abbr, hf_path, tokenizer_class/type, comments,
+# TODO: alias/abbr, description, hf_path, tokenizer_class/type, comments, Organization
 all_tokenizers = [
     ##### bert 系列
     ("bert_base_cased", "", "bert"),
@@ -99,7 +99,7 @@ all_tokenizers = [
     ("chatyuan_large_v2", "", "sentencepiece"),
     ("prompt_clue", "", "sentencepiece"),
 
-    ("llama", "", "sentencepiece"),  # '中文单字': 700, '中文多字': 0
+    ("llama", "", "sentencepiece", "llama use single digits and thus uses 4 tokens to encode the number 1000"),  # '中文单字': 700, '中文多字': 0
     ("llama2", "", "sentencepiece"),
     ("chinese_llama", "", "sentencepiece"),  #
     ("chinese_llama2", "", "sentencepiece"),  #
@@ -168,6 +168,7 @@ all_tokenizers = [
     ("gemma_7b",),
     ("olmo_7b",),
     ("aya_101",),
+    ("zephyr_7b_beta",)
 ]
 
 all_tokenizers = [tokenizer[0] for tokenizer in all_tokenizers]

vocab/zephyr_7b_beta/__init__.py

@@ -0,0 +1,5 @@
+
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("HuggingFaceH4/zephyr-7b-beta")