trust_remote_code=True

app.py

@@ -19,7 +19,7 @@ openai_tokenizer_list = [
 
 # load tokenizers
 hf_tokenizers = [
-    AutoTokenizer.from_pretrained(model_name_or_id, use_fast=use_fast)
+    AutoTokenizer.from_pretrained(model_name_or_id, use_fast=use_fast, trust_remote_code=True)
     for model_name_or_id, use_fast in hf_tokenizer_list
 ]
 

test_transformers.py

@@ -0,0 +1,12 @@
+from transformers import AutoTokenizer
+
+
+tokenizer = AutoTokenizer.from_pretrained('tugstugi/bert-large-mongolian-cased', use_fast=False)
+
+
+test_input = "Мөнгөө тушаачихсаныхаа дараа мэдэгдээрэй"
+
+print("input:", test_input)
+print("tokenizer.encode()", tokenizer.encode(test_input))
+print("tokenizer decode", [(tokenizer.decode(token_id), token_id) for token_id in tokenizer.encode(test_input)])
+print("tokenizer()", tokenizer(test_input))