Update app.py

app.py

@@ -9,6 +9,7 @@ from nltk.tokenize import TreebankWordTokenizer, wordpunct_tokenize, TweetTokeni
 
 qwen_tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-0.5B-Instruct')
 ruadapt_tokenizer = AutoTokenizer.from_pretrained('msu-rcc-lair/RuadaptQwen2.5-32B-instruct')
+aya_tokenizer = AutoTokenizer.from_pretrained('mlx-community/aya-expanse-32b-8bit')
 
 ### User Interface ###
 st.title("Tokenization")
@@ -128,6 +129,16 @@ elif tokenizer == "RuAdapt Tokenizer":
         color = itertools.cycle(colors)
         st.write(stream_wp_token_ids)
 
+elif tokenizer == "Aya-Expanse Tokenizer":
+    with st.expander("About Aya-Expanse Tokenizer"):
+        st.write('')
+    ids = aya_tokenizer.encode(txt)
+    split_tokens = [aya_tokenizer.decode([t]) for t in ids]
+    st.write(stream_data)
+    if token_id == True:
+        color = itertools.cycle(colors)
+        st.write(stream_wp_token_ids)
+
 elif tokenizer == "Byte Pair Encoding (Open AI GPT-4o)":
     with st.expander("About Byte Pair Encoding (BPE)"):
         st.write(bpe_desc)