Update app.py

app.py

@@ -3,9 +3,13 @@ import streamlit as st
 import itertools
 from word_piece_tokenizer import WordPieceTokenizer
 import tiktoken
+from transformers import AutoTokenizer
 
 from nltk.tokenize import TreebankWordTokenizer, wordpunct_tokenize, TweetTokenizer
 
+qwen_tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-0.5B-Instruct')
+ruadapt_tokenizer = AutoTokenizer.from_pretrained('msu-rcc-lair/RuadaptQwen2.5-32B-instruct')
+
 ### User Interface ###
 st.title("Tokenization")
 
@@ -22,9 +26,8 @@ tokenizer = st.selectbox(
     "Tokenizer",
     (
         "White Space",
-        "Penn Treebank (NLTK Default)",
-        "Tweet Tokenizer (NLTK)",
-        "WordPiece (BERT)",
+        "Qwen2.5 Tokenizer",
+        "RuAdapt Tokenizer",
         "Byte Pair Encoding (Open AI GPT-4o)",
     ),
     index=None,
@@ -96,31 +99,21 @@ if tokenizer == "White Space":
         unique_tokens = unique_list(split_tokens)
         st.write(stream_token_ids)
 
-elif tokenizer == "Penn Treebank (NLTK Default)":
-    with st.expander("About Penn Treebank Tokenizer"):
-        st.write(treebank_desc)
-    split_tokens = TreebankWordTokenizer().tokenize(txt)
-    st.write(stream_data)
-    if token_id == True:
-        color = itertools.cycle(colors)
-        unique_tokens = unique_list(split_tokens)
-        st.write(stream_token_ids)
-
-elif tokenizer == "Tweet Tokenizer (NLTK)":
-    with st.expander("About Tweet Tokenizer"):
-        st.write(tweet_desc)
-    split_tokens = TweetTokenizer().tokenize(txt)
+elif tokenizer == "Qwen2.5 Tokenizer":
+    with st.expander("About Qwen2.5 Tokenizer"):
+        st.write('')
+    ids = qwen_tokenizer.encode(txt)
+    split_tokens = qwen_tokenizer.tokenize(txt)
     st.write(stream_data)
     if token_id == True:
         color = itertools.cycle(colors)
-        unique_tokens = unique_list(split_tokens)
-        st.write(stream_token_ids)
+        st.write(stream_wp_token_ids)
 
-elif tokenizer == "WordPiece (BERT)":
-    with st.expander("About WordPiece Tokenizer"):
-        st.write(wordpiece_desc)
-    ids = WordPieceTokenizer().tokenize(txt)
-    split_tokens = WordPieceTokenizer().convert_ids_to_tokens(ids)
+elif tokenizer == "RuAdapt Tokenizer":
+    with st.expander("About RuAdapt Tokenizer"):
+        st.write('')
+    ids = ruadapt_tokenizer.encode(txt)
+    split_tokens = ruadapt_tokenizer.tokenize(txt)
     st.write(stream_data)
     if token_id == True:
         color = itertools.cycle(colors)