Update app.py

app.py

@@ -4,12 +4,14 @@ import itertools
 from word_piece_tokenizer import WordPieceTokenizer
 import tiktoken
 from transformers import AutoTokenizer
+from transformers import GPT2TokenizerFast
 
 from nltk.tokenize import TreebankWordTokenizer, wordpunct_tokenize, TweetTokenizer
 
 qwen_tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-0.5B-Instruct')
 ruadapt_tokenizer = AutoTokenizer.from_pretrained('msu-rcc-lair/RuadaptQwen2.5-32B-instruct')
 aya_tokenizer = AutoTokenizer.from_pretrained('mlx-community/aya-expanse-32b-8bit')
+claude_tokenizer = GPT2TokenizerFast.from_pretrained('Xenova/claude-tokenizer')
 
 ### User Interface ###
 st.title("Tokenization")
@@ -30,7 +32,8 @@ tokenizer = st.selectbox(
         "Qwen2.5 Tokenizer",
         "RuAdapt Tokenizer",
         "Aya-Expanse Tokenizer",
-        "Byte Pair Encoding (Open AI GPT-4o)",
+        "Open AI GPT-4o Tokenizer",
+        "Anthropic Claude Tokenizer",
     ),
     index=None,
     placeholder="Select a tokenizer",
@@ -140,8 +143,8 @@ elif tokenizer == "Aya-Expanse Tokenizer":
         color = itertools.cycle(colors)
         st.write(stream_wp_token_ids)
 
-elif tokenizer == "Byte Pair Encoding (Open AI GPT-4o)":
-    with st.expander("About Byte Pair Encoding (BPE)"):
+elif tokenizer == ""Open AI GPT-4o Tokenizer"":
+    with st.expander("About "Open AI GPT-4o Tokenizer""):
         st.write(bpe_desc)
     encoding = tiktoken.encoding_for_model("gpt-4o")
     ids = encoding.encode(txt)
@@ -153,4 +156,14 @@ elif tokenizer == "Byte Pair Encoding (Open AI GPT-4o)":
         color = itertools.cycle(colors)
         st.write(stream_wp_token_ids)
 
+elif tokenizer == "Anthropic Claude Tokenizer":
+    with st.expander("About Anthropic Claude Tokenizer"):
+        st.write('')
+    ids = claude_tokenizer.encode(txt)
+    split_tokens = [claude_tokenizer.decode([t]) for t in ids]
+    st.write(stream_data)
+    if token_id == True:
+        color = itertools.cycle(colors)
+        st.write(stream_wp_token_ids)
+
 st.write(num_tokens(txt))