Update app.py

app.py

@@ -76,6 +76,15 @@ def stream_wp_token_ids():
     for id in ids:
         yield f":{next(color)}-background[{id}]  "
 
+def num_tokens(txt):
+    words = white_space_tokenizer(txt)
+    n_words = len(n_words) if len(n_words) else 1
+    try:
+        return f'Token count {len(ids)}, f-rate {len(ids)/n_words}'    
+    except:
+        return ''
+        
+    
 
 ### Tokenizer Descriptions ###
 
@@ -131,3 +140,5 @@ elif tokenizer == "Byte Pair Encoding (Open AI GPT-4o)":
     if token_id == True:
         color = itertools.cycle(colors)
         st.write(stream_wp_token_ids)
+
+st.write(num_tokens())