Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -143,8 +143,8 @@ elif tokenizer == "Aya-Expanse Tokenizer":
|
|
| 143 |
color = itertools.cycle(colors)
|
| 144 |
st.write(stream_wp_token_ids)
|
| 145 |
|
| 146 |
-
elif tokenizer == "
|
| 147 |
-
with st.expander("About "Open AI GPT-4o Tokenizer"
|
| 148 |
st.write(bpe_desc)
|
| 149 |
encoding = tiktoken.encoding_for_model("gpt-4o")
|
| 150 |
ids = encoding.encode(txt)
|
|
|
|
| 143 |
color = itertools.cycle(colors)
|
| 144 |
st.write(stream_wp_token_ids)
|
| 145 |
|
| 146 |
+
elif tokenizer == "Open AI GPT-4o Tokenizer":
|
| 147 |
+
with st.expander("About "Open AI GPT-4o Tokenizer"):
|
| 148 |
st.write(bpe_desc)
|
| 149 |
encoding = tiktoken.encoding_for_model("gpt-4o")
|
| 150 |
ids = encoding.encode(txt)
|