Updates for SuperBPE website embed

assets/index-DEbmRw68.js

@@ -7139,7 +7139,7 @@ function jd({
 const Ql = Object.freeze({
   "UW/OLMo2-8B-SuperBPE-t180k": "SuperBPE 200k (t=180k)",
   "UW/OLMo2-8B-SuperBPE-t160k": "SuperBPE 200k (t=160k)",
-  "UW/OLMo2-8B-SuperBPE-t80k": "SuperBPE 200k (t=160k)",
+  "UW/OLMo2-8B-SuperBPE-t80k": "SuperBPE 200k (t=80k)",
   "UW/OLMo2-8B-BPE": "BPE 200k (baseline)",
   "Xenova/gpt-4o": "GPT-4o",
   "Xenova/llama-3-tokenizer": "Llama 3",
@@ -7167,6 +7167,13 @@ function Md() {
     const k = x => {
       l(x.data.token_ids), u(x.data.decoded), s(x.data.margins)
     };
+    if (S.current) {
+      S.current.value = "By the way, I am a fan of the Milky Way!";
+    }
+    c.current.postMessage({
+      model_id: m,
+      text: S.current.value
+    })
     return c.current.addEventListener("message", k), () => c.current.removeEventListener("message", k)
   }, []);
   const a = Z.useCallback(() => {
@@ -7194,19 +7201,17 @@ function Md() {
       text: S.current.value
     })
   }, []);
+  const textEncoder = new TextEncoder();
   return R.jsxs("div", {
-    className: "w-full max-w-[720px] flex flex-col gap-4 items-center",
+    className: "w-full flex flex-col gap-4",
     children: [R.jsxs("div", {
-      children: [R.jsx("h1", {
-        className: "text-5xl font-bold mb-2",
-        children: "SuperBPE Playground"
-      }), R.jsxs("h2", {
+      children: [R.jsxs("h2", {
         className: "text-lg font-normal",
-        children: ["Experiment with different tokenizers (running", " ", R.jsx("a", {
+        children: ["Experiment with our tokenizers (running", " ", R.jsx("a", {
           className: "text-gray-900 underline",
           href: "https://github.com/huggingface/transformers.js",
           children: "locally"
-        }), " ", "in your browser)."]
+        }), " ", "in your browser)!"]
       })]
     }), R.jsxs("div", {
       children: [R.jsx("select", {
@@ -7214,7 +7219,7 @@ function Md() {
         onChange: k => {
           a(), w(""), v(k)
         },
-        className: "bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2",
+        className: "bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block p-2",
         children: Object.entries(Ql).map(([k, x]) => R.jsx("option", {
           value: k,
           children: x
@@ -7226,34 +7231,43 @@ function Md() {
         onChange: k => {
           w(k.target.value), v(k)
         },
-        className: "bg-white border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full py-1 px-2 mt-1"
+        className: "bg-white border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block py-1 px-2 mt-1"
       })]
     }), R.jsx("textarea", {
       ref: S,
       onChange: d,
       rows: "8",
-      className: "font-mono text-lg block w-full p-2.5 text-gray-900 bg-gray-50 rounded-lg border border-gray-200",
+      className: "font-mono text-lg block w-full p-2.5 h-[200px] text-gray-900 bg-gray-50 rounded-lg border border-gray-200",
       placeholder: "Enter some text",
       defaultValue: n ?? ((E = S.current) == null ? void 0 : E.value) ?? ""
     }), R.jsxs("div", {
-      className: "flex justify-center gap-5",
-      children: [R.jsxs("div", {
-        className: "flex flex-col",
-        children: [R.jsx("h2", {
+      className: "",
+      children: [R.jsxs("span", {
+        className: "",
+        children: [R.jsx("span", {
+          className: "font-semibold",
+          children: (((N = S.current) == null ? void 0 : textEncoder.encode(N.value).length) ?? 0).toLocaleString()
+        }), R.jsx("span", {
           className: "font-semibold uppercase leading-4",
-          children: "Tokens"
-        }), R.jsx("h3", {
-          className: "font-semibold text-3xl",
+          children: " Bytes, "
+        })]
+      }), R.jsxs("span", {
+        className: "",
+        children: [R.jsx("span", {
+          className: "font-semibold",
           children: r.length.toLocaleString()
+        }), R.jsx("span", {
+          className: "font-semibold uppercase leading-4",
+          children: " Tokens: "
         })]
-      }), R.jsxs("div", {
-        className: "flex flex-col",
-        children: [R.jsx("h2", {
+      }), R.jsxs("span", {
+        className: "",
+        children: [R.jsx("span", {
+          className: "font-bold",
+          children: ((((N = S.current) == null ? void 0 : textEncoder.encode(N.value).length) ?? 0) / r.length || 0).toLocaleString()
+        }), R.jsx("span", {
           className: "font-semibold uppercase leading-4",
-          children: "Characters"
-        }), R.jsx("h3", {
-          className: "font-semibold text-3xl",
-          children: (((N = S.current) == null ? void 0 : N.value.length) ?? 0).toLocaleString()
+          children: " Bytes/token"
         })]
       })]
     }), R.jsx("div", {

assets/index-Dhl4q2CV.css

@@ -2,8 +2,8 @@
   max-width: 1280px;
   width: 100%;
   margin: 0 auto;
-  padding: 2rem;
-  text-align: center;
+  padding: 0;
+  text-align: left;
   display: flex;
   justify-content: center;
   align-items: center;
@@ -351,7 +351,7 @@ video {
   height: 1rem;
 }
 .h-\[200px\] {
-  height: 200px;
+  height: 150px;
 }
 .w-4 {
   width: 1rem;
@@ -509,8 +509,8 @@ video {
   line-height: 1.5;
   font-weight: 400;
   color-scheme: light dark;
-  color: #ffffffde;
-  background-color: #242424;
+  color: rgb(54, 54, 54);
+  background-color: white;
   font-synthesis: none;
   text-rendering: optimizeLegibility;
   -webkit-font-smoothing: antialiased;
@@ -523,18 +523,6 @@ body {
   place-items: center;
   min-height: 100vh;
 }
-@media (prefers-color-scheme: light) {
-  :root {
-    color: #213547;
-    background-color: #fff;
-  }
-  a:hover {
-    color: #747bff;
-  }
-  button {
-    background-color: #f9f9f9;
-  }
-}
 .focus\:border-blue-500:focus {
   --tw-border-opacity: 1;
   border-color: rgb(59 130 246 / var(--tw-border-opacity));
@@ -543,9 +531,3 @@ body {
   --tw-ring-opacity: 1;
   --tw-ring-color: rgb(59 130 246 / var(--tw-ring-opacity));
 }
-@media (prefers-color-scheme: dark) {
-  .dark\:text-gray-300 {
-    --tw-text-opacity: 1;
-    color: rgb(209 213 219 / var(--tw-text-opacity));
-  }
-}

index.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>The Tokenizer Playground</title>
+    <title>SuperBPE Playground</title>
     <script type="module" crossorigin src="/assets/index-DEbmRw68.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-Dhl4q2CV.css">
     <script