Show token value for token_id in metadata table

app.py

@@ -33,6 +33,8 @@ def init_state(
 
 
 def human_readable_metadata(
+    meta: MetadataState,
+    key: str,
     typ: int,
     val: Any,
 ) -> tuple[str, Any]:
@@ -47,8 +49,13 @@ def human_readable_metadata(
             val = str(val[:8])[:-1] + ', ...]'
         else:
             val = str(val)
+    elif key.endswith('_token_id'):
+        tokens = meta.key.get('tokenizer.ggml.tokens', (-1, []))[1]
+
+        if isinstance(val, int) and val >= 0 and val < len(tokens):
+            val = f'{tokens[val]} ({val})'
 
-    return typ, val
+    return key, typ, val
 
 
 with gr.Blocks(
@@ -318,7 +325,7 @@ with gr.Blocks(
                 meta.var['branch'] = branch
 
                 for k, v in progress.tqdm(metadata, desc = 'Reading metadata...', total = num_metadata, unit = f' of {num_metadata} metadata keys...'):
-                    m.append([k, *human_readable_metadata(v.type, v.value)])
+                    m.append([*human_readable_metadata(meta, k, v.type, v.value)])
                     meta.key[k] = (v.type, v.value)
 
                     # FIXME
@@ -481,7 +488,7 @@ with gr.Blocks(
 
         m = []
         for k, v in meta.key.items():
-            m.append([k, *human_readable_metadata(v[0], v[1])])
+            m.append([*human_readable_metadata(meta, k, v[0], v[1])])
 
         link = str(request.request.url_for('download', repo_file = meta.var['repo_file']).include_query_params(branch = meta.var['branch']))
         if link.startswith('http:'):