Ensure token names are listed for _token_id even when tokens metadata comes afterwards

app.py

@@ -37,7 +37,7 @@ def human_readable_metadata(
     key: str,
     typ: int,
     val: Any,
-) -> tuple[str, Any]:
+) -> tuple[str, int | str, Any]:
     typ = GGUFValueType(typ).name
 
     if typ == 'ARRAY':
@@ -358,8 +358,14 @@ with gr.Blocks(
                 meta.var['repo_file'] = repo_file
                 meta.var['branch'] = branch
 
+                deferred_updates = []
                 for k, v in progress.tqdm(metadata, desc = 'Reading metadata...', total = num_metadata, unit = f' of {num_metadata} metadata keys...'):
-                    m.append([*human_readable_metadata(meta, k, v.type, v.value)])
+                    human = [*human_readable_metadata(meta, k, v.type, v.value)]
+
+                    if k.endswith('_token_id') and 'tokenizer.ggml.tokens' not in meta.key:
+                        deferred_updates.append(((k, v.type, v.value), human))
+
+                    m.append(human)
                     meta.key[k] = (v.type, v.value)
 
                     # FIXME
@@ -368,6 +374,9 @@ with gr.Blocks(
                     #         m,
                     #     ),
                     # }
+
+                for data, human in deferred_updates:
+                    human[:] = human_readable_metadata(meta, *data)
         except Exception as e:
             raise gr.Error(e)