print missing metadata models

src/leaderboard/read_evals.py

@@ -29,7 +29,7 @@ class EvalResult:
     license: str = "?"
     lang: str = "?"
     likes: int = 0
-    num_params: int = 0
+    num_params: int = None
     date: str = "" # submission date of request file
     still_on_hub: bool = False
     n_shot: NShotType = NShotType.n0
@@ -61,6 +61,7 @@ class EvalResult:
 
         org_and_model = re.sub(r"^pretrained=", "", org_and_model)
         org_and_model = org_and_model.replace(",trust_remote_code=True", "")
+        org_and_model = re.sub(",prefix_token_id=\d+", "", org_and_model)
 
         org_and_model = org_and_model.split("/", 1)
 
@@ -118,7 +119,7 @@ class EvalResult:
         try:
             meta=metadata[self.full_model]
             self.model_type = ModelType.from_str(meta.get("type", "?"))
-            self.num_params = meta.get("params", 0)
+            self.num_params = meta.get("params", None)
             self.license = meta.get("license", "?")
             self.lang = meta.get("lang", "?")
             #TODO desc name
@@ -359,6 +360,7 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
             continue
 
     missing_results_for_task = {}
+    missing_metadata = []
     for v in eval_results.values():
         r = v.to_dict()
         for task in Tasks:
@@ -368,11 +370,18 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
                     missing_results_for_task[task_name].append(f"{v.full_model}|{v.org_and_model}")
                 else:
                     missing_results_for_task[task_name] = [f"{v.full_model}|{v.org_and_model}"]
+        if r[AutoEvalColumn.lang.name] is None or r[AutoEvalColumn.lang.name]=="?":
+            missing_metadata.append(f"{v.full_model}")
 
     # print('missing_results_for_task', missing_results_for_task)
     for task, models in missing_results_for_task.items():
         print(f"Missing results for {task} for {len(models)} models")
         print(" ".join(models))
 
+    print(f"Missing metadata for {len(missing_metadata)} models")
+    for model in missing_metadata:
+        print(model)
+    print()
+
 
     return results