warning

src/leaderboard/read_evals.py

@@ -450,6 +450,7 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
     for_run=[]
     for v in eval_results.values():
         r = v.to_dict()
+        in_progress=False
         for task in Tasks:
             if r[task.value.col_name] is None:
                 task_name = f"{r['n_shot']}|{task.value.benchmark}"
@@ -457,12 +458,17 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
                     missing_results_for_task[task_name].append(f"{v.full_model}|{v.org_and_model}")
                     if v.still_on_hub and task.value.benchmark in all_tasks:
                         for_run.append([r["n_shot"], task.value.benchmark, v.full_model])
+                        in_progress=True
                         # print(f'sbatch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
                 else:
                     missing_results_for_task[task_name] = [f"{v.full_model}|{v.org_and_model}"]
                     if v.still_on_hub and task.value.benchmark in all_tasks:
                         for_run.append([r["n_shot"], task.value.benchmark, v.full_model])
+                        in_progress=True
                         # print(f'sbatch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
+        if in_progress:
+            v.model = '⚠️' + v.model
+
         if r[AutoEvalColumn.lang.name] is None or r[AutoEvalColumn.lang.name] == "?":
             missing_metadata.append(f"{v.full_model}")
         all_models.append((v.full_model, v.num_params, v.still_on_hub))