revert logic

src/leaderboard/read_evals.py

@@ -103,6 +103,13 @@ class EvalResult:
                     results[task.benchmark] = 0.0
                     continue
 
+            # New tasks have been added, we need to skip them if not exists
+            if task.benchmark in ["ko_winogrande", "ko_gsm8k", "ko_eq_bench", "ko_inst_follow", "kor_nat_cka", "kor_nat_sva", "ko_harmlessness", "ko_helpfulness"]:
+                accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
+                if accs.size == 0 or any([acc is None for acc in accs]):
+                    results[task.benchmark] = 0.0
+                    continue
+
             # We average all scores of a given metric (mostly for mmlu)
             accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
             if accs.size == 0 or any([acc is None for acc in accs]):
@@ -144,7 +151,28 @@ class EvalResult:
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
 
-        average = sum([v for v in self.results.values() if v is not None]) / sum([1 for v in self.results.values() if v is not None])
+        # Skip the two new tasks for now
+        # TODO: safely remove this code when the task results are all added
+        skip_avg_len = 0
+        if self.results['ko_winogrande'] == 0.0:
+            skip_avg_len += 1
+        if self.results['ko_gsm8k'] == 0.0:
+            skip_avg_len += 1
+        if self.results['ko_eq_bench'] == 0.0:
+            skip_avg_len += 1
+        if self.results['ko_inst_follow'] == 0.0:
+            skip_avg_len += 1
+        if self.results['kor_nat_cka'] == 0.0:
+            skip_avg_len += 1
+        if self.results['kor_nat_sva'] == 0.0:
+            skip_avg_len += 1
+        if self.results['ko_harmlessness'] == 0.0:
+            skip_avg_len += 1
+        if self.results['ko_helpfulness'] == 0.0:
+            skip_avg_len += 1
+
+        average = sum([v for v in self.results.values() if v is not None]) / (len(Tasks) - skip_avg_len)
+
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,

src/populate.py

@@ -16,7 +16,6 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     filter_models(all_data_json)
 
     df = pd.DataFrame.from_records(all_data_json)
-    print(df.to_string())
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
 

src/tools/plots.py

@@ -36,7 +36,25 @@ def create_scores_df(raw_data: list[EvalResult]) -> pd.DataFrame:
 
             current_date = row["date"]
             if task.benchmark == "Average":
-                current_score = np.mean(list(row["results"].values()))
+                avg_skip_len = 0
+                if row["results"]["ko_winogrande"] == 0.0:
+                    avg_skip_len += 1
+                if row["results"]["ko_gsm8k"] == 0.0:
+                    avg_skip_len += 1
+                if row["results"]["ko_eq_bench"] == 0.0:
+                    avg_skip_len += 1
+                if row["results"]["ko_inst_follow"] == 0.0:
+                    avg_skip_len += 1
+                if row["results"]["kor_nat_cka"] == 0.0:
+                    avg_skip_len += 1
+                if row["results"]["kor_nat_sva"] == 0.0:
+                    avg_skip_len += 1
+                if row["results"]["ko_harmlessness"] == 0.0:
+                    avg_skip_len += 1
+                if row["results"]["ko_helpfulness"] == 0.0:
+                    avg_skip_len += 1
+                    
+                current_score = np.sum(list(row["results"].values())) / (len(row["results"]) - avg_skip_len)
             else:
                 current_score = row["results"][task.benchmark]