sync from github

src/display/about.py

@@ -10,9 +10,9 @@ The OPEN-MOE-LLM-LEADERBOARD includes generation and multiple choice tasks to me
 
 
 Tasks:
-- **Generation Self-consistancy** -- [SelfCheckGPT](https://github.com/potsawee/selfcheckgpt)
 - **Multiple Choice Performance** -- [MMLU](https://arxiv.org/abs/2009.03300)
 - **Mathematics Problem-Solving Performance** -- [GSM8K](https://arxiv.org/abs/2110.14168)
+- **AI Judgment Scores for Responses to Complex User Queries** -- [Arena_Hard](https://lmsys.org/blog/2024-04-19-arena-hard/)
 
 Columns and Metrics:
 - Method: The MOE LLMs inference framework.

src/display/utils.py

@@ -37,9 +37,7 @@ gpu_metrics_to_name_map = {
     GPU_Mem: GPU_Mem,
     "batch_size": BATCH_SIZE,
     "precision": PRECISION,
-    GPU_Name: GPU_Name,
-    MFU: MFU,
-    MBU: MBU
+    GPU_Name: GPU_Name
 }
 
 @dataclass

src/leaderboard/read_evals.py

@@ -65,11 +65,11 @@ class EvalResult:
         if len(org_and_model) == 1:
             org = None
             model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
+            result_key = f"{model}_{precision.value.name}_{inference_framework}"
         else:
             org = org_and_model[0]
             model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
+            result_key = f"{org}_{model}_{precision.value.name}_{inference_framework}"
         full_model = "/".join(org_and_model)
 
         still_on_hub, error, model_config = is_model_on_hub(
@@ -120,12 +120,13 @@ class EvalResult:
                         multiplier = 1.0
                     if "batch_" in metric or "Mem" in metric or "Util" in metric:
                         multiplier = 1
-
-
+                        
                     # print('RESULTS', data['results'])
                     # print('XXX', benchmark, metric, value, multiplier)
                     if value == "N/A":
-                        results[benchmark][metric] = None
+                        results[benchmark][metric] = "-"
+                    elif value == "auto":
+                        results[benchmark][metric] = "auto"
                     else:
                         results[benchmark][metric] = value * multiplier
 

src/populate.py

@@ -75,7 +75,7 @@ def get_leaderboard_df(
             df[col] = np.nan
 
     if not df.empty:
-        df = df.round(decimals=4)
+        df = df.round(decimals=2)
 
         # filter out if any of the benchmarks have not been produced
         # df = df[has_no_nan_values(df, benchmark_cols)]