added selection of verified results

app.py

@@ -10,9 +10,12 @@ from os.path import exists
 import threading
 
 
-def get_model_ids():
+def get_model_ids(author=None):
     api = HfApi()
-    models = api.list_models(filter="model-index")
+    if author is None:
+        models = api.list_models(filter="model-index")
+    else:
+        models = api.list_models(filter="model-index", author="autoevaluate")
     model_ids = [x.modelId for x in models]
     return model_ids
 
@@ -42,24 +45,39 @@ def parse_metric_value(value):
     return value
 
 
-def parse_metrics_rows(meta):
+def parse_metrics_rows(meta, from_autoeval=False):
     if not isinstance(meta["model-index"], list) or len(meta["model-index"]) == 0 or "results" not in meta["model-index"][0]:
         return None
     for result in meta["model-index"][0]["results"]:
         if not isinstance(result, dict) or "dataset" not in result or "metrics" not in result or "type" not in result["dataset"]:
             continue
         dataset = result["dataset"]["type"]
-        if "args" not in result["dataset"]:
-            continue
-        row = {"dataset": dataset}
+        row = {"dataset": dataset, "split": None, "config": None, "verified": from_autoeval}
+        if "split" in result["dataset"]:
+            row["split"] = result["dataset"]["split"]
+        if "config" in result["dataset"]:
+            row["config"] = result["dataset"]["config"]
         for metric in result["metrics"]:
             type = metric["type"].lower().strip()
+            if type not in ("dataset", "split", "config", "verified"):
+                # Metrics are not allowed to be named "dataset", "split", "config", or "verified".
+                continue
             value = parse_metric_value(metric.get("value", None))
             if value is None:
                 continue
-            if type not in row or value < row[type]:
-                # overwrite the metric if the new value is lower (e.g. with LM)
-                row[type] = value
+            if type in row:
+                new_metric_better = value < row[type] if type in ascending_metrics else value > row[type]
+            if type not in row or new_metric_better:
+                # overwrite the metric if the new value is better.
+
+                if from_autoeval:
+                    # if the metric is from autoeval, only include it in the leaderboard if
+                    # it is a verified metric. Unverified metrics are already included
+                    # in the leaderboard from the unverified model card.
+                    if "verified" in metric and metric["verified"]:
+                        row[type] = value
+                else:
+                    row[type] = value
         yield row
 
 @st.cache(ttl=3600)
@@ -68,11 +86,12 @@ def get_data_wrapper():
     def get_data():
         data = []
         model_ids = get_model_ids()
+        model_ids_from_autoeval = set(get_model_ids(author="autoevaluate"))
         for model_id in tqdm(model_ids):
             meta = get_metadata(model_id)
             if meta is None:
                 continue
-            for row in parse_metrics_rows(meta):
+            for row in parse_metrics_rows(meta, from_autoeval=model_id in model_ids_from_autoeval):
                 if row is None:
                     continue
                 row["model_id"] = model_id
@@ -108,6 +127,10 @@ if "dataset" in query_params:
     if len(query_params["dataset"]) > 0 and query_params["dataset"][0] in selectable_datasets:
         default_dataset = query_params["dataset"][0]
 
+only_verified_results = st.sidebar.checkbox(
+    "Filter for Verified Results",
+)
+
 dataset = st.sidebar.selectbox(
     "Dataset",
     selectable_datasets,
@@ -118,15 +141,19 @@ st.experimental_set_query_params(**{"dataset": [dataset]})
 dataset_df = dataframe[dataframe.dataset == dataset]
 dataset_df = dataset_df.dropna(axis="columns", how="all")
 
+if only_verified_results:
+    dataset_df = dataset_df[dataset_df["verified"]]
+
 selectable_metrics = list(filter(lambda column: column not in ("model_id", "dataset"), dataset_df.columns))
+
+dataset_df = dataset_df.filter(["model_id"] + selectable_metrics)
+dataset_df = dataset_df.dropna(thresh=2)  # Want at least two non-na values (one for model_id and one for a metric).
+
 sorting_metric = st.sidebar.radio(
     "Sorting Metric",
     selectable_metrics,
 )
 
-dataset_df = dataset_df.filter(["model_id"] + selectable_metrics)
-dataset_df = dataset_df.dropna(thresh=2)  # Want at least two non-na values (one for model_id and one for a metric).
-
 st.markdown(
     "Please click on the model's name to be redirected to its model card."
 )