Archive project

app.py

@@ -126,191 +126,194 @@ def get_data_wrapper():
 
     return dataframe
 
-dataframe = get_data_wrapper()
+# dataframe = get_data_wrapper()
 
 st.markdown("# 🤗 Leaderboards")
-
-query_params = st.experimental_get_query_params()
-if "first_query_params" not in st.session_state:
-    st.session_state.first_query_params = query_params
-first_query_params = st.session_state.first_query_params
-
-default_task = first_query_params.get("task", [None])[0]
-default_only_verified = bool(int(first_query_params.get("only_verified", [0])[0]))
-print(default_only_verified)
-default_dataset = first_query_params.get("dataset", [None])[0]
-default_split = first_query_params.get("split", [None])[0]
-default_config = first_query_params.get("config", [None])[0]
-default_metric = first_query_params.get("metric", [None])[0]
-
-only_verified_results = st.sidebar.checkbox(
-    "Filter for Verified Results",
-    value=default_only_verified,
-    help="Select this checkbox if you want to see only results produced by the Hugging Face model evaluator, and no self-reported results."
-)
-
-selectable_tasks = list(set(dataframe.pipeline_tag))
-if None in selectable_tasks:
-    selectable_tasks.remove(None)
-selectable_tasks.sort(key=lambda name: name.lower())
-selectable_tasks = ["-any-"] + selectable_tasks
-
-task = st.sidebar.selectbox(
-    "Task",
-    selectable_tasks,
-    index=(selectable_tasks).index(default_task) if default_task in selectable_tasks else 0,
-    help="Filter the selectable datasets by task. Leave as \"-any-\" to see all selectable datasets."
-)
-
-if task != "-any-":
-    dataframe = dataframe[dataframe.pipeline_tag == task]
-
-selectable_datasets = ["-any-"] + sorted(list(set(dataframe.dataset.tolist())), key=lambda name: name.lower())
-if "" in selectable_datasets:
-    selectable_datasets.remove("")
-
-dataset = st.sidebar.selectbox(
-    "Dataset",
-    selectable_datasets,
-    index=selectable_datasets.index(default_dataset) if default_dataset in selectable_datasets else 0,
-    help="Select a dataset to see the leaderboard!"
+st.warning(
+    "**⚠️ This project has been archived. If you want to evaluate LLMs, checkout [this collection](https://huggingface.co/collections/clefourrier/llm-leaderboards-and-benchmarks-✨-64f99d2e11e92ca5568a7cce) of leaderboards.**"
 )
 
-dataframe = dataframe[dataframe.only_verified == only_verified_results]
-
-current_query_params = {"dataset": [dataset], "only_verified": [int(only_verified_results)], "task": [task]}
+# query_params = st.experimental_get_query_params()
+# if "first_query_params" not in st.session_state:
+#     st.session_state.first_query_params = query_params
+# first_query_params = st.session_state.first_query_params
+
+# default_task = first_query_params.get("task", [None])[0]
+# default_only_verified = bool(int(first_query_params.get("only_verified", [0])[0]))
+# print(default_only_verified)
+# default_dataset = first_query_params.get("dataset", [None])[0]
+# default_split = first_query_params.get("split", [None])[0]
+# default_config = first_query_params.get("config", [None])[0]
+# default_metric = first_query_params.get("metric", [None])[0]
 
-st.experimental_set_query_params(**current_query_params)
+# only_verified_results = st.sidebar.checkbox(
+#     "Filter for Verified Results",
+#     value=default_only_verified,
+#     help="Select this checkbox if you want to see only results produced by the Hugging Face model evaluator, and no self-reported results."
+# )
+
+# selectable_tasks = list(set(dataframe.pipeline_tag))
+# if None in selectable_tasks:
+#     selectable_tasks.remove(None)
+# selectable_tasks.sort(key=lambda name: name.lower())
+# selectable_tasks = ["-any-"] + selectable_tasks
 
-if dataset != "-any-":
-    dataset_df = dataframe[dataframe.dataset == dataset]
-else:
-    dataset_df = dataframe
-
-dataset_df = dataset_df.dropna(axis="columns", how="all")
-
-if len(dataset_df) > 0:
-    selectable_configs = list(set(dataset_df["config"]))
-    selectable_configs.sort(key=lambda name: name.lower())
-
-    if "-unspecified-" in selectable_configs:
-        selectable_configs.remove("-unspecified-")
-        selectable_configs = ["-unspecified-"] + selectable_configs
-
-    if dataset != "-any-":
-        config = st.sidebar.selectbox(
-            "Config",
-            selectable_configs,
-            index=selectable_configs.index(default_config) if default_config in selectable_configs else 0,
-            help="Filter the results on the current leaderboard by the dataset config. Self-reported results might not report the config, which is why \"-unspecified-\" is an option."
-        )
-        dataset_df = dataset_df[dataset_df.config == config]
-
-        selectable_splits = list(set(dataset_df["split"]))
-        selectable_splits.sort(key=lambda name: name.lower())
-
-        if "-unspecified-" in selectable_splits:
-            selectable_splits.remove("-unspecified-")
-            selectable_splits = ["-unspecified-"] + selectable_splits
-
-        split = st.sidebar.selectbox(
-            "Split",
-            selectable_splits,
-            index=selectable_splits.index(default_split) if default_split in selectable_splits else 0,
-            help="Filter the results on the current leaderboard by the dataset split. Self-reported results might not report the split, which is why \"-unspecified-\" is an option."
-        )
-
-        current_query_params.update({"config": [config], "split": [split]})
-
-        st.experimental_set_query_params(**current_query_params)
-
-        dataset_df = dataset_df[dataset_df.split == split]
-
-    not_selectable_metrics = ["model_id", "dataset", "split", "config", "pipeline_tag", "only_verified"]
-    selectable_metrics = list(filter(lambda column: column not in not_selectable_metrics, dataset_df.columns))
-
-    dataset_df = dataset_df.filter(["model_id"] + (["dataset"] if dataset == "-any-" else []) + selectable_metrics)
-    dataset_df = dataset_df.dropna(thresh=2)  # Want at least two non-na values (one for model_id and one for a metric).
-
-    sorting_metric = st.sidebar.radio(
-        "Sorting Metric",
-        selectable_metrics,
-        index=selectable_metrics.index(default_metric) if default_metric in selectable_metrics else 0,
-        help="Select the metric to sort the leaderboard by. Click on the metric name in the leaderboard to reverse the sorting order."
-    )
-
-    current_query_params.update({"metric": [sorting_metric]})
-
-    st.experimental_set_query_params(**current_query_params)
-
-    st.markdown(
-        "Please click on the model's name to be redirected to its model card."
-    )
-
-    st.markdown(
-        "Want to beat the leaderboard? Don't see your model here? Simply request an automatic evaluation [here](https://huggingface.co/spaces/autoevaluate/model-evaluator)."
-    )
-
-    st.markdown(
-        "If you do not see your self-reported results here, ensure that your results are in the expected range for all metrics. E.g., accuracy is 0-1, not 0-100."
-    )
-
-    if dataset == "-any-":
-        st.info(
-            "Note: you haven't chosen a dataset, so the leaderboard is showing the best scoring model for a random sample of the datasets available."
-        )
-
-    # Make the default metric appear right after model names and dataset names
-    cols = dataset_df.columns.tolist()
-    cols.remove(sorting_metric)
-    sorting_metric_index = 1 if dataset != "-any-" else 2
-    cols = cols[:sorting_metric_index] + [sorting_metric] + cols[sorting_metric_index:]
-    dataset_df = dataset_df[cols]
-
-    # Sort the leaderboard, giving the sorting metric highest priority and then ordering by other metrics in the case of equal values.
-    dataset_df = dataset_df.sort_values(by=cols[sorting_metric_index:], ascending=[metric in ascending_metrics for metric in cols[sorting_metric_index:]])
-    dataset_df = dataset_df.replace(np.nan, '-')
-
-    # If dataset is "-any-", only show the best model for a random sample of 100 datasets.
-    # Otherwise The leaderboard is way too long and doesn't give the users a feel for all of
-    # the datasets available for a task.
-    if dataset == "-any-":
-        filtered_dataset_df_dict = {column: [] for column in dataset_df.columns}
-        seen_datasets = set()
-        for _, row in dataset_df.iterrows():
-            if row["dataset"] not in seen_datasets:
-                for column in dataset_df.columns:
-                    filtered_dataset_df_dict[column].append(row[column])
-                seen_datasets.add(row["dataset"])
-        dataset_df = pd.DataFrame(filtered_dataset_df_dict)
-        dataset_df = dataset_df.sample(min(100, len(dataset_df)))
-
-    # Make the leaderboard
-    gb = GridOptionsBuilder.from_dataframe(dataset_df)
-    gb.configure_default_column(sortable=False)
-    gb.configure_column(
-        "model_id",
-        cellRenderer=JsCode('''function(params) {return '<a target="_blank" href="https://huggingface.co/'+params.value+'">'+params.value+'</a>'}'''),
-    )
-    if dataset == "-any-":
-        gb.configure_column(
-            "dataset",
-            cellRenderer=JsCode('''function(params) {return '<a target="_blank" href="https://huggingface.co/spaces/autoevaluate/leaderboards?dataset='+params.value+'">'+params.value+'</a>'}'''),
-        )
-    for name in selectable_metrics:
-        gb.configure_column(name, type=["numericColumn","numberColumnFilter","customNumericFormat"], precision=4, aggFunc='sum')
-
-    gb.configure_column(
-        sorting_metric,
-        sortable=True,
-        cellStyle=JsCode('''function(params) { return {'backgroundColor': '#FFD21E'}}''')
-    )
-
-    go = gb.build()
-    fit_columns = len(dataset_df.columns) < 10
-    AgGrid(dataset_df, gridOptions=go, height=28*len(dataset_df) + (35 if fit_columns else 41), allow_unsafe_jscode=True, fit_columns_on_grid_load=fit_columns, enable_enterprise_modules=False)
-
-else:
-    st.markdown(
-        "No " + ("verified" if only_verified_results else "unverified") + " results to display. Try toggling the verified results filter."
-    )
+# task = st.sidebar.selectbox(
+#     "Task",
+#     selectable_tasks,
+#     index=(selectable_tasks).index(default_task) if default_task in selectable_tasks else 0,
+#     help="Filter the selectable datasets by task. Leave as \"-any-\" to see all selectable datasets."
+# )
+
+# if task != "-any-":
+#     dataframe = dataframe[dataframe.pipeline_tag == task]
+
+# selectable_datasets = ["-any-"] + sorted(list(set(dataframe.dataset.tolist())), key=lambda name: name.lower())
+# if "" in selectable_datasets:
+#     selectable_datasets.remove("")
+
+# dataset = st.sidebar.selectbox(
+#     "Dataset",
+#     selectable_datasets,
+#     index=selectable_datasets.index(default_dataset) if default_dataset in selectable_datasets else 0,
+#     help="Select a dataset to see the leaderboard!"
+# )
+
+# dataframe = dataframe[dataframe.only_verified == only_verified_results]
+
+# current_query_params = {"dataset": [dataset], "only_verified": [int(only_verified_results)], "task": [task]}
+
+# st.experimental_set_query_params(**current_query_params)
+
+# if dataset != "-any-":
+#     dataset_df = dataframe[dataframe.dataset == dataset]
+# else:
+#     dataset_df = dataframe
+
+# dataset_df = dataset_df.dropna(axis="columns", how="all")
+
+# if len(dataset_df) > 0:
+#     selectable_configs = list(set(dataset_df["config"]))
+#     selectable_configs.sort(key=lambda name: name.lower())
+
+#     if "-unspecified-" in selectable_configs:
+#         selectable_configs.remove("-unspecified-")
+#         selectable_configs = ["-unspecified-"] + selectable_configs
+
+#     if dataset != "-any-":
+#         config = st.sidebar.selectbox(
+#             "Config",
+#             selectable_configs,
+#             index=selectable_configs.index(default_config) if default_config in selectable_configs else 0,
+#             help="Filter the results on the current leaderboard by the dataset config. Self-reported results might not report the config, which is why \"-unspecified-\" is an option."
+#         )
+#         dataset_df = dataset_df[dataset_df.config == config]
+
+#         selectable_splits = list(set(dataset_df["split"]))
+#         selectable_splits.sort(key=lambda name: name.lower())
+
+#         if "-unspecified-" in selectable_splits:
+#             selectable_splits.remove("-unspecified-")
+#             selectable_splits = ["-unspecified-"] + selectable_splits
+
+#         split = st.sidebar.selectbox(
+#             "Split",
+#             selectable_splits,
+#             index=selectable_splits.index(default_split) if default_split in selectable_splits else 0,
+#             help="Filter the results on the current leaderboard by the dataset split. Self-reported results might not report the split, which is why \"-unspecified-\" is an option."
+#         )
+
+#         current_query_params.update({"config": [config], "split": [split]})
+
+#         st.experimental_set_query_params(**current_query_params)
+
+#         dataset_df = dataset_df[dataset_df.split == split]
+
+#     not_selectable_metrics = ["model_id", "dataset", "split", "config", "pipeline_tag", "only_verified"]
+#     selectable_metrics = list(filter(lambda column: column not in not_selectable_metrics, dataset_df.columns))
+
+#     dataset_df = dataset_df.filter(["model_id"] + (["dataset"] if dataset == "-any-" else []) + selectable_metrics)
+#     dataset_df = dataset_df.dropna(thresh=2)  # Want at least two non-na values (one for model_id and one for a metric).
+
+#     sorting_metric = st.sidebar.radio(
+#         "Sorting Metric",
+#         selectable_metrics,
+#         index=selectable_metrics.index(default_metric) if default_metric in selectable_metrics else 0,
+#         help="Select the metric to sort the leaderboard by. Click on the metric name in the leaderboard to reverse the sorting order."
+#     )
+
+#     current_query_params.update({"metric": [sorting_metric]})
+
+#     st.experimental_set_query_params(**current_query_params)
+
+#     st.markdown(
+#         "Please click on the model's name to be redirected to its model card."
+#     )
+
+#     st.markdown(
+#         "Want to beat the leaderboard? Don't see your model here? Simply request an automatic evaluation [here](https://huggingface.co/spaces/autoevaluate/model-evaluator)."
+#     )
+
+#     st.markdown(
+#         "If you do not see your self-reported results here, ensure that your results are in the expected range for all metrics. E.g., accuracy is 0-1, not 0-100."
+#     )
+
+#     if dataset == "-any-":
+#         st.info(
+#             "Note: you haven't chosen a dataset, so the leaderboard is showing the best scoring model for a random sample of the datasets available."
+#         )
+
+#     # Make the default metric appear right after model names and dataset names
+#     cols = dataset_df.columns.tolist()
+#     cols.remove(sorting_metric)
+#     sorting_metric_index = 1 if dataset != "-any-" else 2
+#     cols = cols[:sorting_metric_index] + [sorting_metric] + cols[sorting_metric_index:]
+#     dataset_df = dataset_df[cols]
+
+#     # Sort the leaderboard, giving the sorting metric highest priority and then ordering by other metrics in the case of equal values.
+#     dataset_df = dataset_df.sort_values(by=cols[sorting_metric_index:], ascending=[metric in ascending_metrics for metric in cols[sorting_metric_index:]])
+#     dataset_df = dataset_df.replace(np.nan, '-')
+
+#     # If dataset is "-any-", only show the best model for a random sample of 100 datasets.
+#     # Otherwise The leaderboard is way too long and doesn't give the users a feel for all of
+#     # the datasets available for a task.
+#     if dataset == "-any-":
+#         filtered_dataset_df_dict = {column: [] for column in dataset_df.columns}
+#         seen_datasets = set()
+#         for _, row in dataset_df.iterrows():
+#             if row["dataset"] not in seen_datasets:
+#                 for column in dataset_df.columns:
+#                     filtered_dataset_df_dict[column].append(row[column])
+#                 seen_datasets.add(row["dataset"])
+#         dataset_df = pd.DataFrame(filtered_dataset_df_dict)
+#         dataset_df = dataset_df.sample(min(100, len(dataset_df)))
+
+#     # Make the leaderboard
+#     gb = GridOptionsBuilder.from_dataframe(dataset_df)
+#     gb.configure_default_column(sortable=False)
+#     gb.configure_column(
+#         "model_id",
+#         cellRenderer=JsCode('''function(params) {return '<a target="_blank" href="https://huggingface.co/'+params.value+'">'+params.value+'</a>'}'''),
+#     )
+#     if dataset == "-any-":
+#         gb.configure_column(
+#             "dataset",
+#             cellRenderer=JsCode('''function(params) {return '<a target="_blank" href="https://huggingface.co/spaces/autoevaluate/leaderboards?dataset='+params.value+'">'+params.value+'</a>'}'''),
+#         )
+#     for name in selectable_metrics:
+#         gb.configure_column(name, type=["numericColumn","numberColumnFilter","customNumericFormat"], precision=4, aggFunc='sum')
+
+#     gb.configure_column(
+#         sorting_metric,
+#         sortable=True,
+#         cellStyle=JsCode('''function(params) { return {'backgroundColor': '#FFD21E'}}''')
+#     )
+
+#     go = gb.build()
+#     fit_columns = len(dataset_df.columns) < 10
+#     AgGrid(dataset_df, gridOptions=go, height=28*len(dataset_df) + (35 if fit_columns else 41), allow_unsafe_jscode=True, fit_columns_on_grid_load=fit_columns, enable_enterprise_modules=False)
+
+# else:
+#     st.markdown(
+#         "No " + ("verified" if only_verified_results else "unverified") + " results to display. Try toggling the verified results filter."
+#     )