add new search.

app.py

@@ -25,6 +25,7 @@ from src.display.utils import (
     NUMERIC_INTERVALS,
     TYPES,
     AutoEvalColumn,
+    GroupDtype,
     ModelType,
     fields,
     WeightType,
@@ -105,17 +106,27 @@ def update_table(
     type_query: list,
     precision_query: str,
     size_query: list,
+    params_query: list,
     hide_models: list,
     query: str,
     compute_dtype: str,
     weight_dtype: str,
-    double_quant: str
+    double_quant: str,
+    group_dtype: str
 ):
 
     compute_dtype = [compute_dtype]
     weight_dtype = [weight_dtype]
+    if group_dtype == 'All':
+        group_dtype = [-1, 1024, 256, 128, 32]
+    else:
+        try:
+            group_dtype = [int(group_dtype)]
+        except ValueError:
+            group_dtype = [-1]
+
     double_quant = [str_to_bool(double_quant)]
-    filtered_df = filter_models(df=hidden_df, type_query=type_query, size_query=size_query, precision_query=precision_query, hide_models=hide_models, compute_dtype=compute_dtype, weight_dtype=weight_dtype, double_quant=double_quant)
+    filtered_df = filter_models(df=hidden_df, type_query=type_query, size_query=size_query, precision_query=precision_query, hide_models=hide_models, compute_dtype=compute_dtype, weight_dtype=weight_dtype, double_quant=double_quant, group_dtype=group_dtype, params_query=params_query)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns)
     return df
@@ -161,8 +172,8 @@ def filter_queries(query: str, filtered_df: pd.DataFrame):
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, hide_models: list, compute_dtype: list, weight_dtype: list, double_quant: list
-) -> pd.DataFrame:
+    df: pd.DataFrame, type_query: list, size_query: list, params_query:list, precision_query: list, hide_models: list, compute_dtype: list, weight_dtype: list, double_quant: list, group_dtype: list,
+ ) -> pd.DataFrame:
     # Show all models
     if "Private or deleted" in hide_models:
         filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
@@ -185,24 +196,31 @@ def filter_models(
     filtered_df = filtered_df.loc[df[AutoEvalColumn.weight_dtype.name].isin(weight_dtype)]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.compute_dtype.name].isin(compute_dtype)]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.double_quant.name].isin(double_quant)]
+    filtered_df = filtered_df.loc[df[AutoEvalColumn.group_size.name].isin(group_dtype)]
 
     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
     mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
     filtered_df = filtered_df.loc[mask]
 
+    numeric_interval_params = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in params_query]))
+    params_column_params = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+    mask_params = params_column_params.apply(lambda x: any(numeric_interval_params.contains(x)))
+    filtered_df = filtered_df.loc[mask_params]
+
     return filtered_df
 
 leaderboard_df = filter_models(
     df=leaderboard_df, 
     type_query=[t.to_str(" : ") for t in QuantType], 
     size_query=list(NUMERIC_INTERVALS.keys()), 
+    params_query=list(NUMERIC_INTERVALS.keys()),
     precision_query=[i.value.name for i in Precision],
     hide_models=["Private or deleted", "Contains a merge/moerge", "Flagged"], # Deleted, merges, flagged, MoEs,
     compute_dtype=[i.value.name for i in ComputeDtype],
     weight_dtype=[i.value.name for i in WeightDtype],
-    double_quant=[True, False]
-    
+    double_quant=[True, False],
+    group_dtype=[-1, 1024, 256, 128, 32]
 )
 
 demo = gr.Blocks(css=custom_css)
@@ -236,9 +254,18 @@ with demo:
                             elem_id="column-select",
                             interactive=True,
                         )
+
+                    with gr.Row():
+                        filter_columns_parameters = gr.CheckboxGroup(
+                        label="Model parameters (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
                     with gr.Row():
                         filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
+                        label="Model sizes (GB, int4)",
                         choices=list(NUMERIC_INTERVALS.keys()),
                         value=list(NUMERIC_INTERVALS.keys()),
                         interactive=True,
@@ -266,8 +293,7 @@ with demo:
                             filter_columns_computeDtype = gr.Dropdown(choices=[i.value.name for i in ComputeDtype], label="Compute Dtype", multiselect=False, value="float16", interactive=True,)
                             filter_columns_weightDtype = gr.Dropdown(choices=[i.value.name for i in WeightDtype], label="Weight Dtype", multiselect=False, value="int4", interactive=True,)
                             filter_columns_doubleQuant = gr.Dropdown(choices=["True", "False"], label="Double Quant", multiselect=False, value=False, interactive=True)
-                    # with gr.Row():
-                    #     gr.Checkbox(label="", info=""),
+                            filter_columns_groupDtype = gr.Dropdown(choices=[i.value.name for i in GroupDtype], label="Group Size", multiselect=False, value="All", interactive=True,)
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
@@ -308,11 +334,13 @@ with demo:
                     filter_columns_type,
                     filter_columns_precision,
                     filter_columns_size,
+                    filter_columns_parameters,
                     hide_models,
                     search_bar,
                     filter_columns_computeDtype,
                     filter_columns_weightDtype,
-                    filter_columns_doubleQuant
+                    filter_columns_doubleQuant,
+                    filter_columns_groupDtype
                 ],
                 leaderboard_table,
             )
@@ -341,7 +369,7 @@ with demo:
             demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
             
             """
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, hide_models, filter_columns_computeDtype, filter_columns_weightDtype, filter_columns_doubleQuant]:
+            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, filter_columns_parameters, hide_models, filter_columns_computeDtype, filter_columns_weightDtype, filter_columns_doubleQuant, filter_columns_groupDtype]:
                 selector.change(
                     update_table,
                     [
@@ -350,11 +378,13 @@ with demo:
                         filter_columns_type,
                         filter_columns_precision,
                         filter_columns_size,
+                        filter_columns_parameters,
                         hide_models,
                         search_bar,
                         filter_columns_computeDtype,
                         filter_columns_weightDtype,
-                        filter_columns_doubleQuant
+                        filter_columns_doubleQuant,
+                        filter_columns_groupDtype
                     ],
                     leaderboard_table,
                     queue=True,

src/display/about.py

@@ -59,10 +59,10 @@ python main.py --model=hf-causal-experimental
 - ARC-C: 0-shot, *arc_challenge* (`acc`)
 - ARC-E: 0-shot, *arc_easy* (`acc`)
 - HellaSwag: 0-shot, *hellaswag* (`acc`)
-- TruthfulQA: 0-shot, *truthfulqa_mc2* (`acc`)
+- TruthfulQA(Truthfulqa_mc1): 0-shot, *truthfulqa_mc1* (`acc`)
 - MMLU: 0-shot, *hendrycksTest-abstract_algebra,hendrycksTest-anatomy,hendrycksTest-astronomy,hendrycksTest-business_ethics,hendrycksTest-clinical_knowledge,hendrycksTest-college_biology,hendrycksTest-college_chemistry,hendrycksTest-college_computer_science,hendrycksTest-college_mathematics,hendrycksTest-college_medicine,hendrycksTest-college_physics,hendrycksTest-computer_security,hendrycksTest-conceptual_physics,hendrycksTest-econometrics,hendrycksTest-electrical_engineering,hendrycksTest-elementary_mathematics,hendrycksTest-formal_logic,hendrycksTest-global_facts,hendrycksTest-high_school_biology,hendrycksTest-high_school_chemistry,hendrycksTest-high_school_computer_science,hendrycksTest-high_school_european_history,hendrycksTest-high_school_geography,hendrycksTest-high_school_government_and_politics,hendrycksTest-high_school_macroeconomics,hendrycksTest-high_school_mathematics,hendrycksTest-high_school_microeconomics,hendrycksTest-high_school_physics,hendrycksTest-high_school_psychology,hendrycksTest-high_school_statistics,hendrycksTest-high_school_us_history,hendrycksTest-high_school_world_history,hendrycksTest-human_aging,hendrycksTest-human_sexuality,hendrycksTest-international_law,hendrycksTest-jurisprudence,hendrycksTest-logical_fallacies,hendrycksTest-machine_learning,hendrycksTest-management,hendrycksTest-marketing,hendrycksTest-medical_genetics,hendrycksTest-miscellaneous,hendrycksTest-moral_disputes,hendrycksTest-moral_scenarios,hendrycksTest-nutrition,hendrycksTest-philosophy,hendrycksTest-prehistory,hendrycksTest-professional_accounting,hendrycksTest-professional_law,hendrycksTest-professional_medicine,hendrycksTest-professional_psychology,hendrycksTest-public_relations,hendrycksTest-security_studies,hendrycksTest-sociology,hendrycksTest-us_foreign_policy,hendrycksTest-virology,hendrycksTest-world_religions* (average of all the results `acc`)
 - Winogrande: 0-shot, *winogrande* (`acc`)
-- Lambada_Openai: 0-shot, *lambada_openai* (`acc`)
+- Lambada(Lambada_Openai): 0-shot, *lambada_openai* (`acc`)
 - PIQA: 0-shot, *piqa* (`acc`)
 - OpenBookQA: 0-shot, *openbookqa* (`acc`)
 - BoolQ: 0-shot, *boolq* (`acc`)

src/display/utils.py

@@ -18,12 +18,12 @@ class Tasks(Enum):
     arc_easy = Task("arc:easy", "acc,none", "ARC-e")
     boolq = Task("boolq", "acc,none", "Boolq")
     hellaswag = Task("hellaswag", "acc,none", "HellaSwag")
-    lambada_openai = Task("lambada:openai", "acc,none", "Lambada_openai")
+    lambada_openai = Task("lambada:openai", "acc,none", "Lambada")
     mmlu = Task("mmlu", "acc,none", "MMLU")
     openbookqa = Task("openbookqa", "acc,none", "Openbookqa")
     piqa = Task("piqa", "acc,none", "Piqa")
     # truthfulqa:mc1 / truthfulqa:mc2 -- ?
-    truthfulqa_mc = Task("truthfulqa:mc1", "acc,none", "Truthfulqa_mc1")
+    truthfulqa_mc = Task("truthfulqa:mc1", "acc,none", "Truthfulqa")
     # arc:challenge ?
     # arc_challenge = Task("arc:challenge", "acc_norm,none", "Arc challenge")
     # truthfulqa = Task("truthfulqa:mc", "mc2", "TruthfulQA")
@@ -50,6 +50,8 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", True)])
+auto_eval_column_dict.append(["model_size", ColumnContent, ColumnContent("#Size (G)", "number", True)])
 # Dummy column for the search bar (hidden by the custom CSS)
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 # Model information
@@ -62,13 +64,14 @@ auto_eval_column_dict.append(["weight_dtype", ColumnContent, ColumnContent("Weig
 auto_eval_column_dict.append(["compute_dtype", ColumnContent, ColumnContent("Compute dtype", "str", False)])
 auto_eval_column_dict.append(["merged", ColumnContent, ColumnContent("Merged", "bool", False)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, hidden=True)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
 auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
 auto_eval_column_dict.append(["double_quant", ColumnContent, ColumnContent("Double Quant", "bool", False)])
+auto_eval_column_dict.append(["group_size", ColumnContent, ColumnContent("Group Size", "bool", False)])
 # We use make dataclass to dynamically fill the scores from Tasks
 # auto_eval_column_dict.sort(key=lambda x: x[0])
 sorted_columns = sorted(auto_eval_column_dict[3:], key=lambda x: x[0])
@@ -258,6 +261,28 @@ class ComputeDtype(Enum):
         if compute_dtype in ["float32"]:
             return ComputeDtype.fp32
         return ComputeDtype.Unknown
+    
+class GroupDtype(Enum):
+    group_1 = ModelDetails("-1")
+    group_1024 = ModelDetails("1024")
+    group_256 = ModelDetails("256")
+    group_128 = ModelDetails("128")
+    group_32 = ModelDetails("32")
+
+    group_all = ModelDetails("All")
+
+    def from_str(compute_dtype):
+        if compute_dtype in ["-1"]:
+            return GroupDtype.group_1
+        if compute_dtype in ["1024"]:
+            return GroupDtype.group_1024
+        if compute_dtype in ["256"]:
+            return GroupDtype.group_256
+        if compute_dtype in ["128"]:
+            return GroupDtype.group_128
+        if compute_dtype in ["32"]:
+            return GroupDtype.group_32       
+        return GroupDtype.group_all    
 
 class Precision(Enum):
     # float16 = ModelDetails("float16")

src/leaderboard/read_evals.py

@@ -33,6 +33,8 @@ class EvalResult:
     license: str = "?"
     likes: int = 0
     num_params: int = 0
+    model_size: int = 0
+    group_size: int = -1
     date: str = "" # submission date of request file
     still_on_hub: bool = True
     is_merge: bool = False
@@ -57,6 +59,8 @@ class EvalResult:
         compute_dtype = ComputeDtype.from_str(data["task_info"].get("compute_dtype", "bfloat16"))
         double_quant = data["quantization_config"].get("bnb_4bit_use_double_quant", False)
         model_params = config["model_params"]
+        model_size = config["model_size"]
+        group_size = data["quantization_config"].get("group_size", -1)
 
         local = config.get("local", False)
         if not local:
@@ -109,6 +113,8 @@ class EvalResult:
             double_quant=double_quant,
             revision=config.get("model_sha", "main"),
             num_params=model_params,
+            model_size=model_size,
+            group_size=group_size
         )
 
     def update_with_request_file(self, requests_path):
@@ -160,6 +166,8 @@ class EvalResult:
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.model_size.name: self.model_size,
+            AutoEvalColumn.group_size.name: self.group_size,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
             AutoEvalColumn.merged.name: "merge" in self.tags if self.tags else False,
             AutoEvalColumn.moe.name: ("moe" in self.tags if self.tags else False) or "moe" in self.full_model.lower(),