add top model bar graph

app.py

@@ -54,7 +54,8 @@ from src.tools.plots import (
     create_metric_plot_obj,
     create_plot_df,
     create_scores_df,
-    create_lat_score_mem_plot_obj
+    create_lat_score_mem_plot_obj,
+    create_top_n_models_comparison_plot
 )
 
 # Start ephemeral Spaces on PRs (see config in README.md)
@@ -380,6 +381,25 @@ with demo:
                 )
 
         with gr.TabItem("📈 Metrics", elem_id="llm-benchmark-tab-table", id=4):
+            with gr.Row():
+                with gr.Column():
+                    size_filter = gr.Dropdown(
+                        choices=["All Sizes"] + list(NUMERIC_INTERVALS.keys()),
+                        label="Filter by Model Size",
+                        value="All Sizes",
+                        interactive=True
+                    )
+                    fig = create_top_n_models_comparison_plot(leaderboard_df, top_n=5)
+                    top_n_plot = gr.components.Plot(value=fig, show_label=False)
+                    
+                    def update_top_n_plot(size_option):
+                        return create_top_n_models_comparison_plot(leaderboard_df, top_n=5, size_filter=size_option)
+                    
+                    size_filter.change(
+                        fn=update_top_n_plot,
+                        inputs=[size_filter],
+                        outputs=[top_n_plot]
+                    )
             with gr.Row():
                 with gr.Column():
                     chart = create_metric_plot_obj(

src/display/formatting.py

@@ -48,6 +48,21 @@ def make_clickable_model(model_name, json_path=None, revision=None, precision=No
 
     return model_hyperlink(link, model_name) + "  " + model_hyperlink(details_link, "📑") + " " + posfix
 
+def make_dummy_name(model_name, revision=None, precision=None, num_evals_same_model=1):
+    posfix = ""
+    if revision is not None and revision != "" and revision != "main":
+        if len(revision) > 12:
+            revision = revision[:7]
+        posfix += f" (rev: {revision})"
+    if precision is not None:
+        if num_evals_same_model == 1 and precision in ['float16', 'bfloat16']:
+            pass
+        else:
+            #if precision not in model_name:
+            posfix += f" [{precision}]"
+    posfix = posfix.strip()
+    return f"{model_name} {posfix}"
+
 
 def styled_error(error):
     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"

src/display/utils.py

@@ -256,13 +256,15 @@ class Language(Enum):
 
 #External models
 external_rows = []
+external_eval_results = []  # Initialize the list to store EvalResult objects
 if os.path.exists('external_models_results.json'):
     with open('external_models_results.json', 'r', encoding='utf8') as f:
         all_models = json.load(f)
     for model_data in all_models:
+        #Create external_rows
         model_row = deepcopy(baseline_row)
         model_row[AutoEvalColumn.model.name] = f'<a target="_blank" href="{model_data["link"]}" style="color: var(--text-color); text-decoration: underline;text-decoration-style: dotted;">{model_data["name"]} [{model_data["date"]}]</a>'
-        model_row[AutoEvalColumn.dummy.name] = model_data['model']
+        model_row[AutoEvalColumn.dummy.name] = model_data['name']
         for task in Tasks:
             model_row[task.value.col_name] = round(model_data['result_metrics'][task.value.benchmark]*100, 2)
         model_row[AutoEvalColumn.average.name] = round(model_data['result_metrics_average']*100, 2)
@@ -277,8 +279,81 @@ if os.path.exists('external_models_results.json'):
             model_row[AutoEvalColumn.params.name] = model_data['params']
 
         model_row[AutoEvalColumn.main_language.name] = model_data['main_language']
+        #convert 2025-04-03 to 2025-04-03T00:00:00Z
         external_rows.append(model_row)
 
+        #Create external_eval_results
+        eval_result = dict(
+            eval_name=f"external_{model_data['model']}",
+            full_model=model_data['name'],
+            org="External",  # External models don't have an org in this context
+            model=model_data['name'],
+            # Scale results by 100 to match expected format
+            results={k: v * 100 for k, v in model_data['result_metrics'].items()},
+            model_sha="",  # Not available
+            revision="main", # Default
+            precision=Precision.Unknown, # Not available
+            model_type=model_type, # Already determined above
+            weight_type=WeightType.Original, # Assuming original weights
+            main_language=model_data['main_language'],
+            architecture="Unknown", # Not available
+            license="Proprietary" if model_type == ModelType.proprietary else "?",
+            likes=0, # Not available
+            num_params=model_data.get('params', 0), # Use .get() for safety
+            date=model_data['date']+"T00:00:00Z",
+            still_on_hub=True, # Not applicable
+            is_merge=False, # Not applicable
+            flagged=False, # Not applicable
+            status="FINISHED",
+            tags=None, # Not available
+            json_filename='external_models_results.json', # Not applicable
+            eval_time=0.0, # Not available
+            # Scale average by 100
+            original_benchmark_average=None,#model_data.get('result_metrics_average', 0.0) * 100,
+            hidden=False, # Default
+            num_evals_model_rev=1 # Default
+        )
+        """
+        EvalResult(eval_name='01-ai_Yi-1.5-34B_bfloat16', 
+        '   full_model='01-ai/Yi-1.5-34B', 
+        '   org='01-ai', 
+        '   model='Yi-1.5-34B', 
+        '   results={'enem_challenge': 71.51854443666899, 
+        '   'bluex': 66.62030598052851, 
+        '   'oab_exams': 54.89749430523918, 
+        '   'assin2_rte': 89.76911637262349, 
+        '   'assin2_sts': 81.48786802023537, 
+        '   'faquad_nli': 58.5644163957417, 
+        '   'hatebr_offensive': 83.63023241432246, 
+        '   'portuguese_hate_speech': 69.62399848962205, 
+        '   'tweetsentbr': 72.28749707523902}, 
+        '   model_sha='81136a42efdf6f6a63031ac31639a37813fe6e37', 
+        '   revision='main', 
+        '   precision=<Precision.bfloat16: ModelDetails(name='bfloat16', 
+        '   symbol='')>, 
+        '   model_type=<ModelType.PT: ModelDetails(name='pretrained', 
+        '   symbol='🟢')>, 
+        '   weight_type=<WeightType.Original: ModelDetails(name='Original', 
+        '   symbol='')>, 
+        '   main_language='English', 
+        '   architecture='LlamaForCausalLM', 
+        '   license='?', 
+        '   likes=0, 
+        '   num_params=34.39, 
+        '   date='2024-05-15T17:40:15Z', 
+        '   still_on_hub=True, 
+        '   is_merge=False, 
+        '   flagged=False, 
+        '   status='FINISHED', 
+        '   tags=None, 
+        '   json_filename='results_2024-05-17T10-36-18.336343.json', 
+        '   eval_time=11545.340715408325, 
+        '   original_benchmark_average=None, 
+        '   hidden=False, 
+        '   num_evals_model_rev=1)
+        """
+        external_eval_results.append(eval_result)
+
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn)]

src/leaderboard/read_evals.py

@@ -11,7 +11,7 @@ import numpy as np
 
 from huggingface_hub import ModelCard
 
-from src.display.formatting import make_clickable_model
+from src.display.formatting import make_clickable_model, make_dummy_name
 from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, Language, WeightType, ORIGINAL_TASKS
 from src.envs import GET_ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS, SHOW_INCOMPLETE_EVALS
 
@@ -191,7 +191,7 @@ class EvalResult:
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model, self.json_filename, revision=self.revision, precision=self.precision.value.name, num_evals_same_model=self.num_evals_model_rev),
-            AutoEvalColumn.dummy.name: self.full_model,
+            AutoEvalColumn.dummy.name: make_dummy_name(self.full_model, revision=self.revision, precision=self.precision.value.name, num_evals_same_model=self.num_evals_model_rev),
             AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
             AutoEvalColumn.license.name: self.license,

src/tools/plots.py

@@ -4,8 +4,9 @@ import plotly.express as px
 from plotly.graph_objs import Figure
 
 from src.leaderboard.filter_models import FLAGGED_MODELS
-from src.display.utils import human_baseline_row as HUMAN_BASELINE, AutoEvalColumn, Tasks, Task, BENCHMARK_COLS
+from src.display.utils import human_baseline_row as HUMAN_BASELINE, AutoEvalColumn, Tasks, Task, BENCHMARK_COLS, external_eval_results, NUMERIC_INTERVALS
 from src.leaderboard.read_evals import EvalResult
+import copy
 
 
 
@@ -19,9 +20,16 @@ def create_scores_df(raw_data: list[EvalResult]) -> pd.DataFrame:
     # Step 1: Ensure 'date' is in datetime format and sort the DataFrame by it
 
     #create dataframe with EvalResult dataclass columns, even if raw_data is empty
+    raw_data = copy.deepcopy(raw_data)
+    for external_row in external_eval_results:
+        raw_data.append(EvalResult(**external_row))
     results_df = pd.DataFrame(raw_data, columns=EvalResult.__dataclass_fields__.keys())
 
     #results_df["date"] = pd.to_datetime(results_df["date"], format="mixed", utc=True)
+    #convert date to datetime
+    results_df["date"] = pd.to_datetime(results_df["date"], format="mixed", utc=True)
+    #convert to simple date string 2025-04-26
+    results_df["date"] = results_df["date"].dt.strftime("%Y-%m-%d")
     results_df.sort_values(by="date", inplace=True)
 
     # Step 2: Initialize the scores dictionary
@@ -129,7 +137,7 @@ def create_metric_plot_obj(
     )
 
     # Update the range of the y-axis
-    fig.update_layout(yaxis_range=[0, 100])
+    #fig.update_layout(yaxis_range=[0, 100])
 
     # Create a dictionary to hold the color mapping for each metric
     metric_color_mapping = {}
@@ -212,6 +220,222 @@ def create_lat_score_mem_plot_obj(leaderboard_df):
 
     return fig
 
-# Example Usage:
-# human_baselines dictionary is defined.
-# chart = create_metric_plot_obj(scores_df, ["ARC", "HellaSwag", "MMLU", "TruthfulQA"], human_baselines, "Graph Title")
+def create_top_n_models_comparison_plot(leaderboard_df: pd.DataFrame, top_n: int = 5, size_filter: str = None) -> Figure:
+    """
+    Creates a grouped bar chart comparing the performance of the top N models across all metrics.
+
+    :param leaderboard_df: DataFrame containing the leaderboard data.
+    :param top_n: The number of top models to include in the comparison (default is 5).
+    :param size_filter: If provided, only include models of this specific size category.
+    :return: A Plotly figure object representing the comparison plot.
+    """
+    # Ensure BENCHMARK_COLS contains the correct metric column names
+    metric_cols = BENCHMARK_COLS
+
+    # Filter out non-model rows (like baseline or human) and select relevant columns
+    models_df = leaderboard_df[~leaderboard_df[AutoEvalColumn.dummy.name].isin(["baseline", "human_baseline"])].copy()
+    
+    # Add size group information to the DataFrame
+    models_df['size_group'] = models_df[AutoEvalColumn.params.name].apply(
+        lambda x: next((k for k, v in NUMERIC_INTERVALS.items() if x in v), '?')
+    )
+    
+    # Filter by size category if specified
+    if size_filter and size_filter != 'All Sizes':
+        models_df = models_df[models_df['size_group'] == size_filter]
+        if models_df.empty:
+            # If no models match the size filter, return an empty figure with a message
+            fig = px.bar(
+                x=["No Data"],
+                y=[0],
+                title=f"No models found in the {size_filter} size category"
+            )
+            fig.update_layout(
+                xaxis_title="",
+                yaxis_title="",
+                showlegend=False
+            )
+            return fig
+    
+    # Sort models by average score and select the top N
+    top_models_df = models_df.nlargest(top_n, AutoEvalColumn.average.name)
+
+    # Select only the necessary columns: model name and metric scores
+    plot_data = top_models_df[[AutoEvalColumn.dummy.name] + metric_cols]
+
+    # Melt the DataFrame to long format suitable for plotting
+    # 'id_vars' specifies the column(s) to keep as identifiers
+    # 'value_vars' specifies the columns to unpivot
+    # 'var_name' is the name for the new column containing the original column names (metrics)
+    # 'value_name' is the name for the new column containing the values (scores)
+    melted_df = pd.melt(
+        plot_data,
+        id_vars=[AutoEvalColumn.dummy.name],
+        value_vars=metric_cols,
+        var_name="Metric",
+        value_name="Score",
+    )
+    
+    # Validate and cap scores to ensure they're within a reasonable range (0-100)
+    melted_df['Score'] = melted_df['Score'].apply(lambda x: min(max(x, 0), 100))
+    
+    # Create the grouped bar chart
+    fig = px.bar(
+        melted_df,
+        x="Metric",
+        y="Score",
+        color=AutoEvalColumn.dummy.name,  # Group bars by model name
+        barmode="group",  # Display bars side-by-side for each metric
+        title=f"Top {top_n} Models Comparison Across Metrics",
+        labels={AutoEvalColumn.dummy.name: "Model"}, # Rename legend title
+        custom_data=[AutoEvalColumn.dummy.name, "Metric", "Score"], # Data for hover
+        range_y=[0, 100],  # Force y-axis range to be 0-100
+    )
+
+    # Update hovertemplate
+    fig.update_traces(
+        hovertemplate="<br>".join(
+            [
+                "Model: %{customdata[0]}",
+                "Metric: %{customdata[1]}",
+                "Score: %{customdata[2]:.2f}", # Format score to 2 decimal places
+                "<extra></extra>", # Remove the default trace info
+            ]
+        )
+    )
+
+    # Create title with size filter information if applicable
+    title_text = f"Top {top_n} Models Comparison Across Metrics"
+    if size_filter and size_filter != 'All Sizes':
+        title_text += f" ({size_filter} Models)"
+    
+    # Calculate appropriate y-axis range based on the data
+    min_score = melted_df['Score'].min()
+    max_score = melted_df['Score'].max()
+    
+    # Set y-axis minimum (start at 0 unless all scores are high)
+    y_min = 40 if min_score > 50 else 0
+    
+    # Set y-axis maximum (ensure there's room for annotations)
+    y_max = 100 if max_score < 95 else 105
+    
+    # Optional: Adjust layout for better readability
+    fig.update_layout(
+        title={
+            "text": title_text,
+            "y": 0.95,
+            "x": 0.5,
+            "xanchor": "center",
+            "yanchor": "top",
+        },
+        xaxis_title="Metric",
+        yaxis_title="Score (%)",
+        legend_title="Model",
+        yaxis=dict(
+            range=[y_min, y_max],  # Set y-axis range dynamically
+            constrain="domain",  # Constrain the axis to the domain
+            constraintoward="top"  # Constrain toward the top
+        ),
+        width=1600,
+        height=450,
+    )
+    
+    # Define shape icons for each model
+    shape_icons = {
+        0: "triangle-up",    # First model gets triangle
+        1: "square",         # Second model gets square
+        2: "circle",         # Third model gets circle
+        3: "diamond",        # Fourth model gets diamond
+        4: "star",           # Fifth model gets star
+        5: "pentagon",       # Sixth model gets pentagon
+        6: "hexagon",        # Seventh model gets hexagon
+        7: "cross",          # Eighth model gets cross
+        8: "x",              # Ninth model gets x
+        9: "hourglass",      # Tenth model gets hourglass
+    }
+    
+    # Get the average score for each model
+    model_averages = {}
+    for model in top_models_df[AutoEvalColumn.dummy.name].unique():
+        try:
+            model_averages[model] = top_models_df.loc[top_models_df[AutoEvalColumn.dummy.name] == model, AutoEvalColumn.average.name].values[0]
+        except (IndexError, KeyError):
+            # If average score is not available, use None
+            model_averages[model] = None
+    
+    # Add shapes to the legend and annotations with icons for each bar
+    for i, bar in enumerate(fig.data):
+        model_name = bar.name
+        model_index = list(top_models_df[AutoEvalColumn.dummy.name].unique()).index(model_name) % len(shape_icons)
+        icon_shape = shape_icons[model_index]
+        
+        # Update the name in the legend to include the shape symbol
+        shape_symbol = get_symbol_for_shape(icon_shape)
+        fig.data[i].name = f"{shape_symbol} {model_name}"
+        
+        # For each bar in this trace
+        for j, (x, y) in enumerate(zip(bar.x, bar.y)):
+            # Use the actual bar score instead of the average
+            score_text = f"<b>{y:.1f}</b>"
+                
+            # Calculate the exact position for the annotation
+            # Plotly's grouped bar charts position bars at specific offsets
+            # We need to match these offsets exactly
+            num_models = len(top_models_df[AutoEvalColumn.dummy.name].unique())
+            
+            # The total width allocated for all bars in a group
+            total_group_width = 0.8
+            
+            # Width of each individual bar
+            bar_width = total_group_width / num_models
+            
+            # Calculate the offset for this specific bar within its group
+            # i represents which model in the group (0 is the first model, etc.)
+            # Center of the group is at x, so we need to adjust from there
+            offset = (i - (num_models-1)/2) * bar_width
+            
+            # Add score text directly above its bar
+            fig.add_annotation(
+                x=x,
+                y=y + 2,  # Position slightly above the bar
+                text=score_text,  # Display the actual bar score
+                showarrow=False,
+                font=dict(
+                    size=10,
+                    color=bar.marker.color  # Match the bar color
+                ),
+                opacity=0.9,
+                xshift=offset * 130  # Adjust the multiplier to better center the annotation
+            )
+            
+            # Add the shape icon above the score
+            fig.add_annotation(
+                x=x,
+                y=y - 3,  # Position above the score text
+                text=get_symbol_for_shape(icon_shape),  # Convert shape name to symbol
+                showarrow=False,
+                font=dict(
+                    size=14,
+                    color="black"  # Match the bar color
+                ),
+                opacity=0.9,
+                xshift=offset * 130  # Adjust the multiplier to better center the annotation
+            )
+
+    return fig
+
+def get_symbol_for_shape(shape_name):
+    """Convert shape name to a symbol character that can be used in annotations."""
+    symbols = {
+        "triangle-up": "▲",
+        "square": "■",
+        "circle": "●",
+        "diamond": "◆",
+        "star": "★",
+        "pentagon": "⬟",
+        "hexagon": "⬢",
+        "cross": "✚",
+        "x": "✖",
+        "hourglass": "⧗"
+    }
+    return symbols.get(shape_name, "●")  # Default to circle if shape not found