Display first-timer-friendly models

leaderboard/src/leaderboard/__init__.py

@@ -1,7 +1,7 @@
 """Transformers.js Benchmark Leaderboard"""
 
 from .app import create_leaderboard_ui
-from .data_loader import load_benchmark_data, get_unique_values, flatten_result
+from .data_loader import load_benchmark_data, get_unique_values, flatten_result, get_first_timer_friendly_models
 from .formatters import apply_formatting
 
 __version__ = "0.1.0"
@@ -10,5 +10,6 @@ __all__ = [
     "load_benchmark_data",
     "get_unique_values",
     "flatten_result",
+    "get_first_timer_friendly_models",
     "apply_formatting",
 ]

leaderboard/src/leaderboard/app.py

@@ -13,6 +13,7 @@ from dotenv import load_dotenv
 from leaderboard.data_loader import (
     load_benchmark_data,
     get_unique_values,
+    get_first_timer_friendly_models,
 )
 from leaderboard.formatters import apply_formatting
 
@@ -37,13 +38,17 @@ def load_data() -> pd.DataFrame:
         token=HF_TOKEN,
     )
 
-    # Apply formatting to each row
-    if not df.empty:
-        df = df.apply(lambda row: pd.Series(apply_formatting(row.to_dict())), axis=1)
-
     return df
 
 
+def format_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """Apply formatting to dataframe for display."""
+    if df.empty:
+        return df
+
+    return df.apply(lambda row: pd.Series(apply_formatting(row.to_dict())), axis=1)
+
+
 def filter_data(
     df: pd.DataFrame,
     model_filter: str,
@@ -93,6 +98,7 @@ def create_leaderboard_ui():
 
     # Load initial data
     df = load_data()
+    formatted_df = format_dataframe(df)
 
     with gr.Blocks(title="Transformers.js Benchmark Leaderboard") as demo:
         gr.Markdown("# 🏆 Transformers.js Benchmark Leaderboard")
@@ -106,6 +112,24 @@ def create_leaderboard_ui():
                 "Please set the environment variable to load benchmark data."
             )
 
+        # First-timer-friendly models section
+        with gr.Accordion("✨ First-Timer-Friendly Models", open=True):
+            gr.Markdown(
+                "These models are great for first-timers! They're popular, fast to load, "
+                "and quick to run. Perfect for getting started with Transformers.js.\n\n"
+                "**Showing top 3 models per task type.**"
+            )
+
+            first_timer_models = get_first_timer_friendly_models(df, limit_per_task=3)
+            formatted_first_timer = format_dataframe(first_timer_models)
+
+            first_timer_table = gr.DataFrame(
+                value=formatted_first_timer,
+                label="Top First-Timer-Friendly Models (by Task)",
+                interactive=False,
+                wrap=True,
+            )
+
         with gr.Row():
             refresh_btn = gr.Button("🔄 Refresh Data", variant="primary")
 
@@ -145,8 +169,8 @@ def create_leaderboard_ui():
             )
 
         results_table = gr.DataFrame(
-            value=df,
-            label="Benchmark Results",
+            value=formatted_df,
+            label="All Benchmark Results",
             interactive=False,
             wrap=True,
         )
@@ -166,8 +190,15 @@ def create_leaderboard_ui():
         def update_data():
             """Reload data from HuggingFace."""
             new_df = load_data()
+            formatted_new_df = format_dataframe(new_df)
+
+            # Update first-timer-friendly models (3 per task)
+            new_first_timer = get_first_timer_friendly_models(new_df, limit_per_task=3)
+            formatted_new_first_timer = format_dataframe(new_first_timer)
+
             return (
-                new_df,
+                formatted_new_first_timer,
+                formatted_new_df,
                 gr.update(choices=get_unique_values(new_df, "task")),
                 gr.update(choices=get_unique_values(new_df, "platform")),
                 gr.update(choices=get_unique_values(new_df, "device")),
@@ -175,14 +206,18 @@ def create_leaderboard_ui():
                 gr.update(choices=get_unique_values(new_df, "dtype")),
             )
 
-        def apply_filters(df, model, task, platform, device, mode, dtype):
+        def apply_filters(formatted_df, model, task, platform, device, mode, dtype):
             """Apply filters and return filtered DataFrame."""
-            return filter_data(df, model, task, platform, device, mode, dtype)
+            # Need to reload raw data to filter, then format
+            raw_df = load_data()
+            filtered = filter_data(raw_df, model, task, platform, device, mode, dtype)
+            return format_dataframe(filtered)
 
         # Refresh button updates data and resets filters
         refresh_btn.click(
             fn=update_data,
             outputs=[
+                first_timer_table,
                 results_table,
                 task_filter,
                 platform_filter,

leaderboard/src/leaderboard/data_loader.py

@@ -226,6 +226,109 @@ def enrich_with_hf_metadata(df: pd.DataFrame) -> pd.DataFrame:
     return df
 
 
+def get_first_timer_friendly_models(df: pd.DataFrame, limit_per_task: int = 3) -> pd.DataFrame:
+    """Identify first-timer-friendly models based on popularity and performance, grouped by task.
+
+    A model is considered first-timer-friendly if it:
+    - Has high downloads (popular)
+    - Has fast load times (easy to start)
+    - Has fast inference times (quick results)
+    - Successfully completed benchmarks
+
+    Args:
+        df: DataFrame containing benchmark results
+        limit_per_task: Maximum number of models to return per task
+
+    Returns:
+        DataFrame with top first-timer-friendly models per task
+    """
+    if df.empty:
+        return pd.DataFrame()
+
+    # Filter only successful benchmarks
+    filtered = df[df["status"] == "completed"].copy() if "status" in df.columns else df.copy()
+
+    if filtered.empty:
+        return pd.DataFrame()
+
+    # Check if task column exists
+    if "task" not in filtered.columns:
+        logger.warning("Task column not found in dataframe")
+        return pd.DataFrame()
+
+    # Calculate first-timer-friendliness score per task
+    all_results = []
+
+    for task in filtered["task"].unique():
+        task_df = filtered[filtered["task"] == task].copy()
+
+        if task_df.empty:
+            continue
+
+        # Normalize metrics within this task (lower is better for times, higher is better for popularity)
+
+        # Downloads score (0-1, higher is better)
+        if "downloads" in task_df.columns:
+            max_downloads = task_df["downloads"].max()
+            task_df["downloads_score"] = task_df["downloads"] / max_downloads if max_downloads > 0 else 0
+        else:
+            task_df["downloads_score"] = 0
+
+        # Likes score (0-1, higher is better)
+        if "likes" in task_df.columns:
+            max_likes = task_df["likes"].max()
+            task_df["likes_score"] = task_df["likes"] / max_likes if max_likes > 0 else 0
+        else:
+            task_df["likes_score"] = 0
+
+        # Load time score (0-1, lower time is better)
+        if "load_ms_p50" in task_df.columns:
+            max_load = task_df["load_ms_p50"].max()
+            task_df["load_score"] = 1 - (task_df["load_ms_p50"] / max_load) if max_load > 0 else 0
+        else:
+            task_df["load_score"] = 0
+
+        # Inference time score (0-1, lower time is better)
+        if "first_infer_ms_p50" in task_df.columns:
+            max_infer = task_df["first_infer_ms_p50"].max()
+            task_df["infer_score"] = 1 - (task_df["first_infer_ms_p50"] / max_infer) if max_infer > 0 else 0
+        else:
+            task_df["infer_score"] = 0
+
+        # Calculate weighted first-timer-friendliness score
+        # Weights: popularity (40%), load time (30%), inference time (30%)
+        task_df["first_timer_score"] = (
+            (task_df["downloads_score"] * 0.25) +
+            (task_df["likes_score"] * 0.15) +
+            (task_df["load_score"] * 0.30) +
+            (task_df["infer_score"] * 0.30)
+        )
+
+        # Group by model and take best score for each model within this task
+        best_per_model = task_df.loc[task_df.groupby("modelId")["first_timer_score"].idxmax()]
+
+        # Sort by first-timer score and take top N for this task
+        top_for_task = best_per_model.sort_values("first_timer_score", ascending=False).head(limit_per_task)
+
+        # Drop intermediate scoring columns
+        score_cols = ["downloads_score", "likes_score", "load_score", "infer_score", "first_timer_score"]
+        top_for_task = top_for_task.drop(columns=[col for col in score_cols if col in top_for_task.columns])
+
+        all_results.append(top_for_task)
+
+    if not all_results:
+        return pd.DataFrame()
+
+    # Combine all results
+    result = pd.concat(all_results, ignore_index=True)
+
+    # Sort by task name for better organization
+    if "task" in result.columns:
+        result = result.sort_values("task")
+
+    return result
+
+
 def get_unique_values(df: pd.DataFrame, column: str) -> List[str]:
     """Get unique values from a column for dropdown choices.