Create leaderboard

leaderboard/.env.example

@@ -0,0 +1,6 @@
+# HuggingFace Dataset Repository
+# The dataset repository where benchmark results are stored
+HF_DATASET_REPO=your-username/your-dataset-repo
+
+# HuggingFace API Token (optional, for private datasets)
+HF_TOKEN=your_token_here

leaderboard/.gitignore

@@ -0,0 +1,41 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+.venv/
+venv/
+ENV/
+env/
+
+# Environment variables
+.env
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db

leaderboard/.python-version

@@ -0,0 +1 @@
+3.13

leaderboard/README.md

@@ -0,0 +1,82 @@
+# Transformers.js Benchmark Leaderboard
+
+A Gradio-based leaderboard that displays benchmark results from a HuggingFace Dataset repository.
+
+## Features
+
+- 📊 Display benchmark results in a searchable/filterable table
+- 🔍 Filter by model name, task, platform, device, mode, and dtype
+- 🔄 Refresh data on demand from HuggingFace Dataset
+- 📈 View performance metrics (load time, inference time, p50/p90 percentiles)
+
+## Setup
+
+1. Install dependencies:
+   ```bash
+   uv sync
+   ```
+
+2. Configure environment variables:
+   ```bash
+   cp .env.example .env
+   ```
+
+   Edit `.env` and set:
+   - `HF_DATASET_REPO`: Your HuggingFace dataset repository (e.g., `username/transformersjs-benchmarks`)
+   - `HF_TOKEN`: Your HuggingFace API token (optional, for private datasets)
+
+## Usage
+
+Run the leaderboard:
+
+```bash
+uv run python -m leaderboard.app
+```
+
+Or using the installed script:
+
+```bash
+uv run leaderboard
+```
+
+The leaderboard will be available at: http://localhost:7861
+
+## Data Format
+
+The leaderboard reads JSONL files from the HuggingFace Dataset repository. Each line should be a JSON object with the following structure:
+
+```json
+{
+  "id": "benchmark-id",
+  "platform": "web",
+  "modelId": "Xenova/all-MiniLM-L6-v2",
+  "task": "feature-extraction",
+  "mode": "warm",
+  "repeats": 3,
+  "batchSize": 1,
+  "device": "wasm",
+  "browser": "chromium",
+  "dtype": "fp32",
+  "headed": false,
+  "status": "completed",
+  "timestamp": 1234567890,
+  "result": {
+    "metrics": {
+      "load_ms": {"p50": 100, "p90": 120},
+      "first_infer_ms": {"p50": 10, "p90": 15},
+      "subsequent_infer_ms": {"p50": 8, "p90": 12}
+    },
+    "environment": {
+      "cpuCores": 10,
+      "memory": {"deviceMemory": 8}
+    }
+  }
+}
+```
+
+## Development
+
+The leaderboard is built with:
+- **Gradio**: Web UI framework
+- **Pandas**: Data manipulation
+- **HuggingFace Hub**: Dataset loading

leaderboard/main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from leaderboard!")
+
+
+if __name__ == "__main__":
+    main()

leaderboard/pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "leaderboard"
+version = "0.1.0"
+description = "Transformers.js Benchmark Leaderboard - Display benchmark results from HuggingFace Dataset"
+requires-python = ">=3.13"
+dependencies = [
+    "gradio>=5.49.1",
+    "huggingface-hub>=0.35.3",
+    "pandas>=2.3.3",
+    "python-dotenv>=1.1.1",
+]
+
+[project.scripts]
+leaderboard = "leaderboard.app:create_leaderboard_ui"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/leaderboard"]
+
+[tool.uv]
+package = true

leaderboard/src/leaderboard/__init__.py

@@ -0,0 +1,12 @@
+"""Transformers.js Benchmark Leaderboard"""
+
+from .app import create_leaderboard_ui
+from .data_loader import load_benchmark_data, get_unique_values, flatten_result
+
+__version__ = "0.1.0"
+__all__ = [
+    "create_leaderboard_ui",
+    "load_benchmark_data",
+    "get_unique_values",
+    "flatten_result",
+]

leaderboard/src/leaderboard/app.py

@@ -0,0 +1,222 @@
+"""
+Transformers.js Benchmark Leaderboard
+
+A Gradio app that displays benchmark results from a HuggingFace Dataset repository.
+"""
+
+import os
+import pandas as pd
+import gradio as gr
+from dotenv import load_dotenv
+
+from leaderboard.data_loader import (
+    load_benchmark_data,
+    get_unique_values,
+)
+
+# Load environment variables
+load_dotenv()
+
+HF_DATASET_REPO = os.getenv("HF_DATASET_REPO")
+HF_TOKEN = os.getenv("HF_TOKEN")
+
+
+def load_data() -> pd.DataFrame:
+    """Load benchmark data from configured HF Dataset repository."""
+    return load_benchmark_data(
+        dataset_repo=HF_DATASET_REPO,
+        token=HF_TOKEN,
+    )
+
+
+def filter_data(
+    df: pd.DataFrame,
+    model_filter: str,
+    task_filter: str,
+    platform_filter: str,
+    device_filter: str,
+    mode_filter: str,
+    dtype_filter: str,
+) -> pd.DataFrame:
+    """Filter benchmark data based on user inputs."""
+    if df.empty:
+        return df
+
+    filtered = df.copy()
+
+    # Model name filter
+    if model_filter:
+        filtered = filtered[
+            filtered["modelId"].str.contains(model_filter, case=False, na=False)
+        ]
+
+    # Task filter
+    if task_filter and task_filter != "All":
+        filtered = filtered[filtered["task"] == task_filter]
+
+    # Platform filter
+    if platform_filter and platform_filter != "All":
+        filtered = filtered[filtered["platform"] == platform_filter]
+
+    # Device filter
+    if device_filter and device_filter != "All":
+        filtered = filtered[filtered["device"] == device_filter]
+
+    # Mode filter
+    if mode_filter and mode_filter != "All":
+        filtered = filtered[filtered["mode"] == mode_filter]
+
+    # DType filter
+    if dtype_filter and dtype_filter != "All":
+        filtered = filtered[filtered["dtype"] == dtype_filter]
+
+    return filtered
+
+
+def create_leaderboard_ui():
+    """Create the Gradio UI for the leaderboard."""
+
+    # Load initial data
+    df = load_data()
+
+    with gr.Blocks(title="Transformers.js Benchmark Leaderboard") as demo:
+        gr.Markdown("# 🏆 Transformers.js Benchmark Leaderboard")
+        gr.Markdown(
+            "Compare benchmark results for different models, platforms, and configurations."
+        )
+
+        if not HF_DATASET_REPO:
+            gr.Markdown(
+                "⚠️ **HF_DATASET_REPO not configured.** "
+                "Please set the environment variable to load benchmark data."
+            )
+
+        with gr.Row():
+            refresh_btn = gr.Button("🔄 Refresh Data", variant="primary")
+
+        with gr.Row():
+            model_filter = gr.Textbox(
+                label="Model Name",
+                placeholder="Filter by model name (e.g., 'bert', 'gpt')",
+            )
+            task_filter = gr.Dropdown(
+                label="Task",
+                choices=get_unique_values(df, "task"),
+                value="All",
+            )
+
+        with gr.Row():
+            platform_filter = gr.Dropdown(
+                label="Platform",
+                choices=get_unique_values(df, "platform"),
+                value="All",
+            )
+            device_filter = gr.Dropdown(
+                label="Device",
+                choices=get_unique_values(df, "device"),
+                value="All",
+            )
+
+        with gr.Row():
+            mode_filter = gr.Dropdown(
+                label="Mode",
+                choices=get_unique_values(df, "mode"),
+                value="All",
+            )
+            dtype_filter = gr.Dropdown(
+                label="DType",
+                choices=get_unique_values(df, "dtype"),
+                value="All",
+            )
+
+        results_table = gr.DataFrame(
+            value=df,
+            label="Benchmark Results",
+            interactive=False,
+            wrap=True,
+        )
+
+        gr.Markdown("### 📊 Metrics")
+        gr.Markdown(
+            "- **load_ms**: Model loading time in milliseconds\n"
+            "- **first_infer_ms**: First inference time in milliseconds\n"
+            "- **subsequent_infer_ms**: Subsequent inference time in milliseconds\n"
+            "- **p50/p90**: 50th and 90th percentile values"
+        )
+
+        def update_data():
+            """Reload data from HuggingFace."""
+            new_df = load_data()
+            return (
+                new_df,
+                gr.update(choices=get_unique_values(new_df, "task")),
+                gr.update(choices=get_unique_values(new_df, "platform")),
+                gr.update(choices=get_unique_values(new_df, "device")),
+                gr.update(choices=get_unique_values(new_df, "mode")),
+                gr.update(choices=get_unique_values(new_df, "dtype")),
+            )
+
+        def apply_filters(df, model, task, platform, device, mode, dtype):
+            """Apply filters and return filtered DataFrame."""
+            return filter_data(df, model, task, platform, device, mode, dtype)
+
+        # Refresh button updates data and resets filters
+        refresh_btn.click(
+            fn=update_data,
+            outputs=[
+                results_table,
+                task_filter,
+                platform_filter,
+                device_filter,
+                mode_filter,
+                dtype_filter,
+            ],
+        )
+
+        # Filter inputs update the table
+        filter_inputs = [
+            results_table,
+            model_filter,
+            task_filter,
+            platform_filter,
+            device_filter,
+            mode_filter,
+            dtype_filter,
+        ]
+
+        model_filter.change(
+            fn=apply_filters,
+            inputs=filter_inputs,
+            outputs=results_table,
+        )
+        task_filter.change(
+            fn=apply_filters,
+            inputs=filter_inputs,
+            outputs=results_table,
+        )
+        platform_filter.change(
+            fn=apply_filters,
+            inputs=filter_inputs,
+            outputs=results_table,
+        )
+        device_filter.change(
+            fn=apply_filters,
+            inputs=filter_inputs,
+            outputs=results_table,
+        )
+        mode_filter.change(
+            fn=apply_filters,
+            inputs=filter_inputs,
+            outputs=results_table,
+        )
+        dtype_filter.change(
+            fn=apply_filters,
+            inputs=filter_inputs,
+            outputs=results_table,
+        )
+
+    return demo
+
+
+demo = create_leaderboard_ui()
+demo.launch(server_name="0.0.0.0", server_port=7861)

leaderboard/src/leaderboard/data_loader.py

@@ -0,0 +1,184 @@
+"""
+Data loader module for loading benchmark results from HuggingFace Dataset.
+"""
+
+import json
+from typing import List, Dict, Any, Optional
+import pandas as pd
+from huggingface_hub import HfApi, hf_hub_download
+
+
+def load_benchmark_data(
+    dataset_repo: str,
+    token: Optional[str] = None,
+) -> pd.DataFrame:
+    """Load benchmark data from HuggingFace Dataset repository.
+
+    Args:
+        dataset_repo: HuggingFace dataset repository ID (e.g., "username/dataset-name")
+        token: HuggingFace API token (optional, for private datasets)
+
+    Returns:
+        DataFrame containing all benchmark results
+    """
+    if not dataset_repo:
+        return pd.DataFrame()
+
+    try:
+        api = HfApi(token=token)
+
+        # List all files in the dataset repo
+        files = api.list_repo_files(
+            repo_id=dataset_repo,
+            repo_type="dataset",
+            token=token,
+        )
+
+        # Filter for .json files
+        json_files = [f for f in files if f.endswith(".json")]
+
+        if not json_files:
+            return pd.DataFrame()
+
+        # Load all benchmark results
+        all_results = []
+        for file_path in json_files:
+            try:
+                result = load_single_benchmark_file(
+                    dataset_repo=dataset_repo,
+                    file_path=file_path,
+                    token=token,
+                )
+                if result:
+                    all_results.append(flatten_result(result))
+            except Exception as e:
+                print(f"Error loading {file_path}: {e}")
+                continue
+
+        if not all_results:
+            return pd.DataFrame()
+
+        # Convert to DataFrame
+        df = pd.DataFrame(all_results)
+
+        # Sort by model name and timestamp
+        if "modelId" in df.columns and "timestamp" in df.columns:
+            df = df.sort_values(["modelId", "timestamp"], ascending=[True, False])
+
+        return df
+
+    except Exception as e:
+        print(f"Error loading benchmark data: {e}")
+        return pd.DataFrame()
+
+
+def load_single_benchmark_file(
+    dataset_repo: str,
+    file_path: str,
+    token: Optional[str] = None,
+) -> Optional[Dict[str, Any]]:
+    """Load a single benchmark result file from HuggingFace Dataset.
+
+    Args:
+        dataset_repo: HuggingFace dataset repository ID
+        file_path: Path to the JSON file within the dataset
+        token: HuggingFace API token (optional)
+
+    Returns:
+        Dictionary containing the benchmark result, or None if failed
+    """
+    try:
+        # Download the file
+        local_path = hf_hub_download(
+            repo_id=dataset_repo,
+            filename=file_path,
+            repo_type="dataset",
+            token=token,
+        )
+
+        # Read JSON file (single object per file)
+        with open(local_path, "r") as f:
+            return json.load(f)
+
+    except Exception as e:
+        print(f"Error loading file {file_path}: {e}")
+        return None
+
+
+def flatten_result(result: Dict[str, Any]) -> Dict[str, Any]:
+    """Flatten nested benchmark result for display.
+
+    The HF Dataset format is already flattened by the bench service,
+    so we just need to extract the relevant fields.
+
+    Args:
+        result: Raw benchmark result dictionary
+
+    Returns:
+        Flattened dictionary with extracted fields
+    """
+    flat = {
+        "id": result.get("id", ""),
+        "platform": result.get("platform", ""),
+        "modelId": result.get("modelId", ""),
+        "task": result.get("task", ""),
+        "mode": result.get("mode", ""),
+        "repeats": result.get("repeats", 0),
+        "batchSize": result.get("batchSize", 0),
+        "device": result.get("device", ""),
+        "browser": result.get("browser", ""),
+        "dtype": result.get("dtype", ""),
+        "headed": result.get("headed", False),
+        "status": result.get("status", ""),
+        "timestamp": result.get("timestamp", 0),
+        "runtime": result.get("runtime", ""),
+    }
+
+    # Extract metrics if available (already at top level)
+    if "metrics" in result:
+        metrics = result["metrics"]
+
+        # Load time
+        if "load_ms" in metrics and "p50" in metrics["load_ms"]:
+            flat["load_ms_p50"] = metrics["load_ms"]["p50"]
+            flat["load_ms_p90"] = metrics["load_ms"]["p90"]
+
+        # First inference time
+        if "first_infer_ms" in metrics and "p50" in metrics["first_infer_ms"]:
+            flat["first_infer_ms_p50"] = metrics["first_infer_ms"]["p50"]
+            flat["first_infer_ms_p90"] = metrics["first_infer_ms"]["p90"]
+
+        # Subsequent inference time
+        if "subsequent_infer_ms" in metrics and "p50" in metrics["subsequent_infer_ms"]:
+            flat["subsequent_infer_ms_p50"] = metrics["subsequent_infer_ms"]["p50"]
+            flat["subsequent_infer_ms_p90"] = metrics["subsequent_infer_ms"]["p90"]
+
+    # Extract environment info (already at top level)
+    if "environment" in result:
+        env = result["environment"]
+        flat["cpuCores"] = env.get("cpuCores", 0)
+        if "memory" in env:
+            flat["memory_gb"] = env["memory"].get("deviceMemory", 0)
+
+    # Calculate duration
+    if "completedAt" in result and "startedAt" in result:
+        flat["duration_s"] = (result["completedAt"] - result["startedAt"]) / 1000
+
+    return flat
+
+
+def get_unique_values(df: pd.DataFrame, column: str) -> List[str]:
+    """Get unique values from a column for dropdown choices.
+
+    Args:
+        df: DataFrame to extract values from
+        column: Column name
+
+    Returns:
+        List of unique values with "All" as first item
+    """
+    if df.empty or column not in df.columns:
+        return ["All"]
+
+    values = df[column].dropna().unique().tolist()
+    return ["All"] + sorted([str(v) for v in values])