Current Updated! State (#4)

app.py

@@ -4,9 +4,9 @@ import os
 
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
-import literature_understanding, main_page, c_and_e, data_analysis, e2e
+import literature_understanding, main_page, c_and_e, data_analysis, e2e, submission
 
-from content import TITLE, css
+from content import css
 
 # --- Constants and Configuration  ---
 LOCAL_DEBUG = not (os.environ.get("system") == "spaces")
@@ -15,7 +15,7 @@ OWNER = "allenai"
 PROJECT_NAME = "asta-bench" + ("-internal" if IS_INTERNAL else "")
 LEADERBOARD_PATH = f"{OWNER}/{PROJECT_NAME}-leaderboard"
 api = HfApi()
-LOGO_PATH = "Ai2_logo_pink_padding_RGB.png"
+LOGO_PATH = "assets/logo.svg"
 
 
 
@@ -50,13 +50,22 @@ theme = gr.themes.Base(
     button_primary_background_fill_dark='*primary_900',
     button_primary_background_fill_hover='*secondary_600',
     button_primary_background_fill_hover_dark='*primary_600',
+    button_secondary_background_fill="#9FEAD1",
+    button_secondary_background_fill_dark="#9FEAD1",
+    button_secondary_text_color="*neutral_900",
+    button_secondary_text_color_dark="*neutral_900",
+    block_title_text_color="*neutral_900",
     button_primary_text_color='*neutral_900',
-    button_primary_text_color_dark='*neutral_900'
+    block_title_text_color_dark="#ffffff",
+    checkbox_label_text_color_dark="#000",
+    button_primary_text_color_dark='*neutral_900',
+    block_border_color="#032629",
+    block_border_color_dark="#9fead1",
+    block_background_fill_dark="#032629",
+    block_background_fill="#FAF2E9",
 )
-# --- Gradio App Definition ---
-demo = gr.Blocks(theme=theme, css=css)
-with demo:
-    gr.Image(
+def render_logo():
+    return gr.Image(
         value=LOGO_PATH,
         show_label=False,
         interactive=False,
@@ -65,17 +74,26 @@ with demo:
         show_fullscreen_button=False,
         elem_id="logo-image"
     )
-    gr.HTML(TITLE)
-
+# --- Gradio App Definition ---
+demo = gr.Blocks(theme=theme, css=css)
+with demo:
+    render_logo()
     main_page.demo.render()
 with demo.route("Literature Understanding"):
+    render_logo()
     literature_understanding.demo.render()
 with demo.route("Code & Execution"):
+    render_logo()
     c_and_e.demo.render()
 with demo.route("Data Analysis"):
+    render_logo()
     data_analysis.demo.render()
 with demo.route("Discovery"):
+    render_logo()
     e2e.demo.render()
+with demo.route(" 🚀 Submit an Agent"):
+    render_logo()
+    submission.demo.render()
 
 # --- Scheduler and Launch
 def restart_space_job():

c_and_e.py

@@ -2,13 +2,19 @@ import gradio as gr
 import pandas as pd
 
 # Import our UI factories and the data loader
-from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data
-
+from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data,create_sub_navigation_bar
+from content import PLACEHOLDER_DESCRIPTION
 # Define the category for this page
 CATEGORY_NAME = "Code Execution"
 
 with gr.Blocks() as demo:
-    gr.Markdown(f"## {CATEGORY_NAME} Leaderboard Results")
+    gr.Markdown(f"## {CATEGORY_NAME} Aggregated")
+    validation_df, validation_tag_map = get_full_leaderboard_data("validation")
+    test_df, test_tag_map = get_full_leaderboard_data("test")
+    gr.Markdown(PLACEHOLDER_DESCRIPTION, elem_id="category-intro")
+    if validation_tag_map:
+        create_sub_navigation_bar(validation_tag_map, CATEGORY_NAME)
+
 
     # --- This page now has two main sections: Validation and Test ---
     with gr.Tabs():

content.py

@@ -1,8 +1,5 @@
-TITLE = """<h1 align="center" id="space-title">AstaBench Leaderboard</h1>"""
+TITLE = """<h1 align="left" id="space-title">AstaBench Leaderboard</h1>"""
 
-INTRODUCTION_TEXT = """
-## Introduction
-"""
 INTRO_PARAGRAPH = """
 AI agents are on the rise, promising everything from travel planning to scientific discovery. But evaluating them—especially for real-world research tasks—remains a messy, inconsistent process. Metrics vary, cost is often ignored, and scientific use cases are rarely the focus. <br>
 <br>
@@ -11,9 +8,14 @@ Enter AstaBench, a grand challenge benchmark developed by Ai2 to test how well a
 SCATTER_DISCLAIMER = """
 Only agents that have cost data available will be shown in the scatter plot. If you don't see your agent, please ensure that you have provided cost data in your submission.
 """
-
-SUBMISSION_TEXT = """
-## Submissions
+PARETO_DISCLAIMER = """
+Agents names that are green are Pareto optimal, meaning they achieve the best performance for their cost. 
+"""
+LIT_DESCRIPTION = """
+Several of the evaluations in AstaBench probe an AI model's literature understanding skills — that is, its ability to find research papers based on a description, review questions on citation quality, retrieve information from the literature, and so on.
+"""
+PLACEHOLDER_DESCRIPTION = """
+THIS IS PLACEHOLDER TEXT. AstaBench is a benchmark suite designed to evaluate AI agents on their ability to perform complex tasks that require reasoning, planning, and execution. It includes a variety of benchmarks that test different aspects of agent performance, such as literature understanding, data analysis, and code execution.
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
@@ -66,17 +68,18 @@ def hf_uri_to_web_url(uri: str) -> str:
     return f"https://huggingface.co/datasets/{namespace}/{repo}/tree/main/{path}"
 
 css = """
-.submission-accordion {
-    border-style: solid;
-    border-width: 3px !important;
-    border-color: #ec4899;
+#intro-paragraph {
+    font-size: 18px;
+    max-width: 60%;
 }
-.submission-accordion span.svelte-1w6vloh { 
-    font-weight: bold !important;
-    font-size: 1.2em !important;
+#category-intro {
+    font-size: 18px;
+    max-width: 60%;
 }
 #logo-image { 
-    margin: auto;          
+    margin: 0;
+    margin-bottom: 30px; 
+    justify-content: flex-start;        
     max-width: 250px;       
     height: auto;           
 }
@@ -84,7 +87,6 @@ css = """
     height: auto !important;
     max-height: none !important;
 }
-
 .table-wrap {
     max-height: none !important;
     height: auto !important;
@@ -96,24 +98,55 @@ table.gr-table th, table.gr-table td {
     width: 1%;
     white-space: nowrap;
 }
-
+table.svelte-1e98i6s td {
+    vertical-align: top !important;
+}
 table.gr-table {
     font-size: 14px !important;
 }
-
-/* Example of making the "Agent" column (the 1st column) a bit wider if needed */
-table.gr-table th:nth-child(1),
-table.gr-table td:nth-child(1) {
-    min-width: 150px !important;
-    white-space: normal !important; /* Allow agent names to wrap if long */
-}
 .html-container {
     padding-top: 0 !important;
 }
 #scatter-disclaimer {
     color: #f0529c !important;
 }
+#pareto-disclaimer {
+    color: #f0529c !important;
+}
 thead.svelte-1e98i6s th {
     background: white !important;
 }
+.dark thead.svelte-1e98i6s th {
+    background: #091a1a !important;
+}
+.cell-wrap.svelte-v1pjjd {
+    font-family: 'Manrope';
+    }
+nav.svelte-ti537g.svelte-ti537g {
+    justify-content: flex-start;
+}
+#legend-markdown span {
+    margin-right: 15px !important; 
+}
+#leaderboard-accordion .label-wrap {
+    font-size: 1.4rem !important; 
+}
+.dark #leaderboard-accordion .label-wrap {
+    color: #0FCB8C !important; 
+}
+.dark block.svelte-1svsvh2 {
+    background: #032629 !important;
+}
+.sub-nav-bar {
+    margin-bottom: 20px; /* The space below the nav bar */
+}
+.sub-nav-bar a {
+    font-size: 16px;
+    border-radius: 5px;
+    transition: background-color 0.2s;
+    padding-right: 15px;
+}
+.padding.svelte-phx28p {
+    padding: 0 !important;
+}
 """

data_analysis.py

@@ -2,13 +2,18 @@ import gradio as gr
 import pandas as pd
 
 # Import our UI factories and the data loader
-from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data
-
+from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data, create_sub_navigation_bar
+from content import PLACEHOLDER_DESCRIPTION
 # Define the category for this page
 CATEGORY_NAME = "Data Analysis"
 
 with gr.Blocks() as demo:
-    gr.Markdown(f"## {CATEGORY_NAME} Leaderboard Results")
+    gr.Markdown(f"## {CATEGORY_NAME} Aggregated")
+    validation_df, validation_tag_map = get_full_leaderboard_data("validation")
+    test_df, test_tag_map = get_full_leaderboard_data("test")
+    gr.Markdown(PLACEHOLDER_DESCRIPTION, elem_id="category-intro")
+    if validation_tag_map:
+        create_sub_navigation_bar(validation_tag_map, CATEGORY_NAME)
 
     # --- This page now has two main sections: Validation and Test ---
     with gr.Tabs():

e2e.py

@@ -2,13 +2,18 @@ import gradio as gr
 import pandas as pd
 
 # Import our UI factories and the data loader
-from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data
-
+from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data, create_sub_navigation_bar
+from content import PLACEHOLDER_DESCRIPTION
 # Define the category for this page
 CATEGORY_NAME = "Discovery"
 
 with gr.Blocks() as demo:
-    gr.Markdown(f"## {CATEGORY_NAME} Leaderboard Results")
+    gr.Markdown(f"## {CATEGORY_NAME} Aggregated")
+    validation_df, validation_tag_map = get_full_leaderboard_data("validation")
+    test_df, test_tag_map = get_full_leaderboard_data("test")
+    gr.Markdown(PLACEHOLDER_DESCRIPTION, elem_id="category-intro")
+    if validation_tag_map:
+        create_sub_navigation_bar(validation_tag_map, CATEGORY_NAME)
 
     # --- This page now has two main sections: Validation and Test ---
     with gr.Tabs():

json_leaderboard.py

@@ -1,485 +0,0 @@
-import logging
-from typing import Optional, Any, Dict # Added Dict
-from zoneinfo import ZoneInfo
-
-# datasets import might not be strictly needed by LeaderboardViewer itself anymore,
-# but _get_dataframe might still use types from it if EvalResult refers to them.
-# For now, let's keep it if your EvalResult or SuiteConfig models have dependencies.
-# If not, it can be removed from here.
-import datasets # Potentially removable from this file
-import matplotlib.pyplot as plt
-import plotly.express as px
-import plotly.graph_objects as go
-import numpy as np
-import pandas as pd
-import seaborn as sns
-import json # For loading the local JSON file
-import os # For checking file existence
-
-from agenteval import compute_summary_statistics
-from agenteval.config import SuiteConfig
-from agenteval.models import EvalResult
-
-logger = logging.getLogger(__name__)
-
-import logging
-from typing import Optional, Any, Dict, List # Added List
-from zoneinfo import ZoneInfo # Assuming this might be used by SuiteConfig/EvalResult or _get_dataframe
-import json
-import os
-
-# Assuming these are correctly imported from your project
-from agenteval.config import SuiteConfig
-from agenteval.models import EvalResult
-# from agenteval import compute_summary_statistics # Used by _get_dataframe
-
-
-class DataTransformer:
-    """
-    Load and visualize leaderboard from a single, local JSON result file.
-    """
-    _INFORMAL_TO_FORMAL_NAME_MAP = {
-        "lit": "Literature Understanding",
-        "data": "Data Analysis",
-        "code": "Code Execution",
-        "discovery": "Discovery",
-        "arxivdigestables_validation": "Arxivdigestables Validation",
-        "sqa_dev": "Sqa Dev",
-        "litqa2_validation": "Litqa2 Validation",
-        "paper_finder_validation": "Paper Finder Validation",
-        "discoverybench_validation": "Discoverybench Validation",
-        "core_bench_validation": "Core Bench Validation",
-        "ds1000_validation": "DS1000 Validation",
-        "e2e_discovery_validation": "E2E Discovery Validation",
-        "super_validation": "Super Validation",
-        # Add any other raw names that can appear in task.name or task.tags
-    }
-
-    def __init__(
-            self,
-            json_file_path: str, # Mandatory: path to the local JSON file
-            split: str,          # Still needed for context within the JSON's suite_config
-            is_internal: bool = False
-    ):
-        self._json_file_path = json_file_path
-        self._split = split
-        self._internal = is_internal
-        self._loaded_json_data: Optional[Dict[str, Any]] = None
-        self._cfg: Optional[SuiteConfig] = None
-
-        logger.info(f"Initializing LeaderboardViewer with local JSON file: {self._json_file_path}")
-
-        # --- Load and Validate JSON data ---
-        if not os.path.exists(self._json_file_path):
-            raise FileNotFoundError(f"JSON file not found at path: {self._json_file_path}")
-        try:
-            with open(self._json_file_path, 'r', encoding='utf-8') as f:
-                self._loaded_json_data = json.load(f)
-        except json.JSONDecodeError as e:
-            raise ValueError(f"Failed to parse JSON from local file {self._json_file_path}: {e}")
-        except Exception as e:
-            raise ValueError(f"Error reading local file {self._json_file_path}: {e}")
-
-        if not self._loaded_json_data:
-            raise ValueError(f"No data loaded from JSON file {self._json_file_path}.")
-
-        try:
-            eval_result = EvalResult.model_validate(self._loaded_json_data)
-        except Exception as e:
-            raise ValueError(f"Failed to validate JSON data from file '{self._json_file_path}' against EvalResult model: {e}")
-
-        self._cfg = eval_result.suite_config
-        if not isinstance(self._cfg, SuiteConfig):
-            raise TypeError(f"self._cfg is not a SuiteConfig object after loading from '{self._json_file_path}', got {type(self._cfg)}.")
-
-        # --- Populate Tag Map (Corrected Placement and Helper Function Access) ---
-        self.tag_map: dict[str, list[str]] = {}
-
-        # Access tasks from the loaded config
-        tasks_for_split: List[Any] = self._cfg.get_tasks(self._split) # Assuming get_tasks returns a list of task-like objects
-
-        for task in tasks_for_split:
-            # Ensure task object has 'name' and 'tags' attributes
-            if not hasattr(task, 'name') or not hasattr(task, 'tags'):
-                logger.warning(f"Task object {task} is missing 'name' or 'tags' attribute. Skipping.")
-                continue
-
-            formal_task_display_name = self._get_formal_display_name_static(task.name) # Use the helper method
-
-            if not (task.tags or []):
-                continue
-
-            for raw_tag_name in task.tags:
-                formal_tag_display_name_key = self._get_formal_display_name_static(raw_tag_name)
-
-                self.tag_map.setdefault(formal_tag_display_name_key, []).append(formal_task_display_name)
-
-        for key in self.tag_map:
-            self.tag_map[key] = sorted(list(set(self.tag_map[key])))
-
-    # --- Helper function defined as a static method or regular method ---
-    # Option 1: Static method (doesn't need 'self', uses the class attribute)
-    @staticmethod
-    def _get_formal_display_name_static(raw_name: str) -> str:
-        """
-        Helper function to get the formal display name for a raw tag or task name.
-        Uses the class's map and provides a fallback.
-        """
-        return DataTransformer._INFORMAL_TO_FORMAL_NAME_MAP.get(raw_name, raw_name.replace("_", " ").title())
-
-    def _load(self) -> tuple[pd.DataFrame, dict[str, list[str]]]:
-        """
-        Prepares the DataFrame from the loaded JSON data.
-        The JSON data is already loaded and validated in __init__.
-        """
-        if self._loaded_json_data is None or self._cfg is None:
-            # This should not happen if __init__ completed successfully
-            raise RuntimeError("LeaderboardViewer2 not properly initialized. JSON data or SuiteConfig is missing.")
-
-        # The _get_dataframe function expects a list of records.
-        # Since we have a single JSON file representing one result, wrap it in a list.
-        records_list: list[dict] = [self._loaded_json_data]
-
-        overview_df = _get_dataframe(
-            records_list=records_list,
-            split=self._split,
-            is_internal=self._internal,
-            suite_config=self._cfg, # Pass the SuiteConfig loaded in __init__
-        )
-        return overview_df, self.tag_map
-
-    # --- view method remains the same as your last version ---
-    def view(
-            self,
-            tag: Optional[str] = None,
-            with_plots: bool = False,
-            use_plotly: bool = False,
-    ) -> tuple[pd.DataFrame, dict[str, Any]]:
-        data, tag_map = self._load() # tag_map is also returned by _load now
-        print(f"AHAHASHJDBFGASJHDBJAHSDB,AHDB {tag_map}")
-        print(f"THIS IS THE DATA DATA DTAA {data.columns}")
-        if data.empty or (len(data) == 1 and data.iloc[0].get("Agent") == "No data"):
-            logger.warning("No data available to view. Returning empty DataFrame and plots.")
-            return data, {}
-
-        base_cols = ["Agent", "Submitter", "Date", "Logs"]
-        existing_cols = [col for col in base_cols if col in data.columns]
-
-        primary_score_col: str
-        group_metric_names: list[str]
-
-        if tag is None:
-            primary = "Overall"
-            group = list(tag_map.keys())
-        else:
-            primary = tag
-            group = tag_map.get(tag, [])
-
-        if f"{primary} Score" in data.columns:
-            data = data.sort_values(f"{primary} Score", ascending=False)
-        else:
-            logger.warning(f"Primary metric '{primary}' for sorting not found. Data will not be sorted by it.")
-
-        metrics_to_display = []
-        if f"{primary} Cost" in data.columns:
-            metrics_to_display.append(f"{primary} Cost")
-        if f"{primary} Score" in data.columns:
-            metrics_to_display.append(f"{primary} Score")
-
-        for g_item in group:
-            if g_item in data.columns:
-                metrics_to_display.append(g_item)
-            if f"{g_item} Cost" in data.columns:
-                metrics_to_display.append(f"{g_item} Cost")
-            if f"{g_item} Score" in data.columns:
-                metrics_to_display.append(f"{g_item} Score")
-
-
-        final_cols_to_display = existing_cols + [m for m in metrics_to_display if m in data.columns]
-        final_cols_to_display = sorted(list(set(final_cols_to_display)), key=final_cols_to_display.index)
-
-        df_view = data.loc[:, final_cols_to_display].reset_index(drop=True)
-
-        plots: dict[str, Any] = {}
-        if with_plots:
-            plot_metric_names = [primary] + [g_item for g_item in group if g_item in data.columns]
-            for metric_name in plot_metric_names:
-                score_col = f"{metric_name} Score"
-                cost_col = f"{metric_name} Cost"
-                if score_col in df_view.columns and cost_col in df_view.columns:
-                    if use_plotly:
-                        fig = _plot_scatter_plotly(df_view, x=cost_col, y=score_col, agent_col="Agent")
-                    plots[f"scatter_{metric_name}"] = fig
-                else:
-                    logger.warning(
-                        f"Skipping plot for '{metric_name}': score column '{score_col}' or cost column '{cost_col}' not found."
-                    )
-        return df_view, plots
-
-
-def _safe_round(value, digits=2):
-    return round(value, digits) if isinstance(value, (float, int)) and pd.notna(value) else value
-
-def _get_dataframe(
-        records_list: list[dict],
-        split: str,
-        is_internal: bool,
-        suite_config: SuiteConfig,
-        timezone: str = "US/Pacific",
-) -> pd.DataFrame:
-    # This function remains the same as in the previous version you provided.
-    # It takes a list of records (which will be a list containing one item
-    # from the loaded JSON file) and processes it.
-    if not records_list:
-        logger.warning(f"No records provided to _get_dataframe for split '{split}'. Returning empty DataFrame with placeholder.")
-        expected_pretty_cols = ["Agent Name", "Submitter", "Date", "Overall Score", "Logs"]
-        empty_df = pd.DataFrame({p_col: ["No data"] for p_col in expected_pretty_cols})
-        return empty_df
-
-    cfg = suite_config
-
-    rows = []
-    for itm_idx, itm in enumerate(records_list):
-        if not isinstance(itm, dict):
-            logger.warning(f"Item {itm_idx} in records_list is not a dict, skipping.")
-            continue
-        try:
-            ev = EvalResult.model_validate(itm)
-        except Exception as e:
-            logger.error(f"Failed to validate item {itm_idx} with EvalResult: {itm}. Error: {e}")
-            continue
-
-        sub = ev.submission
-        date_str = None
-        if sub.submit_time is not None:
-            submit_dt = sub.submit_time
-            if not isinstance(submit_dt, pd.Timestamp):
-                if submit_dt.tzinfo is None:
-                    logger.debug(f"Submission time for {sub.agent_name} is timezone-naive, assuming UTC.")
-                    submit_dt = submit_dt.replace(tzinfo=ZoneInfo("UTC"))
-            date_str = pd.Timestamp(submit_dt).tz_convert(ZoneInfo(timezone)).strftime("%Y-%m-%d")
-        else:
-            date_str = None
-
-        if not ev.results:
-            logger.warning(
-                f"Skipping submission {sub.agent_name} ({sub.username or 'N/A'}) "
-                f"({sub.submit_time or 'N/A'}) due to no results."
-            )
-            continue
-        stats = compute_summary_statistics(
-            suite_config=cfg, split=split, results=ev.results
-        )
-        flat = {}
-        print(f"STATS STATS ASTATAS SD T S T A A {stats}")
-        for key, s_obj in stats.items():
-            parts = key.split("/")
-            if parts[0] == "overall":
-                flat["overall/score"] = _safe_round(getattr(s_obj, 'score', np.nan))
-                flat["overall/cost"] = _safe_round(getattr(s_obj, 'cost', np.nan))
-            elif parts[0] == "tag" and len(parts) > 1:
-                tag_name = parts[1]
-                flat[f"tag/{tag_name}/score"] = _safe_round(getattr(s_obj, 'score', np.nan))
-                flat[f"tag/{tag_name}/cost"] = _safe_round(getattr(s_obj, 'cost', np.nan))
-            elif parts[0] == "task" and len(parts) > 1:
-                task_name = parts[1]
-                score = getattr(s_obj, 'score', np.nan)
-                cost = getattr(s_obj, 'cost', np.nan)
-                score_stderr = getattr(s_obj, 'score_stderr', np.nan)
-                cost_stderr = getattr(s_obj, 'cost_stderr', np.nan)
-
-                flat[f"task/{task_name}/score"] = _safe_round(score)
-                flat[f"task/{task_name}/score_ci"] = _safe_round(score_stderr * 1.96 if pd.notna(score_stderr) else np.nan)
-                flat[f"task/{task_name}/cost"] = _safe_round(cost)
-                flat[f"task/{task_name}/cost_ci"] = _safe_round(cost_stderr * 1.96 if pd.notna(cost_stderr) else np.nan)
-            else:
-                logger.debug(f"Uncommon key structure from compute_summary_statistics: '{key}'. Attempting generic add.")
-                if hasattr(s_obj, 'score'):
-                    flat[f"{key}/score"] = _safe_round(s_obj.score)
-                if hasattr(s_obj, 'cost'):
-                    flat[f"{key}/cost"] = _safe_round(s_obj.cost)
-
-        current_logs_url = None
-        if is_internal and sub.logs_url:
-            current_logs_url = str(sub.logs_url)
-        elif not is_internal and sub.logs_url_public:
-            current_logs_url = str(sub.logs_url_public)
-
-        rows.append(
-            {
-                "agent_name": sub.agent_name or "N/A",
-                "username": sub.username or "N/A",
-                "submit_time": date_str,
-                **flat,
-                "logs_url": current_logs_url,
-            }
-        )
-
-    if not rows:
-        logger.warning(f"No valid rows generated from records_list for split '{split}'. Returning empty DataFrame with placeholder.")
-        expected_pretty_cols = ["Agent", "Submitter", "Date", "Overall Score", "Overall Cost", "Logs"]
-        empty_df = pd.DataFrame({p_col: ["No data"] for p_col in expected_pretty_cols})
-        return empty_df
-
-    df = pd.DataFrame(rows)
-    pretty_cols = {c: _pretty_column_name(c) for c in df.columns if c in df.columns}
-    overview = df.rename(columns=pretty_cols)
-    return overview
-
-def _pretty_column_name(col: str) -> str:
-    """Map raw column name to display name."""
-    # --- Step 1: Fixed, direct mappings ---
-    fixed_mappings = {
-        "submit_time": "Date",
-        "agent_name": "Agent",
-        "username": "Submitter",
-        "logs_url": "Logs",
-        "overall/score": "Overall Score",
-        "overall/cost": "Overall Cost",
-    }
-    if col in fixed_mappings:
-        return fixed_mappings[col]
-
-    # --- Step 2: Define your mapping for informal names to descriptive names ---
-    informal_map = DataTransformer._INFORMAL_TO_FORMAL_NAME_MAP
-
-    # --- Step 3: Dynamic mappings for task or tag columns using the informal_to_formal_name_map ---
-    parts = col.split("/")
-    if len(parts) == 3:
-        item_type, informal_name, metric_suffix = parts #
-
-        formal_name = informal_map.get(informal_name)
-        if formal_name is None:
-            formal_name = informal_name.replace("_", " ").title()
-            print(f"[DEBUG _pretty_column_name] Informal name '{informal_name}' not in map, using fallback: '{formal_name}'")
-
-        if metric_suffix == "score":
-            return f"{formal_name} Score"
-        if metric_suffix == "cost":
-            return f"{formal_name} Cost"
-        if metric_suffix == "score_ci":
-            return f"{formal_name} Score 95% CI"
-        if metric_suffix == "cost_ci":
-            return f"{formal_name} Cost 95% CI"
-
-    # --- Step 4: Fallback for columns that don't match the "type/name/metric" pattern ---
-    if "/" not in col:
-        return col.replace("_", " ").title()
-    else:
-        return parts[-1].replace("_", " ").title()
-
-DEFAULT_Y_COLUMN = "Overall Score"
-DUMMY_X_VALUE_FOR_MISSING_COSTS = 0 # Value to use if x-axis data (costs) is missing
-
-def _plot_scatter_plotly(
-        data: pd.DataFrame,
-        x: Optional[str],
-        y: str,
-        agent_col: str = "Agent"
-) -> go.Figure:
-
-    x_col_to_use = x
-    y_col_to_use = y
-
-    # 1. Check if y-column exists
-    if y_col_to_use not in data.columns:
-        logger.error(
-            f"y-axis column '{y_col_to_use}' MUST exist in DataFrame. "
-            f"Cannot generate plot. Available columns: {data.columns.tolist()}"
-        )
-        return go.Figure()
-
-    # 2. Check if agent_col exists
-    if agent_col not in data.columns:
-        logger.warning(
-            f"Agent column '{agent_col}' not found in DataFrame. "
-            f"Available columns: {data.columns.tolist()}. Returning empty figure."
-        )
-        return go.Figure()
-
-    # 3. Prepare data (make a copy, handle numeric conversion for y)
-    data_plot = data.copy()
-    try:
-        data_plot[y_col_to_use] = pd.to_numeric(data_plot[y_col_to_use], errors='coerce')
-    except Exception as e:
-        logger.error(f"Error converting y-column '{y_col_to_use}' to numeric: {e}. Returning empty figure.")
-        return go.Figure()
-
-    # 4. Handle x-column (costs)
-    x_axis_label = x_col_to_use if x_col_to_use else "Cost (Data N/A)" # Label for the x-axis
-    x_data_is_valid = False
-
-    if x_col_to_use and x_col_to_use in data_plot.columns:
-        try:
-            data_plot[x_col_to_use] = pd.to_numeric(data_plot[x_col_to_use], errors='coerce')
-            # Check if there's any non-NaN data after coercion for x
-            if data_plot[x_col_to_use].notna().any():
-                x_data_is_valid = True
-            else:
-                logger.info(f"x-axis column '{x_col_to_use}' exists but contains all NaN/None values after numeric conversion.")
-        except Exception as e:
-            logger.warning(f"Error converting x-column '{x_col_to_use}' to numeric: {e}. Will use dummy x-values.")
-            # x_data_is_valid remains False
-    else:
-        if x_col_to_use: # Name was provided but column doesn't exist
-            logger.warning(f"x-axis column '{x_col_to_use}' not found in DataFrame.")
-        else: # x (column name) was None
-            logger.info("x-axis column name was not provided (is None).")
-
-    if not x_data_is_valid:
-        logger.info(f"Using dummy x-value '{DUMMY_X_VALUE_FOR_MISSING_COSTS}' for all data points as x-data is missing or invalid.")
-        # Create a new column with the dummy x-value for all rows
-        # Use a unique name for this dummy column to avoid potential clashes
-        dummy_x_col_name = "__dummy_x_for_plotting__"
-        data_plot[dummy_x_col_name] = DUMMY_X_VALUE_FOR_MISSING_COSTS
-        x_col_to_use = dummy_x_col_name # Update x_col_to_use to point to our dummy data
-        x_axis_label = x if x else "Cost (Data N/A)" # Use original x name for label if provided
-        # or a generic label if x was None.
-        # Could also be f"Cost (Fixed at {DUMMY_X_VALUE_FOR_MISSING_COSTS})"
-
-
-    # 5. Drop rows where y is NaN (x is now guaranteed to have values, either real or dummy)
-    data_plot.dropna(subset=[y_col_to_use], inplace=True)
-
-    fig = go.Figure()
-
-    if data_plot.empty:
-        logger.warning(f"No valid data to plot for y='{y_col_to_use}' (and x='{x_col_to_use}') after cleaning NaNs from y.")
-        # Still return a figure object, but it will be empty. Update layout for clarity.
-        fig.update_layout(
-            title=f"{y_col_to_use} vs. {x_axis_label} (No Data)",
-            xaxis=dict(title=x_axis_label, range=[DUMMY_X_VALUE_FOR_MISSING_COSTS - 1, DUMMY_X_VALUE_FOR_MISSING_COSTS + 1] if not x_data_is_valid else None),
-            yaxis=dict(title=y_col_to_use)
-        )
-        return fig
-
-
-    for agent, group in data_plot.groupby(agent_col):
-        hover_x_display = "%{x:.2f}" if x_data_is_valid else str(DUMMY_X_VALUE_FOR_MISSING_COSTS) + " (fixed)"
-        fig.add_trace(go.Scatter(
-            x=group[x_col_to_use],
-            y=group[y_col_to_use],
-            mode='markers',
-            name=str(agent),
-            hovertemplate=f"{x_axis_label}: {hover_x_display}<br>{y_col_to_use}: %{{y:.2f}}<extra>{str(agent)}</extra>",
-            marker=dict(size=10)
-        ))
-
-    # Configure layout
-    xaxis_config = dict(title=x_axis_label)
-    if not x_data_is_valid: # If using dummy x, set a tighter, fixed range for x-axis
-        xaxis_config['range'] = [DUMMY_X_VALUE_FOR_MISSING_COSTS - 1, DUMMY_X_VALUE_FOR_MISSING_COSTS + 1]
-        xaxis_config['tickvals'] = [DUMMY_X_VALUE_FOR_MISSING_COSTS] # Show only one tick at the dummy value
-        xaxis_config['ticktext'] = [str(DUMMY_X_VALUE_FOR_MISSING_COSTS)]
-    else: # Real x-data
-        xaxis_config['rangemode'] = "tozero"
-
-
-    fig.update_layout(
-        title=f"{y_col_to_use} vs. {x_axis_label}",
-        xaxis=xaxis_config,
-        yaxis=dict(title=y_col_to_use, rangemode="tozero"),
-        legend_title_text=agent_col
-    )
-
-    return fig

leaderboard_transformer.py

@@ -2,10 +2,8 @@ import plotly.graph_objects as go
 import numpy as np
 import pandas as pd
 import logging
-from typing import Optional, Any, Dict, List # Added List
-from zoneinfo import ZoneInfo # Assuming this might be used by SuiteConfig/EvalResult or _get_dataframe
-import json
-import os
+from typing import Optional
+import base64
 
 logger = logging.getLogger(__name__)
 
@@ -46,14 +44,18 @@ def _pretty_column_name(raw_col: str) -> str:
     """
     # Case 1: Handle fixed, special-case mappings first.
     fixed_mappings = {
+        'id': 'id',
         'Agent': 'Agent',
         'Agent description': 'Agent Description',
         'User/organization': 'Submitter',
         'Submission date': 'Date',
         'Overall': 'Overall Score',
         'Overall cost': 'Overall Cost',
-        'Logs': 'Logs'
+        'Logs': 'Logs',
+        'Openness': 'Openness',
+        'Agent tooling': 'Agent Tooling',
     }
+
     if raw_col in fixed_mappings:
         return fixed_mappings[raw_col]
 
@@ -146,7 +148,6 @@ class DataTransformer:
     def __init__(self, dataframe: pd.DataFrame, tag_map: dict[str, list[str]]):
         """
         Initializes the viewer.
-
         Args:
             dataframe (pd.DataFrame): The presentation-ready leaderboard data.
             tag_map (dict): A map of formal tag names to formal task names.
@@ -188,29 +189,53 @@ class DataTransformer:
         if primary_score_col in self.data.columns:
             df_sorted = self.data.sort_values(primary_score_col, ascending=False, na_position='last')
 
-        # --- 3. Build the List of Columns to Display ---
-        base_cols = ["Agent", "Submitter"]
-        new_cols = ["Openness", "Degree of Control"]
-        ending_cols = ["Date", "Logs"]
+        df_view = df_sorted.copy()
+        #preserve just agent name for scatterplot hover
+        df_view['agent_for_hover'] = df_view['Agent']
+        # 3. Combine "Agent" and "Submitter" into a single HTML-formatted column
+        #    We do this *before* defining the final column list.
+        if 'Agent' in df_view.columns and 'Submitter' in df_view.columns:
+
+            def combine_agent_submitter(row):
+                agent = row['Agent']
+                submitter = row['Submitter']
+
+                # Check if submitter exists and is not empty
+                if pd.notna(submitter) and submitter.strip() != '':
+                    # Create a two-line HTML string with styled submitter text
+                    return (
+                        f"<div>{agent}<br>"
+                        f"<span style='font-size: 0.9em; color: #667876;'>{submitter}</span>"
+                        f"</div>"
+                    )
+                else:
+                    # If no submitter, just return the agent name
+                    return agent
 
-        # Start with the primary metric score and cost
-        metrics_to_display = [primary_score_col, f"{primary_metric} Cost"]
+            # Apply the function to create the new combined 'Agent' column
+            df_view['Agent'] = df_view.apply(combine_agent_submitter, axis=1)
+            # The 'Submitter' column is no longer needed
+            df_view = df_view.drop(columns=['Submitter'])
 
-        # Add the score and cost for each item in our group
+        # 4. Build the List of Columns to Display (now simplified)
+        base_cols = ["id","Agent","agent_for_hover"]
+        new_cols = ["Openness", "Agent Tooling"]
+        ending_cols = ["Logs"]
+
+        metrics_to_display = [primary_score_col, f"{primary_metric} Cost"]
         for item in group_metrics:
             metrics_to_display.append(f"{item} Score")
             metrics_to_display.append(f"{item} Cost")
 
-        # Combine base columns with metric columns, ensuring uniqueness and order
-        final_cols_ordered = base_cols + list(dict.fromkeys(metrics_to_display))+ new_cols + ending_cols
+        final_cols_ordered = new_cols + base_cols +  list(dict.fromkeys(metrics_to_display)) + ending_cols
 
-        # Filter to only include columns that actually exist in our DataFrame
-        df_view = df_sorted.copy()
         for col in final_cols_ordered:
             if col not in df_view.columns:
                 df_view[col] = pd.NA
 
+        # The final selection will now use the new column structure
         df_view = df_view[final_cols_ordered].reset_index(drop=True)
+        cols = len(final_cols_ordered)
 
         # Calculated and add "Categories Attempted" column
         if primary_metric == "Overall":
@@ -220,29 +245,28 @@ class DataTransformer:
 
                 # Return the formatted string with the correct emoji
                 if count == 4:
-                    return f"4/4 ✅"
+                    return f"4/4"
                 if count == 0:
-                    return f"0/4 🚫"
-                return f"{count}/4 ⚠️"
+                    return f"0/4"
+                return f"{count}/4"
 
             # Apply the function row-wise to create the new column
             attempted_column = df_view.apply(calculate_attempted, axis=1)
             # Insert the new column at a nice position (e.g., after "Date")
-            df_view.insert(2, "Categories Attempted", attempted_column)
+            df_view.insert((cols - 1), "Categories Attempted", attempted_column)
         else:
             total_benchmarks = len(group_metrics)
             def calculate_benchmarks_attempted(row):
                 # Count how many benchmarks in this category have COST data reported
                 count = sum(1 for benchmark in group_metrics if pd.notna(row.get(f"{benchmark} Cost")))
                 if count == total_benchmarks:
-                    return f"{count}/{total_benchmarks} ✅"
+                    return f"{count}/{total_benchmarks} "
                 elif count == 0:
-                    return f"{count}/{total_benchmarks} 🚫"
+                    return f"{count}/{total_benchmarks} "
                 else:
-                    return f"{count}/{total_benchmarks}⚠️"
+                    return f"{count}/{total_benchmarks}"
             # Insert the new column, for example, after "Date"
-            df_view.insert(2, "Benchmarks Attempted", df_view.apply(calculate_benchmarks_attempted, axis=1))
-
+            df_view.insert((cols - 1), "Benchmarks Attempted", df_view.apply(calculate_benchmarks_attempted, axis=1))
 
         # --- 4. Generate the Scatter Plot for the Primary Metric ---
         plots: dict[str, go.Figure] = {}
@@ -254,7 +278,7 @@ class DataTransformer:
                     data=df_view,
                     x=primary_cost_col,
                     y=primary_score_col,
-                    agent_col="Agent"
+                    agent_col="agent_for_hover"
                 )
                 # Use a consistent key for easy retrieval later
                 plots['scatter_plot'] = fig
@@ -274,24 +298,37 @@ def _plot_scatter_plotly(
         data: pd.DataFrame,
         x: Optional[str],
         y: str,
-        agent_col: str = "Agent"
+        agent_col: str = 'agent_for_hover'
 ) -> go.Figure:
 
-    # --- Steps 1-4: Data Validation and Preparation ---
+    # --- Section 1: Define Mappings ---
+    color_map = {
+        "Closed": "red",
+        "API Available": "orange",
+        "Open Source": "green",
+        "Open Source + Open Weights": "blue"
+    }
+    category_order = list(color_map.keys())
+
+    shape_map = {
+        "Standard": "star",
+        "Custom with Standard Search": "diamond",
+        "Fully Custom": "circle"
+    }
+    default_shape = 'square'
+
     x_col_to_use = x
     y_col_to_use = y
 
-    if y_col_to_use not in data.columns:
-        logger.error(f"y-axis column '{y_col_to_use}' not found.")
-        return go.Figure()
-    if agent_col not in data.columns:
-        logger.warning(f"Agent column '{agent_col}' not found.")
+    required_cols = [y_col_to_use, agent_col, "Openness", "Agent Tooling"]
+    if not all(col in data.columns for col in required_cols):
+        logger.error(f"Missing one or more required columns for plotting: {required_cols}")
         return go.Figure()
 
     data_plot = data.copy()
     data_plot[y_col_to_use] = pd.to_numeric(data_plot[y_col_to_use], errors='coerce')
 
-    x_axis_label = x if x else "Cost (Data N/A)"
+    x_axis_label = f"{x} (USD)" if x else "Cost (Data N/A)"
     x_data_is_valid = False
     if x and x in data_plot.columns:
         try:
@@ -307,30 +344,27 @@ def _plot_scatter_plotly(
         x_col_to_use = dummy_x_col_name
         logger.info("Using dummy x-values for plotting.")
 
-    # --- Step 5: Clean Data and Initialize Figure ---
-    data_plot.dropna(subset=[y_col_to_use, x_col_to_use], inplace=True)
+    # Clean data based on all necessary columns
+    data_plot.dropna(subset=[y_col_to_use, x_col_to_use, "Openness", "Agent Tooling"], inplace=True)
+
+    # --- Section 3: Initialize Figure ---
     fig = go.Figure()
     if data_plot.empty:
-        logger.warning(f"No valid data to plot for y='{y_col_to_use}' and x='{x_col_to_use}'.")
+        logger.warning(f"No valid data to plot after cleaning.")
         return fig
 
-    # Step 6 - Calculate and Draw the Efficiency Frontier Line ---
+    # --- Section 4: Calculate and Draw Pareto Frontier (Restored from your original code) ---
     if x_data_is_valid:
-        # Sort by cost (ascending), then by score (descending) to break ties
         sorted_data = data_plot.sort_values(by=[x_col_to_use, y_col_to_use], ascending=[True, False])
-
         frontier_points = []
         max_score_so_far = float('-inf')
 
-        for index, row in sorted_data.iterrows():
+        for _, row in sorted_data.iterrows():
             score = row[y_col_to_use]
-            # If this point offers a better score than any we've seen before,
-            # it's part of the frontier.
-            if score > max_score_so_far:
+            if score >= max_score_so_far:
                 frontier_points.append({'x': row[x_col_to_use], 'y': score})
                 max_score_so_far = score
 
-        # Add the frontier line trace to the plot if we found any points
         if frontier_points:
             frontier_df = pd.DataFrame(frontier_points)
             fig.add_trace(go.Scatter(
@@ -339,22 +373,67 @@ def _plot_scatter_plotly(
                 mode='lines',
                 name='Efficiency Frontier',
                 line=dict(color='firebrick', width=2, dash='dash'),
-                hoverinfo='skip' # The line doesn't need a hover tooltip
+                hoverinfo='skip'
             ))
 
-    # --- Step 7: Plot Individual Agent Markers (No changes here) ---
-    for agent, group in data_plot.groupby(agent_col):
-        hover_x_display = "%{x:.2f}" if x_data_is_valid else "N/A"
+    # --- Section 5: Prepare for Marker Plotting ---
+    # Pre-generate hover text and shapes for each point
+    data_plot['hover_text'] = data_plot.apply(
+        lambda row: f"<b>{row[agent_col]}</b><br>{x_axis_label}: ${row[x_col_to_use]:.2f}<br>{y_col_to_use}: {row[y_col_to_use]:.2f}",
+        axis=1
+    )
+    data_plot['shape_symbol'] = data_plot['Agent Tooling'].map(shape_map).fillna(default_shape)
+
+    # --- Section 6: Plot Markers by "Openness" Category ---
+    for category in category_order:
+        group = data_plot[data_plot['Openness'] == category]
+        if group.empty:
+            continue
+
         fig.add_trace(go.Scatter(
             x=group[x_col_to_use],
             y=group[y_col_to_use],
             mode='markers',
-            name=str(agent),
-            hovertemplate=f"<b>{str(agent)}</b><br>{x_axis_label}: {hover_x_display}<br>{y_col_to_use}: %{{y:.2f}}""<extra></extra>",
-            marker=dict(size=10, opacity=0.8)
+            name=category,
+            showlegend=False,
+            text=group['hover_text'],
+            hoverinfo='text',
+            marker=dict(
+                color=color_map.get(category, 'grey'),
+                symbol=group['shape_symbol'],
+                size=10,
+                opacity=0.8,
+                line=dict(width=1, color='DarkSlateGrey')
+            )
+        ))
+    # ---- Add logic for making the legend -----------
+    for i, category in enumerate(category_order):
+        fig.add_trace(go.Scatter(
+            x=[None], y=[None],
+            mode='markers',
+            name=category,
+            legendgroup="openness_group",
+            legendgrouptitle_text="Agent Openness" if i == 0 else None,
+            marker=dict(
+                color=color_map.get(category, 'grey'),
+                symbol='circle',
+                size=12
+            )
+        ))
+
+    # Part B: Dummy traces for the SHAPES ("Agent Tooling")
+    shape_items = list(shape_map.items())
+    for i, (shape_name, shape_symbol) in enumerate(shape_items):
+        fig.add_trace(go.Scatter(
+            x=[None], y=[None],
+            mode='markers',
+            name=shape_name,
+            legendgroup="tooling_group",
+            legendgrouptitle_text="Agent Tooling" if i == 0 else None,
+            marker=dict(color='grey', symbol=shape_symbol, size=12)
         ))
 
-    # --- Step 8: Configure Layout (No changes here) ---
+    # --- Section 8: Configure Layout (Restored from your original code) ---
     xaxis_config = dict(title=x_axis_label)
     if not x_data_is_valid:
         xaxis_config['range'] = [DUMMY_X_VALUE_FOR_MISSING_COSTS - 1, DUMMY_X_VALUE_FOR_MISSING_COSTS + 1]
@@ -362,15 +441,32 @@ def _plot_scatter_plotly(
     else:
         xaxis_config['rangemode'] = "tozero"
 
+    logo_data_uri = svg_to_data_uri("assets/just-icon.svg")
+
     fig.update_layout(
+        template="plotly_white",
         title=f"{y_col_to_use} vs. {x_axis_label}",
         xaxis=xaxis_config,
         yaxis=dict(title=y_col_to_use, rangemode="tozero"),
-        legend_title_text=agent_col
+        legend=dict(
+            bgcolor='#FAF2E9',
+        )
+    )
+    fig.add_layout_image(
+        dict(
+            source=logo_data_uri,
+            xref="x domain", yref="y domain",
+            x=1.1, y=1.1,
+            sizex=0.2, sizey=0.2,
+            xanchor="left",
+            yanchor="bottom",
+            layer="above",
+        ),
     )
 
     return fig
 
+
 def format_cost_column(df: pd.DataFrame, cost_col_name: str) -> pd.DataFrame:
     """
     Applies custom formatting to a cost column based on its corresponding score column.
@@ -398,7 +494,7 @@ def format_cost_column(df: pd.DataFrame, cost_col_name: str) -> pd.DataFrame:
         if pd.notna(cost_value) and isinstance(cost_value, (int, float)):
             return f"${cost_value:.2f}"
         elif pd.notna(score_value):
-            return f'<span style="color: {status_color};">Missing Cost</span>'  # Score exists, but cost is missing
+            return f'<span style="color: {status_color};">Missing</span>'  # Score exists, but cost is missing
         else:
             return f'<span style="color: {status_color};">Not Attempted</span>'  # Neither score nor cost exists
 
@@ -434,3 +530,43 @@ def format_score_column(df: pd.DataFrame, score_col_name: str) -> pd.DataFrame:
     # Apply the formatting and return the updated DataFrame
     return df.assign(**{score_col_name: df[score_col_name].apply(apply_formatting)})
 
+
+def get_pareto_df(data):
+    # This is a placeholder; use your actual function that handles dynamic column names
+    # A robust version might look for any column with "Cost" and "Score"
+    cost_cols = [c for c in data.columns if 'Cost' in c]
+    score_cols = [c for c in data.columns if 'Score' in c]
+    if not cost_cols or not score_cols:
+        return pd.DataFrame()
+
+    x_col, y_col = cost_cols[0], score_cols[0]
+
+    frontier_data = data.dropna(subset=[x_col, y_col]).copy()
+    frontier_data[y_col] = pd.to_numeric(frontier_data[y_col], errors='coerce')
+    frontier_data[x_col] = pd.to_numeric(frontier_data[x_col], errors='coerce')
+    frontier_data.dropna(subset=[x_col, y_col], inplace=True)
+    if frontier_data.empty:
+        return pd.DataFrame()
+
+    frontier_data = frontier_data.sort_values(by=[x_col, y_col], ascending=[True, False])
+
+    pareto_points = []
+    max_score_at_cost = -np.inf
+
+    for _, row in frontier_data.iterrows():
+        if row[y_col] >= max_score_at_cost:
+            pareto_points.append(row)
+            max_score_at_cost = row[y_col]
+
+    return pd.DataFrame(pareto_points)
+
+
+def svg_to_data_uri(path: str) -> str:
+    """Reads an SVG file and encodes it as a Data URI for Plotly."""
+    try:
+        with open(path, "rb") as f:
+            encoded_string = base64.b64encode(f.read()).decode()
+        return f"data:image/svg+xml;base64,{encoded_string}"
+    except FileNotFoundError:
+        logger.warning(f"SVG file not found at: {path}")
+        return None

leaderboard_viewer.py

@@ -1,319 +0,0 @@
-"""
-View and plot leaderboard results.
-"""
-
-import logging
-from typing import Optional
-from zoneinfo import ZoneInfo
-
-import datasets
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-import seaborn as sns
-
-from agenteval import compute_summary_statistics
-from agenteval.config import SuiteConfig
-from agenteval.models import EvalResult
-
-logger = logging.getLogger(__name__)
-
-
-class LeaderboardViewer:
-    """
-    Load and visualize leaderboard for a given HF dataset split.
-    """
-
-    def __init__(
-            self, repo_id: str, config: str, split: str, is_internal: bool = False
-    ):
-        self._repo_id = repo_id
-        self._config = config
-        self._split = split
-        self._internal = is_internal
-
-        # build suite_config and mapping from tags to tasks from the first result
-        # TODO: Verify the sort order
-        ds = datasets.load_dataset(repo_id, name=config).get(split)
-        if not ds:
-            raise ValueError(f"Split '{split}' not found in dataset results")
-        suite = EvalResult.model_validate(ds[0]).suite_config
-        self._cfg = suite
-        self.tag_map: dict[str, list[str]] = {}
-        for task in suite.get_tasks(split):
-            for t in task.tags or []:
-                self.tag_map.setdefault(t, []).append(task.name)
-
-    def _load(self):
-        results = datasets.load_dataset(self._repo_id, name=self._config)
-        overview = _get_dataframe(
-            eval_results=results,
-            split=self._split,
-            is_internal=self._internal,
-            suite_config=self._cfg,
-        )
-        return overview, self.tag_map
-
-    def view(
-            self, tag: Optional[str] = None, with_plots: bool = False
-    ) -> tuple[pd.DataFrame, dict[str, plt.Figure]]:
-        """
-        If tag is None, primary="Overall" and group=all tags.
-        Otherwise primary=tag and group=tasks under that tag.
-        """
-        data, tag_map = self._load()
-        cols = [
-            "Agent",
-            "Submitter",
-            "Completeness",
-            "LLM Base",
-            "Openness" ,
-            "Date",
-            "Logs",
-        ]
-
-        # choose primary metric and its sub‐group
-        if tag is None:
-            primary = "Overall"
-            group = list(tag_map.keys())
-        else:
-            primary = tag
-            group = tag_map.get(tag, [])
-        data = data.sort_values(primary, ascending=False)
-
-        # build full metric list: primary + its cost + each member and its cost
-        metrics = [primary, f"{primary} cost"] + [
-            m for t in group for m in (t, f"{t} cost")
-        ]
-
-        # filter to relevant columns
-        ci_cols = [f"{m} 95% CI" for m in metrics if f"{m} 95% CI" in data.columns]
-        df = data.loc[
-             :,
-             cols + [c for c in metrics if c in data.columns] + ci_cols,
-             ].reset_index(drop=True)
-
-        plots: dict[str, plt.Figure] = {}
-        if with_plots:
-            avail = [c for c in metrics if c in df.columns]
-            for m in [primary] + group:
-                x, y = f"{m} cost", m
-                if x in df.columns and y in df.columns:
-                    plots[f"scatter_{m}"] = _plot_scatter(
-                        df, x=x, y=y, agent_col="Agent"
-                    )
-
-        return df, plots
-
-
-def _get_dataframe(
-        eval_results: datasets.DatasetDict,
-        split: str,
-        is_internal: bool,
-        suite_config: SuiteConfig,
-        timezone: str = "US/Pacific",
-) -> pd.DataFrame:
-    """
-    Load leaderboard results from the given dataset split and return a DataFrame.
-    """
-    ds = eval_results.get(split)
-    if not ds:
-        cols = ["agent_name", "agent_description", "username", "submit_time"]
-        pretty = [_pretty_column_name(c) for c in cols]
-        empty = pd.DataFrame({c: ["No data"] for c in pretty})
-        return empty
-
-    cfg = suite_config
-
-    rows = []
-    for itm in ds:
-        ev = EvalResult.model_validate(itm)
-        sub = ev.submission
-        # only format if submit_time present, else leave as None
-        ts = sub.submit_time
-        if ts is not None:
-            date = ts.astimezone(ZoneInfo(timezone)).strftime("%Y-%m-%d")
-        else:
-            date = None
-
-        if not ev.results:
-            logger.warning(
-                f"Skipping submission {sub.agent_name} ({sub.username}) "
-                f"({sub.submit_time}) with no results"
-            )
-            continue
-        stats = compute_summary_statistics(
-            suite_config=cfg, split=split, results=ev.results
-        )
-        flat = {}
-        for key, s in stats.items():
-            parts = key.split("/")
-            if parts[0] == "overall":
-                flat["overall/score"], flat["overall/cost"] = s.score, s.cost
-            elif parts[0] == "tag":
-                flat[f"tag/{parts[1]}/score"], flat[f"tag/{parts[1]}/cost"] = (
-                    s.score,
-                    s.cost,
-                )
-            else:  # task
-                t0 = parts[1]
-                # compute 95% CI half-width from stderr
-                flat.update(
-                    {
-                        f"task/{t0}/score": s.score,
-                        f"task/{t0}/score_ci": (
-                            (s.score_stderr * 1.96)
-                            if s.score_stderr is not None
-                            else np.nan
-                        ),
-                        f"task/{t0}/cost": s.cost,
-                        f"task/{t0}/cost_ci": (
-                            (s.cost_stderr * 1.96)
-                            if s.cost_stderr is not None
-                            else np.nan
-                        ),
-                    }
-                )
-
-        rows.append(
-            {
-                "agent_name": sub.agent_name,
-                "username": sub.username or "",
-                "submit_time": date,
-                **flat,
-                "logs_url": sub.logs_url if is_internal else sub.logs_url_public,
-            }
-        )
-
-    df = pd.DataFrame(rows)
-
-    # prepare pretty column mapping
-    pretty_cols = {c: _pretty_column_name(c) for c in df.columns}
-
-    # construct overview table with human-friendly names
-    overview = df.rename(columns=pretty_cols)
-
-    return overview
-
-
-def _pretty_column_name(col: str) -> str:
-    """Map raw column name to display name."""
-    # fixed mappings
-    mapping = {
-        "submit_time": "Date",
-        "agent_name": "Agent",
-        "username": "User/organization",
-        "logs_url": "Logs",
-        "overall/score": "Score",
-        "overall/cost": "Cost (USD)",
-    }
-    if col in mapping:
-        return mapping[col]
-    # dynamic: task/{name}/{metric} or tag/{name}/{metric}
-    parts = col.split("/")
-    if len(parts) == 3:
-        _, name, metric = parts
-        if metric == "score":
-            return name
-        if metric == "cost":
-            return f"{name} cost"
-        if metric == "score_ci":
-            return f"{name} 95% CI"
-        if metric == "cost_ci":
-            return f"{name} cost 95% CI"
-    # fallback to last segment
-    return parts[-1]
-
-
-
-def _plot_scatter(
-        data: pd.DataFrame,
-        x: str,  # Cost column name (e.g., "Overall cost")
-        y: str,  # Score column name (e.g., "Overall score")
-        agent_col: str,
-) -> plt.Figure:
-    """Scatter plot of agent results, showing score vs cost with Pareto frontier."""
-    fig, ax = plt.subplots(figsize=(20,7))
-
-    # Make a copy for manipulation to find frontier without affecting original data
-    plot_data = data.copy()
-
-    # Ensure score (y) and cost (x) are numeric and drop NaNs for frontier calculation
-    plot_data[y] = pd.to_numeric(plot_data[y], errors='coerce')
-    plot_data[x] = pd.to_numeric(plot_data[x], errors='coerce')
-    frontier_data = plot_data.dropna(subset=[y, x])
-
-    if not frontier_data.empty:
-        # Sort by cost (x) ascending, then by score (y) descending for tie-breaking
-        frontier_data = frontier_data.sort_values(by=[x, y], ascending=[True, False])
-
-        pareto_points = []
-        max_score_at_cost = -np.inf  # Initialize with negative infinity
-
-        for index, row in frontier_data.iterrows():
-            current_score = row[y]
-            current_cost = row[x]
-            # Only add point if it offers a higher score than any previous point
-            # on the frontier with less or equal cost (implicit by sorting).
-            # More strictly, for a point to be on the frontier here, it must improve the score.
-            if current_score > max_score_at_cost:
-                # Optional: If allowing same score but lower cost (already handled by sort somewhat)
-                # you might need to check if a point with same score but lower cost exists
-                # For this algorithm, we simply take points that strictly increase score.
-                pareto_points.append(row)
-                max_score_at_cost = current_score
-
-        if pareto_points:
-            pareto_df = pd.DataFrame(pareto_points)
-            # Sort pareto_df by cost again just to be sure for plotting line
-            pareto_df = pareto_df.sort_values(by=x)
-            # Plot the Pareto frontier line
-            ax.plot(pareto_df[x], pareto_df[y], marker='o', linestyle='-', color='red', alpha=0.7, linewidth=2, markersize=5, label='Pareto Frontier')
-
-    # Plot all data points
-    sns.scatterplot(data=data, x=x, y=y, hue=agent_col, s=100, ax=ax, legend="auto")
-
-    # Error bars (if they exist)
-    x_ci_col = f"{x} 95% CI"
-    y_ci_col = f"{y} 95% CI"
-    if x_ci_col in data.columns or y_ci_col in data.columns:
-        # Filter data for error bars to only include rows present in the original 'data'
-        # This is important if 'frontier_data' subset was used for some logic but error bars are for all.
-        error_bar_data = data.copy() # Use original data for error bars
-        error_bar_data[x_ci_col] = pd.to_numeric(error_bar_data.get(x_ci_col), errors='coerce')
-        error_bar_data[y_ci_col] = pd.to_numeric(error_bar_data.get(y_ci_col), errors='coerce')
-
-        ax.errorbar(
-            x=error_bar_data[x], # Use original data's x
-            y=error_bar_data[y], # Use original data's y
-            xerr=error_bar_data.get(x_ci_col),
-            yerr=error_bar_data.get(y_ci_col),
-            fmt="none",
-            ecolor="gray",
-            alpha=0.5,
-            capsize=3,
-            zorder=0 # Draw error bars behind scatter points
-        )
-
-    ax.set_xlim(left=0)
-    ax.set_ylim(bottom=0) # Scores and costs are typically non-negative
-    ax.set_xlabel(x) # x is cost
-    ax.set_ylabel(y) # y is score
-
-    # Adjust legend: Get handles and labels from seaborn plot, then add frontier's
-    handles, labels = ax.get_legend_handles_labels()
-    # Check if "Pareto Frontier" was actually plotted and add its handle/label if so
-    if pareto_points and "Pareto Frontier" not in labels: # Avoid duplicate legend items
-        # Find the frontier line object to get its handle
-        frontier_line = next((line for line in ax.get_lines() if line.get_label() == 'Pareto Frontier'), None)
-        if frontier_line:
-            handles.append(frontier_line)
-            labels.append('Pareto Frontier')
-
-    ax.legend(handles=handles, labels=labels, title=agent_col, bbox_to_anchor=(1.02, 1), loc='upper left', borderaxespad=0.)
-
-    plt.tight_layout(rect=[0, 0, 0.85, 1])
-    return fig
-
-
-__all__ = ["LeaderboardViewer"]

literature_understanding.py

@@ -2,13 +2,19 @@ import gradio as gr
 import pandas as pd
 
 # Import our UI factories and the data loader
-from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data
-
+from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data, create_sub_navigation_bar
+from content import LIT_DESCRIPTION
 # Define the category for this page
 CATEGORY_NAME = "Literature Understanding"
 
 with gr.Blocks() as demo:
-    gr.Markdown(f"## {CATEGORY_NAME} Leaderboard Results")
+    gr.Markdown(f"## {CATEGORY_NAME} Aggregated")
+
+    validation_df, validation_tag_map = get_full_leaderboard_data("validation")
+    test_df, test_tag_map = get_full_leaderboard_data("test")
+    gr.Markdown(LIT_DESCRIPTION, elem_id="category-intro")
+    if validation_tag_map:
+        create_sub_navigation_bar(validation_tag_map, CATEGORY_NAME)
 
     # --- This page now has two main sections: Validation and Test ---
     with gr.Tabs():

main_page.py

@@ -1,341 +1,26 @@
 import matplotlib
 matplotlib.use('Agg')
-
-import os
-import shutil
-import tarfile
-import tempfile
-from datetime import datetime, timedelta, timezone
-from email.utils import parseaddr
-from pathlib import Path
-# from zoneinfo import ZoneInfo # LeaderboardViewer uses this, ensure it's available
-
 import gradio as gr
-import requests
-from agenteval import (
-    # compute_summary_statistics, # This will now be used by LeaderboardViewer
-    process_eval_logs,
-    upload_folder_to_hf,
-    upload_summary_to_hf,
-)
-from agenteval.models import EvalResult # Used by submission and LeaderboardViewer (implicitly)
-from agenteval.leaderboard.upload import sanitize_path_component
-from datasets import Dataset, DatasetDict, VerificationMode, load_dataset # load_dataset used by LV
-from datasets.data_files import EmptyDatasetError
-from huggingface_hub import HfApi
+
 
 from ui_components import create_leaderboard_display, get_full_leaderboard_data
 
 from content import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
-    INTRODUCTION_TEXT,
-    SUBMISSION_TEXT,
-    INTRO_PARAGRAPH,
-    SCATTER_DISCLAIMER,
-    format_error,
-    format_log,
-    format_warning,
+    INTRO_PARAGRAPH
 )
 
-# --- Constants and Configuration  ---
-LOCAL_DEBUG = not (os.environ.get("system") == "spaces")
-CONFIG_NAME = "1.0.0-dev1" # This corresponds to 'config' in LeaderboardViewer
-IS_INTERNAL = os.environ.get("IS_INTERNAL", "false").lower() == "true"
-
-OWNER = "allenai"
-PROJECT_NAME = "asta-bench" + ("-internal" if IS_INTERNAL else "")
-SUBMISSION_DATASET = f"{OWNER}/{PROJECT_NAME}-submissions"
-SUBMISSION_DATASET_PUBLIC = f"{OWNER}/{PROJECT_NAME}-submissions-public"
-CONTACT_DATASET = f"{OWNER}/{PROJECT_NAME}-contact-info"
-RESULTS_DATASET = f"{OWNER}/{PROJECT_NAME}-results" # This is the repo_id for LeaderboardViewer
-LEADERBOARD_PATH = f"{OWNER}/{PROJECT_NAME}-leaderboard"
-
-if LOCAL_DEBUG:
-    DATA_DIR = os.path.join(os.path.dirname(__file__), "data", CONFIG_NAME)
-else:
-    DATA_DIR = "/home/user/data/" + CONFIG_NAME
-EXTRACTED_DATA_DIR = os.path.join(DATA_DIR, "extracted")
-
-api = HfApi()
-MAX_UPLOAD_BYTES = 100 * 1024**2
-AGENTEVAL_MANIFEST_NAME = "agenteval.json"
-os.makedirs(EXTRACTED_DATA_DIR, exist_ok=True)
-
 # --- Global State for Viewers (simple caching) ---
 CACHED_VIEWERS = {}
 CACHED_TAG_MAPS = {}
 
-# --- Submission Logic (largely unchanged from original, ensure EvalResult and other deps are fine) ---
-def try_load_dataset_submission(*args, **kwargs) -> DatasetDict: # Renamed to avoid conflict if LV has one
-    try:
-        return load_dataset(*args, **kwargs)
-    except EmptyDatasetError:
-        return DatasetDict()
-    except ValueError: # Handles cases where dataset is empty or ill-formed
-        return DatasetDict()
-
-def checked_upload_folder(
-        api_hf: HfApi, # Renamed to avoid conflict with global api
-        folder_path: str,
-        repo_id: str,
-        config_name_ul: str, # Renamed
-        split_ul: str, # Renamed
-        submission_name_ul: str, # Renamed
-) -> str:
-    total = 0
-    for root, _, files in os.walk(folder_path):
-        for f_ul in files: # Renamed
-            total += os.path.getsize(os.path.join(root, f_ul))
-            if total > MAX_UPLOAD_BYTES:
-                raise ValueError(
-                    f"Upload too large: exceeds {MAX_UPLOAD_BYTES // (1024**2)} MB limit."
-                )
-    return upload_folder_to_hf(
-        api=api_hf, # Use renamed parameter
-        folder_path=folder_path,
-        repo_id=repo_id,
-        config_name=config_name_ul,
-        split=split_ul,
-        submission_name=submission_name_ul,
-    )
-
-def add_new_eval(
-        val_or_test: str,
-        agent_name: str | None,
-        agent_description: str,
-        agent_url: str,
-        openness: str | None,
-        degree_of_control: str | None,
-        path_to_file: tempfile._TemporaryFileWrapper | None,
-        username: str,
-        mail: str,
-        profile: gr.OAuthProfile,
-        # We need global eval_results for checks; this might need rethinking if it's purely display driven now
-        # For now, let's assume we still load it for submission checks
-):
-    # Load current eval_results for submission checks
-    # This is a bit redundant if display part reloads it, but submission needs its own consistent view
-    current_eval_results_for_submission = try_load_dataset_submission(
-        RESULTS_DATASET,
-        CONFIG_NAME,
-        download_mode="force_redownload", # Or a less aggressive mode
-        verification_mode=VerificationMode.NO_CHECKS,
-        trust_remote_code=True,
-    )
-    if not agent_name:
-        return format_warning("Please provide an agent name.")
-
-    submission_time = datetime.now(timezone.utc)
-    if not username or username.strip() == "":
-        username = profile.username # Default to HF username
-
-    # User account age check
-    try:
-        user_data_resp = requests.get(f"https://huggingface.co/api/users/{profile.username}/overview")
-        user_data_resp.raise_for_status()
-        creation_date_str = user_data_resp.json()["createdAt"]
-        created_at = datetime.strptime(creation_date_str, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
-        if submission_time - created_at < timedelta(days=60):
-            return format_error("This account is not authorized to submit here (account too new).")
-    except Exception as e:
-        print(f"Error checking user account age: {e}")
-        return format_error("Could not verify account age. Please try again later.")
-
-    # Submission frequency check
-    contact_infos = try_load_dataset_submission(
-        CONTACT_DATASET, CONFIG_NAME, download_mode="force_redownload",
-        verification_mode=VerificationMode.NO_CHECKS, trust_remote_code=True
-    )
-    user_submission_dates = sorted(
-        datetime.fromisoformat(row["submit_time"])
-        for row in contact_infos.get(val_or_test, []) if row["username_auth"] == profile.username
-    )
-    if user_submission_dates and (submission_time - user_submission_dates[-1] < timedelta(days=1)):
-        return format_error("You already submitted once in the last 24h for this split; please try again later.")
-
-    # Email validation
-    _, parsed_mail = parseaddr(mail)
-    if "@" not in parsed_mail:
-        return format_warning("Please provide a valid email address.")
-
-    # Duplicate submission check
-    if val_or_test in current_eval_results_for_submission and len(current_eval_results_for_submission[val_or_test]) > 0:
-        existing_submissions = current_eval_results_for_submission[val_or_test].to_dict().get("submission", [])
-        for sub_item in existing_submissions:
-            if (sub_item.get("agent_name", "").lower() == agent_name.lower() and
-                    sub_item.get("username", "").lower() == username.lower()):
-                return format_warning("This agent name by this user has already been submitted to this split.")
-
-    if path_to_file is None:
-        return format_warning("Please attach a .tar.gz file.")
-
-    safe_username = sanitize_path_component(username)
-    safe_agent_name = sanitize_path_component(agent_name)
-    extracted_dir = os.path.join(EXTRACTED_DATA_DIR, f"{safe_username}_{safe_agent_name}")
-
-    # File extraction
-    if not LOCAL_DEBUG:
-        try:
-            if os.path.exists(extracted_dir): shutil.rmtree(extracted_dir)
-            os.makedirs(extracted_dir, exist_ok=True)
-            with tarfile.open(path_to_file.name, "r:gz") as tar:
-                members_extracted = 0
-                for member in tar.getmembers():
-                    if not member.isreg(): continue
-                    fname = os.path.basename(member.name)
-                    if not fname or fname.startswith("."): continue
-                    fobj = tar.extractfile(member)
-                    if not fobj: continue
-                    with open(os.path.join(extracted_dir, fname), "wb") as out:
-                        out.write(fobj.read())
-                    members_extracted +=1
-                if members_extracted == 0:
-                    return format_error("Submission tarball is empty or contains no valid files.")
-        except Exception as e:
-            return format_error(f"Error extracting file: {e}. Ensure it's a valid .tar.gz.")
-    else: print("mock extracted file", flush=True)
-
-
-    submission_name = f"{safe_username}_{safe_agent_name}_{submission_time.strftime('%Y-%m-%d_%H-%M-%S')}"
-
-    # 1. Upload raw (unscored) submission files
-    if not LOCAL_DEBUG:
-        try:
-            checked_upload_folder(api, extracted_dir, SUBMISSION_DATASET, CONFIG_NAME, val_or_test, submission_name)
-        except ValueError as e: return format_error(str(e))
-        except Exception as e: return format_error(f"Failed to upload raw submission: {e}")
-    else: print("mock uploaded raw submission", flush=True)
-
-    # 2. Save contact information
-    contact_info = {
-        "agent_name": agent_name, "agent_description": agent_description, "url": agent_url,
-        "username": username, "username_auth": profile.username, "mail": mail,
-        "submit_time": submission_time.isoformat(),
-    }
-    if val_or_test in contact_infos:
-        contact_infos[val_or_test] = contact_infos[val_or_test].add_item(contact_info)
-    else:
-        contact_infos[val_or_test] = Dataset.from_list([contact_info])
-
-    if not LOCAL_DEBUG:
-        try:
-            contact_infos.push_to_hub(CONTACT_DATASET, config_name=CONFIG_NAME)
-        except Exception as e: return format_warning(f"Submission recorded, but contact info failed to save: {e}")
-    else: print("mock uploaded contact info", flush=True)
-
-
-    # 3. Process and score the submission
-    eval_result_obj = None # Define to avoid NameError
-    try:
-        json_path = Path(extracted_dir) / AGENTEVAL_MANIFEST_NAME
-        if not json_path.exists():
-            return format_error(f"Missing manifest {AGENTEVAL_MANIFEST_NAME} in submission.")
-
-        eval_result_obj = EvalResult.model_validate_json(json_path.read_text(encoding="utf-8"))
-        if eval_result_obj.suite_config.version != CONFIG_NAME:
-            return format_error(f"Suite version mismatch: expected {CONFIG_NAME}, got {eval_result_obj.suite_config.version}.")
-        if eval_result_obj.split != val_or_test:
-            return format_error(f"Split mismatch: expected {val_or_test}, got {eval_result_obj.split}.")
-
-        # Re-compute results from logs for integrity
-        eval_result_obj.results = process_eval_logs(extracted_dir)[0] # Assuming process_eval_logs returns a tuple/list
-        eval_result_obj.save_json(str(json_path)) # Save the re-processed manifest
-
-    except Exception as e:
-        return format_error(f"Error scoring submission: {e}. Check manifest and log files.")
-
-    # 4. Upload scored submission files
-    logs_url_private_val, logs_url_public_val = None, None
-    scored_submission_name = f"{submission_name}_scored"
-    if not LOCAL_DEBUG:
-        try:
-            logs_url_private_val = checked_upload_folder(api, extracted_dir, SUBMISSION_DATASET, CONFIG_NAME, val_or_test, scored_submission_name)
-            if val_or_test == "validation" and not IS_INTERNAL: # Public copy for validation
-                logs_url_public_val = checked_upload_folder(api, extracted_dir, SUBMISSION_DATASET_PUBLIC, CONFIG_NAME, val_or_test, scored_submission_name)
-        except ValueError as e: return format_error(str(e))
-        except Exception as e: return format_error(f"Failed to upload scored submission: {e}")
-    else: print("mock uploaded scored submission", flush=True)
-
-
-    # Update EvalResult with submission details
-    eval_result_obj.submission.agent_name = agent_name
-    eval_result_obj.submission.agent_description = agent_description
-    eval_result_obj.submission.agent_url = agent_url
-    eval_result_obj.submission.openness = openness
-    eval_result_obj.submission.degree_of_control = degree_of_control
-    eval_result_obj.submission.username = username
-    eval_result_obj.submission.submit_time = submission_time
-    eval_result_obj.submission.logs_url = logs_url_private_val
-    eval_result_obj.submission.logs_url_public = logs_url_public_val
-
-    # 5. Upload summary statistics to RESULTS_DATASET (for the leaderboard)
-    if not LOCAL_DEBUG:
-        try:
-            upload_summary_to_hf(api, eval_result_obj, RESULTS_DATASET, CONFIG_NAME, val_or_test, scored_submission_name)
-        except Exception as e:
-            return format_error(f"Failed to upload summary results to leaderboard: {e}")
-    else: print("mock uploaded results to lb", flush=True)
-
-    # Invalidate viewer cache for the split that was updated
-    if val_or_test in CACHED_VIEWERS:
-        del CACHED_VIEWERS[val_or_test]
-    if val_or_test in CACHED_TAG_MAPS:
-        del CACHED_TAG_MAPS[val_or_test]
-
-
-    return format_log(
-        f"Agent '{agent_name}' submitted successfully by '{username}' to '{val_or_test}' split. "
-        "Please refresh the leaderboard in a few moments. It may take some time for changes to propagate."
-    )
-
-with gr.Blocks() as demo:
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-    gr.HTML(INTRO_PARAGRAPH, elem_id="intro-paragraph")
-
-    # --- Submission Accordion ---
-    with gr.Accordion("🚀 Submit a new agent for evaluation", open=False, elem_classes="submission-accordion"):
-        gr.Markdown(SUBMISSION_TEXT, elem_id="markdown-text")
-        with gr.Row():
-            with gr.Column():
-                level_of_test_radio = gr.Radio(["validation", "test"], value="validation", label="Split")
-                agent_name_tb = gr.Textbox(label="Agent Name")
-                agent_desc_tb = gr.Textbox(label="Agent Description")
-                agent_url_tb = gr.Textbox(label="URL to Agent Information")
-                openness_radio = gr.Radio(["Open Source", "API", "UI"], value=None, label="Openness of Agent")
-                degree_of_control_radio = gr.Radio(["Standard", "Custom"], value=None, label="Degree of Control")
-            with gr.Column():
-                username_tb = gr.Textbox(label="Organization or User Name (Defaults to HF username)")
-                mail_tb = gr.Textbox(label="Contact Email (Private, for submission issues)")
-                file_upload_comp = gr.File(
-                    label="Submission File (.tar.gz ...)", # Shortened for brevity
-                    file_types=[".gz", ".tar.gz"]
-                )
-        with gr.Row():
-            gr.LoginButton()
-            submit_eval_button = gr.Button("Submit Evaluation")
-        submission_result = gr.Markdown()
-
-        submit_eval_button.click(
-            add_new_eval,
-            [
-                level_of_test_radio,
-                agent_name_tb,
-                agent_desc_tb,
-                agent_url_tb,
-                openness_radio,
-                degree_of_control_radio,
-                file_upload_comp,
-                username_tb,
-                mail_tb
-            ],
-            submission_result,
-        )
-
+with gr.Blocks(fill_width=True) as demo:
+    gr.Markdown(INTRO_PARAGRAPH, elem_id="intro-paragraph")
     # --- Leaderboard Display Section ---
     gr.Markdown("---")
     CATEGORY_NAME = "Overall"
-    gr.Markdown(f"## {CATEGORY_NAME} Leaderboard Results")
+    gr.Markdown(f"## {CATEGORY_NAME} Categories Aggregated")
 
     with gr.Tabs() as tabs:
         with gr.Tab("Results: Validation"):
@@ -352,7 +37,6 @@ with gr.Blocks() as demo:
                     split_name="validation"
                 )
             else:
-                # Display a message if no data is available
                 gr.Markdown("No data available for validation split.")
 
         with gr.Tab("Results: Test"):

requirements.txt

@@ -1,5 +1,131 @@
-datasets
-gradio[oauth]==5.30.0
-huggingface-hub
-APScheduler
-agent-eval==0.1.9
+agent-eval==0.1.13
+aiobotocore==2.22.0
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.18
+aioitertools==0.12.0
+aiosignal==1.3.2
+annotated-types==0.7.0
+anyio==4.9.0
+APScheduler==3.11.0
+async-timeout==5.0.1
+attrs==25.3.0
+Authlib==1.5.2
+beautifulsoup4==4.13.4
+black==25.1.0
+botocore==1.37.3
+certifi==2025.4.26
+cffi==1.17.1
+charset-normalizer==3.4.2
+click==8.1.8
+contourpy==1.3.2
+cryptography==44.0.3
+cycler==0.12.1
+datasets==3.6.0
+debugpy==1.8.14
+dill==0.3.8
+distro==1.9.0
+docstring_parser==0.16
+exceptiongroup==1.2.2
+fastapi==0.115.12
+ffmpy==0.5.0
+filelock==3.18.0
+fonttools==4.58.1
+frozenlist==1.6.0
+fsspec==2025.3.0
+gradio==5.30.0
+gradio_client==1.10.1
+groovy==0.1.2
+h11==0.16.0
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.30.2
+idna==3.10
+ijson==3.3.0
+importlib_metadata==8.7.0
+inspect_ai==0.3.94
+isort==6.0.1
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jiter==0.9.0
+jmespath==1.0.1
+jsonlines==4.0.0
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2025.4.1
+kiwisolver==1.4.8
+linkify-it-py==2.0.3
+litellm==1.68.1
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+matplotlib==3.10.3
+mdit-py-plugins==0.4.2
+mdurl==0.1.2
+mmh3==5.1.0
+mplcursors==0.6
+multidict==6.4.3
+multiprocess==0.70.16
+mypy_extensions==1.1.0
+narwhals==1.38.2
+nest-asyncio==1.6.0
+numpy==2.2.5
+openai==1.75.0
+orjson==3.10.18
+packaging==25.0
+pandas==2.2.3
+pathspec==0.12.1
+pillow==11.2.1
+platformdirs==4.3.7
+plotly==6.0.1
+propcache==0.3.1
+psutil==7.0.0
+pyarrow==20.0.0
+pycparser==2.22
+pydantic==2.11.4
+pydantic_core==2.33.2
+pydub==0.25.1
+Pygments==2.19.1
+pyparsing==3.2.3
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.0
+python-multipart==0.0.20
+pytz==2025.2
+PyYAML==6.0.2
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+rich==13.9.4
+rpds-py==0.24.0
+ruff==0.11.8
+s3fs==2025.3.0
+safehttpx==0.1.6
+seaborn==0.13.2
+semantic-version==2.10.0
+semver==3.0.4
+shellingham==1.5.4
+shortuuid==1.0.13
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.7
+starlette==0.46.2
+tenacity==9.1.2
+textual==3.2.0
+tiktoken==0.9.0
+tokenizers==0.21.1
+tomli==2.2.1
+tomlkit==0.13.2
+tqdm==4.67.1
+typer==0.15.3
+typing-inspection==0.4.0
+typing_extensions==4.13.2
+tzdata==2025.2
+tzlocal==5.3.1
+uc-micro-py==1.0.3
+urllib3==2.4.0
+uvicorn==0.34.2
+websockets==15.0.1
+wrapt==1.17.2
+xxhash==3.5.0
+yarl==1.20.0
+zipp==3.21.0

submission.py

@@ -0,0 +1,324 @@
+import matplotlib
+matplotlib.use('Agg')
+
+import os
+import shutil
+import tarfile
+import tempfile
+from datetime import datetime, timedelta, timezone
+from email.utils import parseaddr
+from pathlib import Path
+
+import gradio as gr
+import requests
+from agenteval import (
+    process_eval_logs,
+    upload_folder_to_hf,
+    upload_summary_to_hf,
+)
+from agenteval.models import EvalResult
+from agenteval.leaderboard.upload import sanitize_path_component
+from datasets import Dataset, DatasetDict, VerificationMode, load_dataset
+from datasets.data_files import EmptyDatasetError
+from huggingface_hub import HfApi
+
+from content import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    format_error,
+    format_log,
+    format_warning,
+)
+
+# --- Constants and Configuration  ---
+LOCAL_DEBUG = not (os.environ.get("system") == "spaces")
+CONFIG_NAME = "1.0.0-dev1" # This corresponds to 'config' in LeaderboardViewer
+IS_INTERNAL = os.environ.get("IS_INTERNAL", "false").lower() == "true"
+
+OWNER = "allenai"
+PROJECT_NAME = "asta-bench" + ("-internal" if IS_INTERNAL else "")
+SUBMISSION_DATASET = f"{OWNER}/{PROJECT_NAME}-submissions"
+SUBMISSION_DATASET_PUBLIC = f"{OWNER}/{PROJECT_NAME}-submissions-public"
+CONTACT_DATASET = f"{OWNER}/{PROJECT_NAME}-contact-info"
+RESULTS_DATASET = f"{OWNER}/{PROJECT_NAME}-results" # This is the repo_id for LeaderboardViewer
+LEADERBOARD_PATH = f"{OWNER}/{PROJECT_NAME}-leaderboard"
+
+if LOCAL_DEBUG:
+    DATA_DIR = os.path.join(os.path.dirname(__file__), "data", CONFIG_NAME)
+else:
+    DATA_DIR = "/home/user/data/" + CONFIG_NAME
+EXTRACTED_DATA_DIR = os.path.join(DATA_DIR, "extracted")
+
+api = HfApi()
+MAX_UPLOAD_BYTES = 100 * 1024**2
+AGENTEVAL_MANIFEST_NAME = "agenteval.json"
+os.makedirs(EXTRACTED_DATA_DIR, exist_ok=True)
+
+# --- Global State for Viewers (simple caching) ---
+CACHED_VIEWERS = {}
+CACHED_TAG_MAPS = {}
+
+# --- Submission Logic (largely unchanged from original, ensure EvalResult and other deps are fine) ---
+def try_load_dataset_submission(*args, **kwargs) -> DatasetDict: # Renamed to avoid conflict if LV has one
+    try:
+        return load_dataset(*args, **kwargs)
+    except EmptyDatasetError:
+        return DatasetDict()
+    except ValueError: # Handles cases where dataset is empty or ill-formed
+        return DatasetDict()
+
+def checked_upload_folder(
+        api_hf: HfApi, # Renamed to avoid conflict with global api
+        folder_path: str,
+        repo_id: str,
+        config_name_ul: str, # Renamed
+        split_ul: str, # Renamed
+        submission_name_ul: str, # Renamed
+) -> str:
+    total = 0
+    for root, _, files in os.walk(folder_path):
+        for f_ul in files: # Renamed
+            total += os.path.getsize(os.path.join(root, f_ul))
+            if total > MAX_UPLOAD_BYTES:
+                raise ValueError(
+                    f"Upload too large: exceeds {MAX_UPLOAD_BYTES // (1024**2)} MB limit."
+                )
+    return upload_folder_to_hf(
+        api=api_hf, # Use renamed parameter
+        folder_path=folder_path,
+        repo_id=repo_id,
+        config_name=config_name_ul,
+        split=split_ul,
+        submission_name=submission_name_ul,
+    )
+
+def add_new_eval(
+        val_or_test: str,
+        agent_name: str | None,
+        agent_description: str,
+        agent_url: str,
+        openness: str | None,
+        degree_of_control: str | None,
+        path_to_file: tempfile._TemporaryFileWrapper | None,
+        username: str,
+        mail: str,
+        profile: gr.OAuthProfile,
+        # We need global eval_results for checks; this might need rethinking if it's purely display driven now
+        # For now, let's assume we still load it for submission checks
+):
+    # Load current eval_results for submission checks
+    # This is a bit redundant if display part reloads it, but submission needs its own consistent view
+    current_eval_results_for_submission = try_load_dataset_submission(
+        RESULTS_DATASET,
+        CONFIG_NAME,
+        download_mode="force_redownload", # Or a less aggressive mode
+        verification_mode=VerificationMode.NO_CHECKS,
+        trust_remote_code=True,
+    )
+    if not agent_name:
+        return format_warning("Please provide an agent name.")
+
+    submission_time = datetime.now(timezone.utc)
+    if not username or username.strip() == "":
+        username = profile.username # Default to HF username
+
+    # User account age check
+    try:
+        user_data_resp = requests.get(f"https://huggingface.co/api/users/{profile.username}/overview")
+        user_data_resp.raise_for_status()
+        creation_date_str = user_data_resp.json()["createdAt"]
+        created_at = datetime.strptime(creation_date_str, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
+        if submission_time - created_at < timedelta(days=60):
+            return format_error("This account is not authorized to submit here (account too new).")
+    except Exception as e:
+        print(f"Error checking user account age: {e}")
+        return format_error("Could not verify account age. Please try again later.")
+
+    # Submission frequency check
+    contact_infos = try_load_dataset_submission(
+        CONTACT_DATASET, CONFIG_NAME, download_mode="force_redownload",
+        verification_mode=VerificationMode.NO_CHECKS, trust_remote_code=True
+    )
+    user_submission_dates = sorted(
+        datetime.fromisoformat(row["submit_time"])
+        for row in contact_infos.get(val_or_test, []) if row["username_auth"] == profile.username
+    )
+    if user_submission_dates and (submission_time - user_submission_dates[-1] < timedelta(days=1)):
+        return format_error("You already submitted once in the last 24h for this split; please try again later.")
+
+    # Email validation
+    _, parsed_mail = parseaddr(mail)
+    if "@" not in parsed_mail:
+        return format_warning("Please provide a valid email address.")
+
+    # Duplicate submission check
+    if val_or_test in current_eval_results_for_submission and len(current_eval_results_for_submission[val_or_test]) > 0:
+        existing_submissions = current_eval_results_for_submission[val_or_test].to_dict().get("submission", [])
+        for sub_item in existing_submissions:
+            if (sub_item.get("agent_name", "").lower() == agent_name.lower() and
+                    sub_item.get("username", "").lower() == username.lower()):
+                return format_warning("This agent name by this user has already been submitted to this split.")
+
+    if path_to_file is None:
+        return format_warning("Please attach a .tar.gz file.")
+
+    safe_username = sanitize_path_component(username)
+    safe_agent_name = sanitize_path_component(agent_name)
+    extracted_dir = os.path.join(EXTRACTED_DATA_DIR, f"{safe_username}_{safe_agent_name}")
+
+    # File extraction
+    if not LOCAL_DEBUG:
+        try:
+            if os.path.exists(extracted_dir): shutil.rmtree(extracted_dir)
+            os.makedirs(extracted_dir, exist_ok=True)
+            with tarfile.open(path_to_file.name, "r:gz") as tar:
+                members_extracted = 0
+                for member in tar.getmembers():
+                    if not member.isreg(): continue
+                    fname = os.path.basename(member.name)
+                    if not fname or fname.startswith("."): continue
+                    fobj = tar.extractfile(member)
+                    if not fobj: continue
+                    with open(os.path.join(extracted_dir, fname), "wb") as out:
+                        out.write(fobj.read())
+                    members_extracted +=1
+                if members_extracted == 0:
+                    return format_error("Submission tarball is empty or contains no valid files.")
+        except Exception as e:
+            return format_error(f"Error extracting file: {e}. Ensure it's a valid .tar.gz.")
+    else: print("mock extracted file", flush=True)
+
+
+    submission_name = f"{safe_username}_{safe_agent_name}_{submission_time.strftime('%Y-%m-%d_%H-%M-%S')}"
+
+    # 1. Upload raw (unscored) submission files
+    if not LOCAL_DEBUG:
+        try:
+            checked_upload_folder(api, extracted_dir, SUBMISSION_DATASET, CONFIG_NAME, val_or_test, submission_name)
+        except ValueError as e: return format_error(str(e))
+        except Exception as e: return format_error(f"Failed to upload raw submission: {e}")
+    else: print("mock uploaded raw submission", flush=True)
+
+    # 2. Save contact information
+    contact_info = {
+        "agent_name": agent_name, "agent_description": agent_description, "url": agent_url,
+        "username": username, "username_auth": profile.username, "mail": mail,
+        "submit_time": submission_time.isoformat(),
+    }
+    if val_or_test in contact_infos:
+        contact_infos[val_or_test] = contact_infos[val_or_test].add_item(contact_info)
+    else:
+        contact_infos[val_or_test] = Dataset.from_list([contact_info])
+
+    if not LOCAL_DEBUG:
+        try:
+            contact_infos.push_to_hub(CONTACT_DATASET, config_name=CONFIG_NAME)
+        except Exception as e: return format_warning(f"Submission recorded, but contact info failed to save: {e}")
+    else: print("mock uploaded contact info", flush=True)
+
+
+    # 3. Process and score the submission
+    eval_result_obj = None # Define to avoid NameError
+    try:
+        json_path = Path(extracted_dir) / AGENTEVAL_MANIFEST_NAME
+        if not json_path.exists():
+            return format_error(f"Missing manifest {AGENTEVAL_MANIFEST_NAME} in submission.")
+
+        eval_result_obj = EvalResult.model_validate_json(json_path.read_text(encoding="utf-8"))
+        if eval_result_obj.suite_config.version != CONFIG_NAME:
+            return format_error(f"Suite version mismatch: expected {CONFIG_NAME}, got {eval_result_obj.suite_config.version}.")
+        if eval_result_obj.split != val_or_test:
+            return format_error(f"Split mismatch: expected {val_or_test}, got {eval_result_obj.split}.")
+
+        # Re-compute results from logs for integrity
+        eval_result_obj.results = process_eval_logs(extracted_dir)[0] # Assuming process_eval_logs returns a tuple/list
+        eval_result_obj.save_json(str(json_path)) # Save the re-processed manifest
+
+    except Exception as e:
+        return format_error(f"Error scoring submission: {e}. Check manifest and log files.")
+
+    # 4. Upload scored submission files
+    logs_url_private_val, logs_url_public_val = None, None
+    scored_submission_name = f"{submission_name}_scored"
+    if not LOCAL_DEBUG:
+        try:
+            logs_url_private_val = checked_upload_folder(api, extracted_dir, SUBMISSION_DATASET, CONFIG_NAME, val_or_test, scored_submission_name)
+            if val_or_test == "validation" and not IS_INTERNAL: # Public copy for validation
+                logs_url_public_val = checked_upload_folder(api, extracted_dir, SUBMISSION_DATASET_PUBLIC, CONFIG_NAME, val_or_test, scored_submission_name)
+        except ValueError as e: return format_error(str(e))
+        except Exception as e: return format_error(f"Failed to upload scored submission: {e}")
+    else: print("mock uploaded scored submission", flush=True)
+
+
+    # Update EvalResult with submission details
+    eval_result_obj.submission.agent_name = agent_name
+    eval_result_obj.submission.agent_description = agent_description
+    eval_result_obj.submission.agent_url = agent_url
+    eval_result_obj.submission.openness = openness
+    eval_result_obj.submission.degree_of_control = degree_of_control
+    eval_result_obj.submission.username = username
+    eval_result_obj.submission.submit_time = submission_time
+    eval_result_obj.submission.logs_url = logs_url_private_val
+    eval_result_obj.submission.logs_url_public = logs_url_public_val
+
+    # 5. Upload summary statistics to RESULTS_DATASET (for the leaderboard)
+    if not LOCAL_DEBUG:
+        try:
+            upload_summary_to_hf(api, eval_result_obj, RESULTS_DATASET, CONFIG_NAME, val_or_test, scored_submission_name)
+        except Exception as e:
+            return format_error(f"Failed to upload summary results to leaderboard: {e}")
+    else: print("mock uploaded results to lb", flush=True)
+
+    # Invalidate viewer cache for the split that was updated
+    if val_or_test in CACHED_VIEWERS:
+        del CACHED_VIEWERS[val_or_test]
+    if val_or_test in CACHED_TAG_MAPS:
+        del CACHED_TAG_MAPS[val_or_test]
+
+
+    return format_log(
+        f"Agent '{agent_name}' submitted successfully by '{username}' to '{val_or_test}' split. "
+        "Please refresh the leaderboard in a few moments. It may take some time for changes to propagate."
+    )
+
+
+# --- Submission Accordion ---
+with gr.Blocks() as demo:
+    gr.Markdown(f"## 🚀 Submit a new agent for evaluation", elem_id="markdown-text")
+    with gr.Row():
+        with gr.Column():
+            level_of_test_radio = gr.Radio(["validation", "test"], value="validation", label="Split")
+            agent_name_tb = gr.Textbox(label="Agent Name")
+            agent_desc_tb = gr.Textbox(label="Agent Description")
+            agent_url_tb = gr.Textbox(label="URL to Agent Information")
+            openness_radio = gr.Radio(["Open Source","Open Source Open Weights", "API Available", "Closed"], value=None, label="Openness of Agent")
+            degree_of_control_radio = gr.Radio(["Standard","Custom with Standard Search", "Fully Custom"], value=None, label="Agent Tooling")
+        with gr.Column():
+            username_tb = gr.Textbox(label="Organization or User Name (Defaults to HF username)")
+            mail_tb = gr.Textbox(label="Contact Email (Private, for submission issues)")
+            file_upload_comp = gr.File(
+                label="Submission File (.tar.gz ...)", # Shortened for brevity
+                file_types=[".gz", ".tar.gz"]
+            )
+    with gr.Row():
+        gr.LoginButton()
+        submit_eval_button = gr.Button("Submit Evaluation")
+    submission_result = gr.Markdown()
+
+    submit_eval_button.click(
+        add_new_eval,
+        [
+            level_of_test_radio,
+            agent_name_tb,
+            agent_desc_tb,
+            agent_url_tb,
+            openness_radio,
+            degree_of_control_radio,
+            file_upload_comp,
+            username_tb,
+            mail_tb
+        ],
+        submission_result,
+    )
+    with gr.Accordion("📙 Citation", open=False):
+        gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, elem_id="citation-button-main", interactive=False)

ui_components.py

@@ -1,13 +1,22 @@
 import gradio as gr
-from gradio.events import SelectData
 import pandas as pd
 import plotly.graph_objects as go
 import os
+import re
 
 from agenteval.leaderboard.view import LeaderboardViewer
 from huggingface_hub import HfApi
 
-from leaderboard_transformer import DataTransformer, transform_raw_dataframe, create_pretty_tag_map, INFORMAL_TO_FORMAL_NAME_MAP, _plot_scatter_plotly, format_cost_column, format_score_column
+from leaderboard_transformer import (
+    DataTransformer,
+    transform_raw_dataframe,
+    create_pretty_tag_map,
+    INFORMAL_TO_FORMAL_NAME_MAP,
+    _plot_scatter_plotly,
+    format_cost_column,
+    format_score_column,
+    get_pareto_df,
+)
 from content import (
     SCATTER_DISCLAIMER,
     format_error,
@@ -19,7 +28,7 @@ from content import (
 
 # --- Constants and Configuration  ---
 LOCAL_DEBUG = not (os.environ.get("system") == "spaces")
-CONFIG_NAME = "1.0.0-dev1" # This corresponds to 'config' in LeaderboardViewer
+CONFIG_NAME = "1.0.0-dev2" # This corresponds to 'config' in LeaderboardViewer
 IS_INTERNAL = os.environ.get("IS_INTERNAL", "false").lower() == "true"
 
 OWNER = "allenai"
@@ -41,6 +50,24 @@ MAX_UPLOAD_BYTES = 100 * 1024**2
 AGENTEVAL_MANIFEST_NAME = "agenteval.json"
 os.makedirs(EXTRACTED_DATA_DIR, exist_ok=True)
 
+# Global variables
+openness_emoji_map = {
+    "Closed": '🔴',
+    "API Available": '🟠',
+    "Open Source": '🟢',
+    "Open Source + Open Weights": '🔵'
+}
+control_emoji_map = {
+    "Standard": "⭐",
+    "Custom with Standard Search": "🔶",
+    "Fully Custom": "⚪️",
+}
+legend_markdown = """
+    <span>On pareto curve:📈</span>
+    <span>**Agent Openness**:</span>   <span>🔴 Closed</span>    <span>🟠 API Available</span>    <span>🟢 Open Source</span>    <span>🔵 Open Source + Open Weights</span>
+    <span>**Agent Tooling**:</span>   <span>⭐ Standard</span>    <span>🔶 Custom with Standard Search</span>    <span>⚪️ Fully Custom</span>
+    <span>**COMING SOON:** COLUMN DESCRIPTIONS</span>
+    """
 
 # --- Global State for Viewers (simple caching) ---
 CACHED_VIEWERS = {}
@@ -117,29 +144,48 @@ def create_leaderboard_display(
     # The function no longer loads data itself; it filters the data it receives.
     transformer = DataTransformer(full_df, tag_map)
     df_view, plots_dict = transformer.view(tag=category_name, use_plotly=True)
-    # format cost columns
+    pareto_df = get_pareto_df(df_view)
+    # Get the list of agents on the frontier. We'll use this list later.
+    if not pareto_df.empty and 'id' in pareto_df.columns:
+        pareto_agent_names = pareto_df['id'].tolist()
+    else:
+        pareto_agent_names = []
+    df_view['Pareto'] = df_view.apply(
+        lambda row: '📈' if row['id'] in pareto_agent_names else '',
+        axis=1
+    )
+    # Create mapping for Openness
+    original_openness = df_view['Openness']
+    df_view['Openness'] = df_view['Openness'].map(openness_emoji_map).fillna(original_openness)
+
+    # For this column, we'll use .apply() to handle the "Other" case cleanly.
+    df_view['Agent Tooling'] = df_view['Agent Tooling'].apply(
+        lambda ctrl: control_emoji_map.get(ctrl, f"{ctrl}" if pd.notna(ctrl) else "")
+    )
+
+
+    # Format cost columns
     for col in df_view.columns:
         if "Cost" in col:
             df_view = format_cost_column(df_view, col)
 
-    # 2. Fill NaN scores with 0
+    # Fill NaN scores with 0
     for col in df_view.columns:
         if "Score" in col:
             df_view = format_score_column(df_view, col)
     scatter_plot = plots_dict.get('scatter_plot', go.Figure())
 
-    # 2. Define the UI components with the filtered data.
+
+    all_cols = df_view.columns.tolist()
+    # Remove 'Pareto' from the list and insert it at the beginning
+    all_cols.insert(0, all_cols.pop(all_cols.index('Pareto')))
+    df_view = df_view[all_cols]
+    # Drop internally used columns that are not needed in the display
+    columns_to_drop = ['id', 'agent_for_hover']
+    df_view = df_view.drop(columns=columns_to_drop, errors='ignore')
+
     df_headers = df_view.columns.tolist()
-    df_datatypes = ["markdown" if col == "Logs" or "Cost" in col or "Score" in col else "str" for col in df_headers]
-
-    dataframe_component = gr.DataFrame(
-        headers=df_headers,
-        value=df_view,
-        datatype=df_datatypes,
-        interactive=False,
-        wrap=True,
-        column_widths=[100, 100, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 75, 75, 50, 50]
-    )
+    df_datatypes = ["markdown" if col == "Logs" or col == "Agent" or "Cost" in col or "Score" in col else "str" for col in df_headers]
 
     plot_component = gr.Plot(
         value=scatter_plot,
@@ -147,8 +193,20 @@ def create_leaderboard_display(
     )
     gr.HTML(SCATTER_DISCLAIMER, elem_id="scatter-disclaimer")
 
+    # Put table and key into an accordion
+    with gr.Accordion("See Table", open=False, elem_id="leaderboard-accordion"):
+        dataframe_component = gr.DataFrame(
+            headers=df_headers,
+            value=df_view,
+            datatype=df_datatypes,
+            interactive=False,
+            wrap=True,
+            column_widths=[30, 30, 30, 100, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 50, 30]
+        )
+        gr.Markdown(value=legend_markdown, elem_id="legend-markdown")
+
     # Return the components so they can be referenced elsewhere.
-    return dataframe_component, plot_component
+    return plot_component, dataframe_component,
 
 def get_full_leaderboard_data(split: str) -> tuple[pd.DataFrame, dict]:
     """
@@ -178,8 +236,36 @@ def get_full_leaderboard_data(split: str) -> tuple[pd.DataFrame, dict]:
 
     # Fallback for unexpected types
     return pd.DataFrame(), {}
+# Create sub-nav bar for benchmarks
+def create_gradio_anchor_id(text: str) -> str:
+    """
+    Replicates the ID format created by gr.Markdown(header_links=True).
+    Example: "Paper Finder Validation" -> "h-paper-finder-validation"
+    """
+    text = text.lower()
+    text = re.sub(r'\s+', '-', text) # Replace spaces with hyphens
+    text = re.sub(r'[^\w-]', '', text) # Remove non-word characters
+    return f"h-{text}"
+def create_sub_navigation_bar(tag_map: dict, category_name: str):
+    """
+    Generates and renders the HTML for the anchor-link sub-navigation bar.
+    """
+    benchmark_names = tag_map.get(category_name, [])
+    if not benchmark_names:
+        return # Do nothing if there are no benchmarks
+
+    anchor_links = []
+    for name in benchmark_names:
+        # Use the helper function to create the correct ID format
+        target_id = create_gradio_anchor_id(name)
+        anchor_links.append(f"<a href='#{target_id}'>{name}</a>")
+
+    nav_bar_html = f"<div class='sub-nav-bar'>{'   '.join(anchor_links)}</div>"
 
-# --- Detailed Benchmark Display ---
+    # Use gr.HTML to render the links correctly
+    gr.HTML(nav_bar_html)
+
+# # --- Detailed Benchmark Display ---
 def create_benchmark_details_display(
         full_df: pd.DataFrame,
         tag_map: dict,
@@ -206,14 +292,14 @@ def create_benchmark_details_display(
     # 2. Loop through each benchmark and create its UI components
     for benchmark_name in benchmark_names:
         with gr.Blocks():
-            gr.Markdown(f"### {benchmark_name}")
+            gr.Markdown(f"### {benchmark_name}", header_links=True)
 
             # 3. Prepare the data for this specific benchmark's table and plot
             benchmark_score_col = f"{benchmark_name} Score"
             benchmark_cost_col = f"{benchmark_name} Cost"
 
             # Define the columns needed for the detailed table
-            table_cols = ['Agent', 'Submitter', 'Date', benchmark_score_col, benchmark_cost_col,'Logs']
+            table_cols = ['Agent','Openness','Agent Tooling', 'Submitter', 'Date', benchmark_score_col, benchmark_cost_col,'Logs','id']
 
             # Filter to only columns that actually exist in the full dataframe
             existing_table_cols = [col for col in table_cols if col in full_df.columns]
@@ -224,11 +310,29 @@ def create_benchmark_details_display(
 
             # Create a specific DataFrame for the table view
             benchmark_table_df = full_df[existing_table_cols].copy()
+            pareto_df = get_pareto_df(benchmark_table_df)
+            # Get the list of agents on the frontier. We'll use this list later.
+            if not pareto_df.empty and 'id' in pareto_df.columns:
+                pareto_agent_names = pareto_df['id'].tolist()
+            else:
+                pareto_agent_names = []
+            benchmark_table_df['Pareto'] = benchmark_table_df.apply(
+                lambda row: '📈' if row['id'] in pareto_agent_names else '',
+                axis=1
+            )
+
+            original_openness = benchmark_table_df['Openness']
+            benchmark_table_df['Openness'] = benchmark_table_df['Openness'].map(openness_emoji_map).fillna(original_openness)
+
+            # For this column, we'll use .apply() to handle the "Other" case cleanly.
+            benchmark_table_df['Agent Tooling'] = benchmark_table_df['Agent Tooling'].apply(
+                lambda ctrl: control_emoji_map.get(ctrl, f"{ctrl}" if pd.notna(ctrl) else "")
+            )
+
             # Calculated and add "Benchmark Attempted" column
             def check_benchmark_status(row):
                 has_score = pd.notna(row.get(benchmark_score_col))
                 has_cost = pd.notna(row.get(benchmark_cost_col))
-
                 if has_score and has_cost:
                     return "✅"
                 if has_score or has_cost:
@@ -246,14 +350,14 @@ def create_benchmark_details_display(
             benchmark_table_df = format_cost_column(benchmark_table_df, benchmark_cost_col)
             benchmark_table_df = format_score_column(benchmark_table_df, benchmark_score_col)
             desired_cols_in_order = [
+                'Pareto',
+                'Openness',
+                'Agent Tooling',
                 'Agent',
                 'Submitter',
                 'Attempted Benchmark',
                 benchmark_score_col,
                 benchmark_cost_col,
-                'Openness',
-                'Degree of Control',
-                'Date',
                 'Logs'
             ]
             for col in desired_cols_in_order:
@@ -261,25 +365,13 @@ def create_benchmark_details_display(
                     benchmark_table_df[col] = pd.NA # Add as an empty column
             benchmark_table_df = benchmark_table_df[desired_cols_in_order]
             # Rename columns for a cleaner table display, as requested
-            benchmark_table_df.rename(columns={
+            benchmark_table_df.rename({
                 benchmark_score_col: 'Score',
-                benchmark_cost_col: 'Cost'
+                benchmark_cost_col: 'Cost',
             }, inplace=True)
             # Ensure the 'Logs' column is formatted correctly
-            table_headers = benchmark_table_df.columns.tolist()
-            # If the column is 'Logs', render as markdown; otherwise, as a string.
-            df_datatypes = [
-                "markdown" if col in ["Logs", "Cost", "Score"] else "str"
-                for col in table_headers
-            ]
-
-            # Create the Gradio component, now with the correct datatypes
-            gr.DataFrame(
-                value=benchmark_table_df,
-                datatype=df_datatypes,
-                interactive=False,
-                wrap=True,
-            )
+            df_headers = benchmark_table_df.columns.tolist()
+            df_datatypes = ["markdown" if col == "Logs" or col == "Agent" or "Cost" in col or "Score" in col else "str" for col in df_headers]
 
             # Create the scatter plot using the full data for context, but plotting benchmark metrics
             # This shows all agents on the same axis for better comparison.
@@ -291,3 +383,14 @@ def create_benchmark_details_display(
             )
             gr.Plot(value=benchmark_plot)
             gr.HTML(SCATTER_DISCLAIMER, elem_id="scatter-disclaimer")
+            # Put table and key into an accordion
+            with gr.Accordion("See Table", open=False, elem_id="leaderboard-accordion"):
+                gr.DataFrame(
+                    headers=df_headers,
+                    value=benchmark_table_df,
+                    datatype=df_datatypes,
+                    interactive=False,
+                    wrap=True,
+                )
+                gr.Markdown(value=legend_markdown, elem_id="legend-markdown")
+