initial commit

.gitignore

@@ -3,11 +3,15 @@ venv/
 __pycache__/
 .env
 .ipynb_checkpoints
-*ipynb
 .vscode/
+.idea
 
 eval-queue/
 eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/
+data
+contact_info.jsonl
+.DS_Store
+runs

.pre-commit-config.yaml

@@ -1,53 +0,0 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-default_language_version:
-  python: python3
-
-ci:
-  autofix_prs: true
-  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
-  autoupdate_schedule: quarterly
-
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
-    hooks:
-      - id: check-yaml
-      - id: check-case-conflict
-      - id: detect-private-key
-      - id: check-added-large-files
-        args: ['--maxkb=1000']
-      - id: requirements-txt-fixer
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
-
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        name: Format imports
-
-  - repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-      - id: black
-        name: Format code
-        additional_dependencies: ['click==8.0.2']
-
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    # Ruff version.
-    rev: 'v0.0.267'
-    hooks:
-      - id: ruff

Makefile

@@ -1,13 +0,0 @@
-.PHONY: style format
-
-
-style:
-	python -m black --line-length 119 .
-	python -m isort .
-	ruff check --fix .
-
-
-quality:
-	python -m black --check --line-length 119 .
-	python -m isort --check-only .
-	ruff check .

README.md

@@ -1,44 +1,21 @@
 ---
-title: Demo Leaderboard
-emoji: 🥇
-colorFrom: green
+title: DABstep Leaderboard
+emoji: 🏆
+colorFrom: yellow
 colorTo: indigo
 sdk: gradio
 app_file: app.py
 pinned: true
 license: apache-2.0
+tags:
+- leaderboard
+short_description: DABstep Reasoning Benchmark Leaderboard
 ---
 
-# Start the configuration
+to run
 
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
+```
+gradio app.py
 ```
 
-Request files are created automatically by this tool.
-
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
-
-# Code logic for more complex edits
-
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+(assumes `HF_TOKEN` env var is set)

app.py

@@ -1,204 +1,87 @@
+import os
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
 
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+from apscheduler.schedulers.background import BackgroundScheduler
+from dabstep_benchmark.content import TITLE, INTRODUCTION_TEXT, SUBMISSION_TEXT, CITATION_BUTTON_TEXT, CITATION_BUTTON_LABEL
+from dabstep_benchmark.leaderboard import *
 
 
 def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+    HF_API.restart_space(repo_id=HF_LEADERBOARD)
+
+
+if __name__ == "__main__":
+    os.makedirs("data/task_scores", exist_ok=True)
+    refresh(only_leaderboard=False)
+
+    demo = gr.Blocks()
+    with demo:
+        gr.Markdown(TITLE)
+        gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+        leaderboard_table = gr.components.Dataframe(
+            value=generate_leaderboard_df(),
+            datatype=["markdown", "str", "str", "str", "markdown", "str", "str", "str"],
+            interactive=False,
+            column_widths=["20%"],
+            wrap=True,
+        )
+        # create a Gradio event listener that runs when the page is loaded to populate the dataframe
+        demo.load(lambda: generate_leaderboard_df(), None, leaderboard_table)
+
+        refresh_button = gr.Button("Refresh")
+        refresh_button.click(
+            refresh,
+            inputs=[
+                gr.Checkbox(value=True, visible=False)
+            ],
+            outputs=[
+                leaderboard_table,
+            ],
+        )
+        with gr.Row():
+            with gr.Accordion("📙 Citation", open=False):
+                citation_button = gr.Textbox(
+                    value=CITATION_BUTTON_TEXT,
+                    label=CITATION_BUTTON_LABEL,
+                    lines=len(CITATION_BUTTON_TEXT.split("\n")),
+                    elem_id="citation-button",
+                )  # .style(show_copy_button=True)
+
+        with gr.Accordion("Submit new agent answers for evaluation"):
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
+                gr.Markdown(SUBMISSION_TEXT, elem_classes="markdown-text")
             with gr.Row():
                 with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
+                    split = gr.Radio(["all"], value="all", label="Split", visible=False)
+                    agent_name_textbox = gr.Textbox(label="Agent name")
+                    model_family_textbox = gr.Textbox(label="Model family")
+                    system_prompt_textbox = gr.Textbox(label="System prompt example")
+                    repo_url_textbox = gr.Textbox(label="Repo URL with agent code")
                 with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                    organisation = gr.Textbox(label="Organisation")
+                    mail = gr.Textbox(
+                        label="Contact email (will be stored privately, & used if there is an issue with your submission)")
+                    file_output = gr.File()
 
-            submit_button = gr.Button("Submit Eval")
+            submit_button = gr.Button("Submit answers")
             submission_result = gr.Markdown()
             submit_button.click(
-                add_new_eval,
+                process_submission,
                 [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
+                    split,
+                    agent_name_textbox,
+                    model_family_textbox,
+                    repo_url_textbox,
+                    file_output,
+                    organisation,
+                    mail
                 ],
                 submission_result,
             )
 
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(restart_space, "interval", seconds=3600)
+    scheduler.start()
+    demo.launch(debug=True)
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()

baseline/README.md

@@ -0,0 +1,29 @@
+1. Setup environment
+```
+export MODEL_ID=openai/o3-mini
+export API_KEY=<your_api_key>
+pip install -r baseline/requirements.txt
+```
+
+2. Launch trace collector
+```
+python -m phoenix.server.main serve &
+```
+
+3. Launch baseline
+```
+# Runs 10 tasks from dev split
+python baseline/run.py --model-id $MODEL_ID --api-key $API_KEY --max-tasks 10 --split dev --concurrency 1
+
+# Run all tasks from the dev split
+python baseline/run.py --model-id $MODEL_ID --api-key $API_KEY --max-tasks -1 --split dev --concurrency 1
+
+# Run against 10 tasks from the full benchmark (default) split
+python baseline/run.py --model-id $MODEL_ID --api-key $API_KEY --max-tasks 10 --split default --concurrency 1
+
+# Run 10 tasks in parallel
+python baseline/run.py --model-id $MODEL_ID --api-key $API_KEY --max-tasks 10 --split default --concurrency 10
+
+# Run specific task ids
+python baseline/run.py --model-id $MODEL_ID --api-key $API_KEY --tasks-ids 49 5 1273 --split default --concurrency 3
+```

baseline/constants.py

@@ -0,0 +1,2 @@
+REPO_ID = "adyen/DABstep"
+ADDITIONAL_AUTHORIZED_IMPORTS = ["numpy", "pandas", "json", "csv", "glob", "markdown", "os"]

baseline/custom_agent.py

@@ -0,0 +1,6 @@
+from smolagents import CodeAgent
+
+class CustomCodeAgent(CodeAgent):
+
+    def initialize_system_prompt(self):
+        return self.system_prompt_template

baseline/custom_litellm.py

@@ -0,0 +1,29 @@
+from typing import Optional
+
+from smolagents import LiteLLMModel
+from tenacity import retry, stop_after_attempt, before_sleep_log, retry_if_exception_type, wait_exponential, wait_random
+import litellm
+import logging
+
+logging.basicConfig(level=logging.WARNING)
+logger = logging.getLogger(__name__)
+
+class LiteLLMModelWithBackOff(LiteLLMModel):
+    def __init__(self, max_tokens: Optional[int] = 1500, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.max_tokens = max_tokens
+
+    @retry(
+        stop=stop_after_attempt(450),
+        wait=wait_exponential(min=1, max=120, exp_base=2, multiplier=1) + wait_random(0, 5),
+        before_sleep=before_sleep_log(logger, logging.WARNING),
+        retry=retry_if_exception_type((
+                litellm.Timeout,
+                litellm.RateLimitError,
+                litellm.APIConnectionError,
+                litellm.InternalServerError
+        ))
+    )
+    def __call__(self, *args, **kwargs):
+        return super().__call__(max_tokens=self.max_tokens, *args, **kwargs)
+

baseline/prompts.py

@@ -0,0 +1,67 @@
+from smolagents.prompts import CODE_SYSTEM_PROMPT
+
+reasoning_llm_system_prompt = """You are an expert data analyst who can solve any task using code blobs. You will be given a task to solve as best as you can. 
+In the environment there exists data which will help you solve your data analyst task, this data is spread out across files in this directory: `{ctx_path}`.
+For each task you try to solve you will follow a hierarchy of workflows: a root-level task workflow and a leaf-level step workflow.
+There is one root-level workflow per task which will be composed of multiple leafs self-contained step workflows.
+
+When solving a task you must follow this root-level workflow: 'Explore' → 'Plan' → 'Execute' → 'Conclude'.
+Root Task Workflow:
+    1.  Explore: Perform data exploration on the environment in the directory `{ctx_path}` and become one with the data. Understand what is available, what can you do with such data and what limitations are there.
+    2.  Plan: Draft a high-level plan based on the results of the 'Explore' step.
+    3.  Execute: Execute and operationalize such plan you have drafted. If while executing such plan, it turns out to be unsuccessful start over from the 'Explore' step.
+    4.  Conclude: Based on the output of your executed plan, distil all findings into an answer for the proposed task to solve.
+
+In order to advance through the Root Task Workflow you will need to perform a series of steps, for each step you take you must follow this workflow: 'Thought:' → 'Code:' → 'Observation:'.
+Step Workflow:
+ 1. Thought: Explain your reasoning and the code you'll use.
+ 2. Code: Write Python code inside:
+Code:
+```py
+your_python_code
+```<end_code>
+    Use print() to retain important outputs.
+ 3. Observation: Review printed outputs before proceeding.
+
+Rules:
+ - ALWAYS check the `{ctx_path}` directory for relevant documentation or data before assuming information is unavailable.
+ - ALWAYS validate your assumptions with the available documentation before executing.
+ - IF AND ONLY IF you have exhausted all possibles solution plans you can come up with and still can not find a valid answer, then provide "Not Applicable" as a final answer.
+ - Use only defined variables and correct valid python statements.
+ - Avoid chaining long unpredictable code snippets in one step.
+ - Use python only when needed, and never re-generate the same python code if you already know is not helping you solve the task.
+ - Never create any notional variables in our code, as having these in your logs will derail you from the true variables.
+ - Imports and variables persist between executions.
+ - Solve the task yourself, don't just provide instructions.
+ - You can import from this list: {{authorized_imports}}
+ - Never try to import final_answer, you have it already!
+
+Available Tools:
+ - final_answer(answer: any): Use this tool to return the final solution, as in:
+Code:
+```py
+answer = df["result"].mean()
+final_answer(answer)
+```<end_code>
+"""
+
+chat_llm_system_prompt = CODE_SYSTEM_PROMPT
+
+reasoning_llm_task_prompt = """
+{question}
+
+You must follow these guidelines when you produce your final answer:
+{guidelines}
+
+Now Begin! If you solve the task correctly, you will receive a reward of $1,000,000.
+"""
+
+chat_llm_task_prompt = """You are an expert data analyst and you will answer factoid questions by referencing files in the data directory: `{ctx_path}`
+Don't forget to reference any documentation in the data dir before answering a question.
+
+Here is the question you need to answer: {question}
+
+Here are the guidelines you MUST follow when answering the question above: {guidelines}
+
+Before answering the question, reference any documentation in the data dir and leverage its information in your reasoning / planning.
+"""

baseline/requirements.txt

@@ -0,0 +1,7 @@
+-i https://pypi.org/simple
+datasets
+huggingface-hub
+smolagents
+litellm
+tenacity
+arize-phoenix

baseline/run.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+
+import yaml
+from opentelemetry.sdk.trace import TracerProvider
+
+from openinference.instrumentation.smolagents import SmolagentsInstrumentor
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+
+endpoint = "http://0.0.0.0:6006/v1/traces"
+trace_provider = TracerProvider()
+trace_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint)))
+
+SmolagentsInstrumentor().instrument(tracer_provider=trace_provider)
+
+import argparse
+import logging
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+import datasets
+import pandas as pd
+from dabstep_benchmark.utils import evaluate
+from smolagents.utils import console
+from utils import TqdmLoggingHandler
+from constants import REPO_ID
+from tqdm import tqdm
+from prompts import (
+    reasoning_llm_system_prompt,
+    reasoning_llm_task_prompt,
+    chat_llm_task_prompt,
+    chat_llm_system_prompt
+)
+from utils import (
+    is_reasoning_llm,
+    create_code_agent_with_chat_llm,
+    create_code_agent_with_reasoning_llm,
+    get_tasks_to_run,
+    append_answer,
+    append_console_output,
+    download_context
+)
+
+logging.basicConfig(level=logging.WARNING, handlers=[TqdmLoggingHandler()])
+logger = logging.getLogger(__name__)
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--concurrency", type=int, default=4)
+    parser.add_argument("--model-id", type=str, default="openai/o3-mini")
+    parser.add_argument("--experiment", type=str, default=None)
+    parser.add_argument("--max-tasks", type=int, default=-1)
+    parser.add_argument("--max-steps", type=int, default=10)
+    parser.add_argument("--tasks-ids", type=int, nargs="+", default=None)
+    parser.add_argument("--api-base", type=str, default=None)
+    parser.add_argument("--api-key", type=str, default=None)
+    parser.add_argument("--split", type=str, default="default", choices=["default", "dev"])
+    parser.add_argument("--timestamp", type=str, default=None)
+    return parser.parse_args()
+
+
+def run_single_task(
+        task: dict,
+        model_id: str,
+        api_base: str,
+        api_key: str,
+        ctx_path: str,
+        base_filename: Path,
+        is_dev_data: bool,
+        max_steps: int
+):
+    if is_reasoning_llm(model_id):
+        prompt = reasoning_llm_task_prompt.format(
+            question=task["question"],
+            guidelines=task["guidelines"]
+        )
+        agent = create_code_agent_with_reasoning_llm(model_id, api_base, api_key, max_steps, ctx_path)
+    else:
+        prompt = chat_llm_task_prompt.format(
+            ctx_path=ctx_path,
+            question=task["question"],
+            guidelines=task["guidelines"]
+        )
+        agent = create_code_agent_with_chat_llm(model_id, api_base, api_key, max_steps)
+
+    with console.capture() as capture:
+        answer = agent.run(prompt)
+
+    logger.warning(f"Task id: {task['task_id']}\tQuestion: {task['question']} Answer: {answer}\n{'=' * 50}")
+
+    answer_dict = {"task_id": str(task["task_id"]), "agent_answer": str(answer)}
+    answers_file = base_filename / "answers.jsonl"
+    logs_file = base_filename / "logs.txt"
+
+    if is_dev_data:
+        scores = evaluate(agent_answers=pd.DataFrame([answer_dict]), tasks_with_gt=pd.DataFrame([task]))
+        entry = {**answer_dict, "answer": task["answer"], "score": scores[0]["score"], "level": scores[0]["level"]}
+        append_answer(entry, answers_file)
+    else:
+        append_answer(answer_dict, answers_file)
+    append_console_output(capture.get(), logs_file)
+
+
+def main():
+    args = parse_args()
+    logger.warning(f"Starting run with arguments: {args}")
+
+    ctx_path = download_context(str(Path().resolve()))
+
+    runs_dir = Path().resolve() / "runs"
+    runs_dir.mkdir(parents=True, exist_ok=True)
+    timestamp = time.time() if not args.timestamp else args.timestamp
+    base_filename = runs_dir / f"{args.model_id.replace('/', '_').replace('.', '_')}/{args.split}/{int(timestamp)}"
+
+    # save config
+    os.makedirs(base_filename, exist_ok=True)
+    with open(base_filename / "config.yaml", "w", encoding="utf-8") as f:
+        if is_reasoning_llm(args.model_id):
+            args.system_prompt = reasoning_llm_system_prompt
+        else:
+            args.system_prompt = chat_llm_system_prompt
+        args_dict = vars(args)
+        yaml.dump(args_dict, f, default_flow_style=False)
+
+    # Load dataset with user-chosen split
+    data = datasets.load_dataset(REPO_ID, name="tasks", split=args.split, download_mode='force_redownload')
+
+    if args.max_tasks >= 0 and args.tasks_ids is not None:
+        logger.error(f"Can not provide {args.max_tasks=} and {args.tasks_ids=} at the same time")
+    total = len(data) if args.max_tasks < 0 else min(len(data), args.max_tasks)
+
+    tasks_to_run = get_tasks_to_run(data, total, base_filename, args.tasks_ids)
+    with ThreadPoolExecutor(max_workers=args.concurrency) as exe:
+        futures = [
+            exe.submit(
+                run_single_task,
+               task,
+               args.model_id,
+               args.api_base,
+               args.api_key,
+               ctx_path,
+               base_filename,
+               (args.split == "dev"),
+               args.max_steps
+            )
+            for task in tasks_to_run
+        ]
+        for f in tqdm(as_completed(futures), total=len(tasks_to_run), desc="Processing tasks"):
+            f.result()
+
+    logger.warning("All tasks processed.")
+
+
+if __name__ == "__main__":
+    main()

baseline/utils.py

@@ -0,0 +1,109 @@
+import json
+from tqdm import tqdm
+import logging
+import threading
+from smolagents import CodeAgent
+from custom_agent import CustomCodeAgent
+from custom_litellm import LiteLLMModelWithBackOff
+from huggingface_hub import hf_hub_download
+from constants import REPO_ID, ADDITIONAL_AUTHORIZED_IMPORTS
+from pathlib import Path
+from prompts import reasoning_llm_system_prompt, chat_llm_system_prompt
+
+append_answer_lock = threading.Lock()
+append_console_output_lock = threading.Lock()
+
+class TqdmLoggingHandler(logging.Handler):
+    def emit(self, record):
+        tqdm.write(self.format(record))
+
+def read_only_open(*a, **kw):
+    if (len(a) > 1 and isinstance(a[1], str) and a[1] != 'r') or kw.get('mode', 'r') != 'r':
+        raise Exception("Only mode='r' allowed for the function open")
+    return open(*a, **kw)
+
+def download_context(base_dir: str) -> str:
+    ctx_files = [
+        "data/context/acquirer_countries.csv",
+        "data/context/payments.csv",
+        "data/context/merchant_category_codes.csv",
+        "data/context/fees.json",
+        "data/context/merchant_data.json",
+        "data/context/manual.md",
+        "data/context/payments-readme.md"
+    ]
+    for f in ctx_files:
+        hf_hub_download(REPO_ID, repo_type="dataset", filename=f, local_dir=base_dir, force_download=True)
+
+    root_dir = Path(__file__).resolve().parent.parent
+    full_path = Path(base_dir) / Path(ctx_files[0]).parent
+    relative_path = full_path.relative_to(root_dir)
+    return str(relative_path)
+
+def is_reasoning_llm(model_id: str) -> bool:
+    reasoning_llm_list = [
+        "openai/o1",
+        "openai/o3",
+        "openai/o3-mini",
+        "deepseek/deepseek-reasoner"
+    ]
+    return model_id in reasoning_llm_list
+
+def get_tasks_to_run(data, total: int, base_filename: Path, tasks_ids: list[int]):
+    import json
+    f = base_filename.parent / f"{base_filename.stem}_answers.jsonl"
+    done = set()
+    if f.exists():
+        with open(f, encoding="utf-8") as fh:
+            done = {json.loads(line)["task_id"] for line in fh if line.strip()}
+
+    tasks = []
+    for i in range(total):
+        task_id = int(data[i]["task_id"])
+        if task_id not in done:
+            if tasks_ids is not None:
+                if task_id in tasks_ids:
+                    tasks.append(data[i])
+            else:
+                tasks.append(data[i])
+    return tasks
+
+
+def append_answer(entry: dict, jsonl_file: Path) -> None:
+    jsonl_file.parent.mkdir(parents=True, exist_ok=True)
+    with append_answer_lock, open(jsonl_file, "a", encoding="utf-8") as fp:
+        fp.write(json.dumps(entry) + "\n")
+
+
+def append_console_output(captured_text: str, txt_file: Path) -> None:
+    txt_file.parent.mkdir(parents=True, exist_ok=True)
+    with append_console_output_lock, open(txt_file, "a", encoding="utf-8") as fp:
+        fp.write(captured_text + "\n")
+
+def create_code_agent_with_reasoning_llm(model_id: str, api_base=None, api_key=None, max_steps=10, ctx_path=None):
+    agent = CustomCodeAgent(
+        system_prompt=reasoning_llm_system_prompt,
+        tools=[],
+        model=LiteLLMModelWithBackOff(
+            model_id=model_id, api_base=api_base, api_key=api_key, max_tokens=None, max_completion_tokens=3000),
+        additional_authorized_imports=ADDITIONAL_AUTHORIZED_IMPORTS,
+        max_steps=max_steps,
+        verbosity_level=3,
+    )
+    agent.python_executor.static_tools.update({"open": read_only_open})
+
+    agent.system_prompt = agent.system_prompt.format(ctx_path=ctx_path)
+    return agent
+
+def create_code_agent_with_chat_llm(model_id: str, api_base=None, api_key=None, max_steps=10):
+    agent = CodeAgent(
+        system_prompt=chat_llm_system_prompt,
+        tools=[],
+        model=LiteLLMModelWithBackOff(model_id=model_id, api_base=api_base, api_key=api_key, max_tokens=3000),
+        additional_authorized_imports=ADDITIONAL_AUTHORIZED_IMPORTS,
+        max_steps=max_steps,
+        verbosity_level=3,
+    )
+
+    agent.python_executor.static_tools.update({"open": read_only_open})
+    return agent

dabstep_benchmark/content.py

@@ -0,0 +1,35 @@
+TITLE = """# 🏅 DABStep Leaderboard"""
+
+INTRODUCTION_TEXT = """
+The Data Agent Benchmark for Multi-step Reasoning (DABStep) is looking to measure and push the state-of-the-art in Data Analysis by LLMs.
+The benchmark is composed of ~450 data analysis questions ([Dataset Link](https://huggingface.co/datasets/adyen/data-agents-benchmark)) centered around 1 or more documents that agents will have to understand and cross reference in order to answer correctly.
+
+We have set up a notebook to quickly get an agent baseline using the free Huggingface Inference API: [Colab Notebook](https://colab.research.google.com/drive/1pXi5ffBFNJQ5nn1111SnIfjfKCOlunxu)
+"""
+
+SUBMISSION_TEXT = """
+## Submissions
+Scores are expressed as the percentage of correct answers. 
+
+Each question calls for an answer that is either a string (one or a few words), a number, or a comma separated list of strings or floats, unless specified otherwise. There is only one correct answer. 
+Hence, evaluation is done via quasi exact match between a model’s answer and the ground truth (up to some normalization that is tied to the “type” of the ground truth).
+
+
+We expect submissions to be json-line files with the following format. 
+Mandatory fields are: `task_id` and `agent_answer`. However, `reasoning_trace` is optional:
+```
+{"task_id": "task_id_1", "agent_answer": "Answer 1 from your agent", "reasoning_trace": "The different steps by which your model reached answer 1"}
+{"task_id": "task_id_2", "agent_answer": "Answer 2 from your agent", "reasoning_trace": "The different steps by which your model reached answer 2"}
+```
+
+Our scoring function can be found [here](https://huggingface.co/spaces/adyen/data-agents-benchmark/blob/main/data_agents_benchmark/evaluation/scorer.py).
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{data_agents_benchmark_2025,
+      title={Data Agents Benchmark}, 
+      author={Martin Iglesias, Alex Egg, Friso Kingma},
+      year={2025},
+      month={January},
+      url={TBD}
+}"""

dabstep_benchmark/evaluation/scorer.py

@@ -0,0 +1,129 @@
+import re
+from typing import Union
+import math
+from difflib import SequenceMatcher
+
+def is_numeric_with_commas(value: str) -> bool:
+    # Check if the string is a number with comma separators
+    return bool(re.match(r'^\$?(\d{1,3}(,\d{3})*(\.\d+)?|\.\d+)$', value.strip()))
+
+def question_scorer(input1: str, input2: str) -> bool:
+    # Remove leading/trailing whitespace and convert to lowercase
+    input1 = input1.strip().lower()
+    input2 = input2.strip().lower()
+
+    # Check if inputs are numeric with commas
+    if is_numeric_with_commas(input1) or is_numeric_with_commas(input2):
+        num1 = extract_numeric(input1)
+        num2 = extract_numeric(input2)
+        return compare_numeric(num1, num2) if num1 is not None and num2 is not None else False
+
+    # Check for list match
+    if ';' in input1 or ';' in input2 or ',' in input1 or ',' in input2:
+        return compare_lists(input1, input2)
+
+    # Extract numeric values if present
+    num1 = extract_numeric(input1)
+    num2 = extract_numeric(input2)
+
+    # If both inputs have numeric values, compare them
+    if num1 is not None and num2 is not None:
+        return compare_numeric(num1, num2)
+
+    # Check for string match or subset
+    return compare_strings(input1, input2)
+
+def extract_numeric(value: str) -> Union[float, None]:
+    # Remove commas and currency symbols from the value string
+    value = value.replace(',', '').replace('$', '')
+    
+    # Extract the first occurrence of a numeric value (including percentages and leading decimal point)
+    match = re.search(r'(\d*\.\d+|\d+\.?\d*)%?', value)
+    if match:
+        num_str = match.group(1)
+
+        # @martini: now ground truths are expressed in % not in ratios anymore so no need for this
+        ## Handle percentages
+        # if '%' in value:
+        #     try:
+        #         return float(num_str) / 100
+        #     except ValueError:
+        #         return None
+        # else:
+        #     try:
+        #         return float(num_str)
+        #     except ValueError:
+        #         return None
+        try:
+            return float(num_str)
+        except ValueError:
+            return None
+    return None
+
+def compare_numeric(num1: float, num2: float) -> bool:
+    # Check for exact equality first
+    if num1 == num2:
+        return True
+
+    # For percentages and small numbers, use a more lenient comparison
+    if num1 < 1 and num2 < 1:
+        return math.isclose(num1, num2, rel_tol=1e-2, abs_tol=1e-4)
+
+    # For larger numbers, use the original comparison method
+    dec_places1 = len(str(num1).split('.')[-1]) if '.' in str(num1) else 0
+    dec_places2 = len(str(num2).split('.')[-1]) if '.' in str(num2) else 0
+    round_to = min(dec_places1, dec_places2)
+    rounded1 = round(num1, round_to)
+    rounded2 = round(num2, round_to)
+
+    if rounded1 == rounded2:
+        return True
+
+    return math.isclose(num1, num2, rel_tol=1e-2, abs_tol=1e-2)
+
+def compare_strings(str1: str, str2: str) -> bool:
+    # Remove all whitespace and punctuation
+    clean1 = re.sub(r'[^\w]', '', str1)
+    clean2 = re.sub(r'[^\w]', '', str2)
+    
+    if clean1 == clean2:
+        return True
+
+    words1 = re.findall(r'\b\w+\b', str1.lower())
+    words2 = re.findall(r'\b\w+\b', str2.lower())
+
+    # Only do subset comparison if neither list is empty
+    if (len(words1) == 1 or len(words2) == 1) and words1 and words2:
+        return set(words1).issubset(set(words2)) or set(words2).issubset(set(words1))
+
+    # Debugging: Log similarity score
+    similarity = SequenceMatcher(None, str1, str2).ratio()
+
+    return similarity > 0.95
+
+def compare_lists(list1: str, list2: str) -> bool:
+    # Normalize list representations by removing brackets
+    list1 = re.sub(r'^\[|\]$', '', list1.strip())
+    list2 = re.sub(r'^\[|\]$', '', list2.strip())
+
+    # Split the lists and remove whitespace
+    items1 = [item.strip() for item in re.split(r'[,;]', list1) if item.strip()]
+    items2 = [item.strip() for item in re.split(r'[,;]', list2) if item.strip()]
+
+    # Sort the items to handle different order
+    items1.sort()
+    items2.sort()
+
+    # Check if the lists are identical
+    if items1 == items2:
+        return True
+
+    # If lists are not identical, compare each item
+    if len(items1) != len(items2):
+        return False
+
+    for item1, item2 in zip(items1, items2):
+        if not question_scorer(item1, item2):
+            return False
+
+    return True

dabstep_benchmark/leaderboard.py

@@ -0,0 +1,310 @@
+import re
+import gradio as gr
+import json
+import datetime
+from email.utils import parseaddr
+
+import pandas as pd
+from datasets import load_dataset
+from huggingface_hub import HfApi
+
+from dabstep_benchmark.utils import format_log, format_error, format_warning, is_valid_https_url, evaluate
+
+OWNER = "adyen"
+
+HF_API = HfApi()
+HF_LEADERBOARD = f"{OWNER}/DABstep"
+HF_DATASET_PATH = f"{OWNER}/DABstep"
+HF_INTERNAL_DATASET_PATH = f"{OWNER}/DABstep-internal"
+HF_DATASET_CONFIGS = [
+    "tasks",
+    "submissions",
+    "task_scores"
+]
+DATASETS = {}
+
+def refresh(only_leaderboard: bool = False):
+    if only_leaderboard:
+        for config_name in ["task_scores", "submissions"]:
+            DATASETS[f"{config_name}"] = load_dataset(
+                path=HF_DATASET_PATH,
+                name=config_name,
+                split="default",
+            )
+            print(f"Downloaded {HF_DATASET_PATH}/{config_name}")
+
+    else:
+        for config_name in HF_DATASET_CONFIGS:
+            DATASETS[f"{config_name}"] = load_dataset(
+                path=HF_DATASET_PATH,
+                name=config_name,
+                split="default",
+            )
+            print(f"Downloaded {HF_DATASET_PATH}/{config_name}")
+
+        DATASETS["internal_tasks"] = load_dataset(
+            path=HF_INTERNAL_DATASET_PATH,
+            name="tasks",
+            split="default",
+        )
+        print(f"Downloaded {HF_INTERNAL_DATASET_PATH}/tasks")
+        DATASETS["contact_info"] = load_dataset(
+            path=HF_INTERNAL_DATASET_PATH,
+            name="contact_info",
+            split="default",
+        )
+        print(f"Downloaded {HF_INTERNAL_DATASET_PATH}/contact_info")
+
+    return generate_leaderboard_df()
+
+
+def validate_submission(submission_df: pd.DataFrame):
+    # mandatory_columns = ["agent_answer", "task_id", "num_steps"]
+    mandatory_columns = ["agent_answer", "task_id"]
+    expected_columns = [*mandatory_columns, "reasoning_trace"]
+
+    # Check for missing mandatory columns
+    missing_columns = [col for col in mandatory_columns if col not in submission_df.columns]
+    if missing_columns:
+        return format_error(f"Missing mandatory columns: {', '.join(missing_columns)}")
+
+    # Check for unexpected columns
+    unexpected_columns = [col for col in submission_df.columns if col not in expected_columns]
+    if unexpected_columns:
+        return format_error(f"Unexpected columns: {', '.join(unexpected_columns)}")
+
+    # Check for NaN values in any column
+    if submission_df.isnull().values.any():
+        return format_error("Submission contains NaN values. Please ensure no missing data.")
+
+    # Check if all columns are of string type
+    non_string_columns = [col for col in submission_df.columns if submission_df[col].dtype != 'object']
+    if non_string_columns:
+        return format_error(f"Columns with non-string data type: {', '.join(non_string_columns)}")
+
+    return None  # No errors
+
+def process_submission(
+        split: str,
+        agent_name: str,
+        model_family: str,
+        repo_url: str,
+        path_to_file: str,
+        organisation: str,
+        mail: str,
+):
+    if agent_name == "":
+        return format_warning("Please provide an agent name")
+    if organisation == "":
+        return format_warning("Please provide an organisation")
+    if mail == "":
+        return format_warning("Please provide an email")
+    if model_family == "":
+        return format_warning("Please provide a model family")
+
+    allowed_pattern = re.compile(r'^[a-zA-Z0-9 _.-]+$')
+    if not allowed_pattern.match(agent_name):
+        return format_warning(
+            f"{agent_name=} can only contain alphanumeric characters, spaces, dashes (-), and underscores (_)")
+
+    if not allowed_pattern.match(organisation):
+        return format_warning(
+            f"{organisation=} can only contain alphanumeric characters, spaces, dashes (-), and underscores (_)")
+
+
+    # very basic email parsing
+    _, parsed_mail = parseaddr(mail)
+    if not "@" in parsed_mail:
+        return format_warning("Please provide a valid email address.")
+
+    if repo_url != "" and not is_valid_https_url(repo_url):
+        return format_warning("If you provide a URL it must be a valid one. You can also leave it empty")
+
+    # submission file validation
+    if path_to_file == None:
+        return format_warning("Please attach a file.")
+    submission_path = path_to_file.name
+    try:
+        submission_df = pd.read_json(submission_path, lines=True, dtype=str)
+        validation_error = validate_submission(submission_df)
+        if validation_error:
+            return validation_error
+    except Exception as exc:
+        return format_error(f"Submission file is incorrectly formatted. Please fix it and resubmit your file. {str(exc)}")
+
+
+    print(f"Processing submission_id={organisation}-{agent_name}...")
+    gr.Info(f"Processing submission of {agent_name}...")
+    refresh(only_leaderboard=False)
+    submissions_df = DATASETS["submissions"].to_pandas()
+    contact_info_df = DATASETS["contact_info"].to_pandas()
+    internal_tasks_df = DATASETS["internal_tasks"].to_pandas()
+
+
+    # check if this agent already was submitted
+    submission_id = f"{organisation}-{agent_name}"
+    if submission_id in submissions_df['submission_id'].values:
+        return format_warning(f"This {submission_id} pair has been already submitted.")
+
+    # process submission
+    submission_df["submission_id"] = submission_id
+    submission_df["agent_name"] = agent_name
+    submission_df["model_family"] = model_family
+    submission_df["organisation"] = organisation
+    submission_df["repo_url"] = repo_url
+    submission_df["date"] = datetime.date.today().strftime("%d-%m-%Y")
+
+    # add empty reasoning trace if one is not provided to not break schema of datasets
+    if "reasoning_trace" not in submission_df.columns:
+        submission_df["reasoning_trace"] = ""
+
+    # overwrite submission
+    submission_df.to_json(submission_path, orient="records", lines=True)
+
+    try:
+        task_scores = evaluate(
+            agent_answers=submission_df,
+            tasks_with_gt=internal_tasks_df,
+            submission_id=submission_id
+        )
+    except KeyError as exc:
+        return format_error(str(exc))
+
+
+    # save submitted file once evaluation has run correctly
+    filename_id = f"v1__{organisation}-{agent_name}__{datetime.datetime.today().strftime('%d-%m-%Y')}"
+    path_in_repo = f"data/submissions/{filename_id}.jsonl"
+    HF_API.upload_file(
+        repo_id=HF_DATASET_PATH,
+        path_or_fileobj=submission_path,
+        path_in_repo=path_in_repo,
+        repo_type="dataset",
+    )
+    print(f"[submission_id={organisation}-{agent_name}] Pushed submission to {HF_DATASET_PATH}/{path_in_repo} !")
+
+    # write scores to disk
+    with open(f"data/task_scores/{filename_id}.jsonl", "w") as f:
+        for score in task_scores:
+            f.write(json.dumps(score) + "\n")
+
+    # upload scores to hub dataset
+    path_in_repo = f"data/task_scores/{filename_id}.jsonl"
+    HF_API.upload_file(
+        repo_id=HF_DATASET_PATH,
+        path_or_fileobj=f"data/task_scores/{filename_id}.jsonl",
+        path_in_repo=path_in_repo,
+        repo_type="dataset",
+    )
+    print(f"[submission_id={organisation}-{agent_name}] Pushed task_scores to {HF_DATASET_PATH}/{path_in_repo} !")
+
+    # if we already have this email dont save its metadata
+    if mail not in contact_info_df["mail"].values:
+        contact_info = {
+            "submission_id": submission_id,
+            "agent_name": agent_name,
+            "model_family": model_family,
+            "repo_url": repo_url,
+            "organisation": organisation,
+            "mail": mail,
+            "date": datetime.date.today().strftime("%d-%m-%Y"),
+        }
+        contact_info_df = pd.concat([contact_info_df, pd.DataFrame([contact_info])], ignore_index=True)
+        contact_info_df.to_json("contact_info.jsonl", orient="records", lines=True)
+
+        HF_API.upload_file(
+            repo_id=HF_INTERNAL_DATASET_PATH,
+            path_or_fileobj="contact_info.jsonl",
+            path_in_repo="contact_info.jsonl",
+            repo_type="dataset",
+        )
+        print(f"[submission_id={organisation}-{agent_name}] Pushed contact_info to {HF_INTERNAL_DATASET_PATH}/contact_info.jsonl !")
+
+
+    return format_log(
+        f"""
+        Agent {agent_name} submitted by {organisation} successfully.
+        Please refresh the leaderboard to see your score displayed.
+""")
+
+def generate_leaderboard_df() -> pd.DataFrame:
+    task_scores_df = DATASETS["task_scores"].to_pandas()
+    submissions_df = DATASETS["submissions"].to_pandas()
+
+    # get metadata of each submssion_id
+    submissions_df = (
+        submissions_df.groupby("submission_id")
+        .first()
+        .reset_index()[
+            [
+                "submission_id",
+                "agent_name",
+                "model_family",
+                "organisation",
+                "repo_url",
+                "date"
+            ]
+        ]
+    )
+
+    # make num_steps a number
+    # task_scores_df["num_steps"] = pd.to_numeric(task_scores_df["num_steps"], errors="coerce")
+
+    # group scores per submission
+    leaderboard_df = (
+        task_scores_df.groupby(["submission_id", "level"])
+        .agg(
+            avg_score=("score", "mean"),
+            # avg_num_steps=("num_steps", "mean")
+        )
+        .reset_index()
+    )
+
+    # reshape
+    # leaderboard_df = leaderboard_df.pivot(index="submission_id", columns="level", values=["avg_score", "avg_num_steps"])
+    leaderboard_df = leaderboard_df.pivot(index="submission_id", columns="level", values=["avg_score"])
+    leaderboard_df.columns = [f"{metric}_lvl_{level}" for metric, level in leaderboard_df.columns]
+    leaderboard_df = leaderboard_df.reset_index()
+
+    # leaderboard_df["overall_avg_steps"] = (
+    #         leaderboard_df.get("avg_num_steps_lvl_1", 0) +
+    #         leaderboard_df.get("avg_num_steps_lvl_2", 0) +
+    #         leaderboard_df.get("avg_num_steps_lvl_3", 0)
+    # )
+    # leaderboard_df["overall_avg_steps"] = leaderboard_df["overall_avg_steps"] / 3
+
+    # join scores and submission metadata
+    leaderboard_df = pd.merge(submissions_df, leaderboard_df, on="submission_id", how="inner")
+
+    # renaming
+    col_map = {
+        "agent_name": "Agent",
+        "avg_score_lvl_easy": "Easy Level Accuracy (%)",
+        "avg_score_lvl_hard": "Hard Level Accuracy (%)",
+        # "overall_avg_steps": "Overall Avg Reasoning Steps",
+        # "avg_num_steps_lvl_1": "Level 1 Avg Reasoning Steps",
+        # "avg_num_steps_lvl_2": "Level 2 Avg Reasoning Steps",
+        # "avg_num_steps_lvl_3": "Level 3 Avg Reasoning Steps",
+        "organisation": "Organization",
+        "repo_url": "Repo URL",
+        "model_family": "Model Family",
+        "date": "Date"
+    }
+    col_order = [new_col_name for new_col_name in col_map.values()]
+    leaderboard_df.rename(columns=col_map, inplace=True)
+    df = leaderboard_df[col_order].copy()
+
+    # formatting
+    # convert scores to %
+    df["Easy Level Accuracy (%)"] = df["Easy Level Accuracy (%)"].apply(lambda x: round(x * 100, 2))
+    df["Hard Level Accuracy (%)"] = df["Hard Level Accuracy (%)"].apply(lambda x: round(x * 100, 2))
+
+    # make repo url clickable in markdown
+    df["Repo URL"] = df["Repo URL"].apply(lambda x: f"[Link]({x})" if x != "" else x)
+
+    # make agent name bold
+    df["Agent"] = df["Agent"].apply(lambda x: f"**{x}**")
+
+    # sort-by best score
+    df.sort_values(by="Hard Level Accuracy (%)", ascending=False, inplace=True)
+
+    return df

dabstep_benchmark/tests/test_scorer.py

@@ -0,0 +1,90 @@
+import pytest
+from ..evaluation.scorer import question_scorer
+
+@pytest.mark.parametrize("input1, input2, expected", [
+    ("42", "42", True),
+    ("$42.00", "42", True),
+    ("43", "42", False),
+    ("10,765", "10765", True),
+    ("22520", "22520", True)
+])
+def test_numeric_match(input1, input2, expected):
+    assert question_scorer(input1, input2) == expected
+
+@pytest.mark.parametrize("input1, input2, expected", [
+    ("hello world", "Hello World", True),
+    (" sea  gull ", "seagull", True),
+    ("hello", "world", False),
+    ("Other", "Other", True),
+    #("A. Netherlands", "B. Netherlands", False), #ignoring as outlier
+    ("", "Inditex", False),
+    (" ", "Inditex", False),
+    ("The average transaction amount is 91.85 EUR.", "91.852", True),
+    ("The top 2 merchants account for approx 59.96% of all transactions.", "0.5996", False),
+    ("Netherlands", "NL", False),
+    ("", "", True) # @martini
+])
+def test_string_match(input1, input2, expected):
+    assert question_scorer(input1, input2) == expected
+
+@pytest.mark.parametrize("input1, input2, expected", [
+    ("1, 2, 3", "1,2,3", True),
+    ("apple; banana; cherry", "apple;banana;cherry", True),
+    ("1, 2", "1, 2, 3", False),
+    ("apple; banana", "apple; banana; cherry", False),
+    ("uber, spotify, nike, netflix, inditex", "Nike, Netflix, Uber, Inditex, Spotify", True),
+    ("a, b, c", "['a', 'b', 'c']", True),
+    (
+        "C: 69.36, F: 77.90, B: 86.22, A: 87.79, D: 94.34, G: 115.23",
+        "[C: 69.36, F: 77.9, B: 86.22, A: 87.79, D: 94.34, G: 115.23]",
+        True
+    ),
+    (
+        "[BE: 85.3, IT: 93.82, FR: 98.57, NL: 99.87, LU: 111.42, SE: 114.36, ES: 134.41, GR: 169.92, ]",
+        "BE: 85.3, IT: 93.82, FR: 98.57, NL: 99.87, LU: 111.42, SE: 114.36, ES: 134.41, GR: 169.92",
+        True
+    )
+])
+def test_list_match(input1, input2, expected):
+    assert question_scorer(input1, input2) == expected
+
+@pytest.mark.parametrize("input1, input2, expected", [
+    ("42, hello", "42, hello", True),
+    ("42, world", "42, hello", False),
+])
+def test_mixed_list_match(input1, input2, expected):
+    assert question_scorer(input1, input2) == expected
+
+@pytest.mark.parametrize("input1, input2, expected", [
+    ("3.14", "3.1483", True),
+    ("3.14", "3.20", False),
+    ("1", "1.0", True),
+    ("1.0", "1", True),
+    ("0.731495413640441", "0.731495", True),
+    ("C", "C) both ip_address and email_address", True),
+    ("0.36706256984345176", "0.3670625698434518", True),
+    ("$0.10", "$0.10 per retry", True),
+    ("D", "D) Apples", True),
+    ("D", "A) Oranges", False),
+    ("25.0", "0.250", False) #input is not a percentage
+])
+def test_approximate_numeric_match(input1, input2, expected):
+    assert question_scorer(input1, input2) == expected
+    
+@pytest.mark.parametrize("input1, input2, expected", [
+    ("73.15%", "73.1495", True),
+    ("42%", "42", True),
+    ("30%", "30.1", True),
+    ("25", "25%", True),
+    ("100%", "100", True),
+    ("0.1%", "0.1", True),
+    ("73%", "74", False),  # This should fail as the difference is too large
+    ("90%", "89.99971063977545", True),
+    ("7.79%", "7.787407043027865", True)
+    # ("7.787407 %", "0.07787407043027865", True) #TODO FIX
+
+])
+def test_percentages_match(input1, input2, expected):
+    assert question_scorer(input1, input2) == expected
+
+

dabstep_benchmark/utils.py

@@ -0,0 +1,59 @@
+import re
+import pandas as pd
+from dabstep_benchmark.evaluation.scorer import question_scorer
+
+
+def format_error(msg):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def format_warning(msg):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def format_log(msg):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>"
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+def is_valid_https_url(url):
+    pattern = re.compile(
+        r'^https://'  # URL must start with 'https://'
+        r'(?!10(?:\.\d{1,3}){3})'  # Exclude private IP 10.x.x.x
+        r'(?!127(?:\.\d{1,3}){3})'  # Exclude loopback IP 127.x.x.x
+        r'(?!169\.254(?:\.\d{1,3}){2})'  # Exclude link-local IP 169.254.x.x
+        r'(?!192\.168(?:\.\d{1,3}){2})'  # Exclude private IP 192.168.x.x
+        r'(?!172\.(?:1[6-9]|2[0-9]|3[0-1])(?:\.\d{1,3}){2})'  # Exclude private IP 172.16.x.x - 172.31.x.x
+        r'(?:(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,})'  # Match domain name
+        r'(?::\d{2,5})?'  # Optional port
+        r'(?:/[^\s]*)?$',  # Optional path
+        re.IGNORECASE
+    )
+    return re.match(pattern, url) is not None
+
+
+def evaluate(agent_answers: pd.DataFrame, tasks_with_gt: pd.DataFrame, submission_id: str = ""):
+    task_scores = []
+    for index, row in tasks_with_gt.iterrows():
+          correct_answer = row["answer"]
+          level = str(row["level"])
+          task_id = str(row["task_id"])
+
+          if task_id not in agent_answers["task_id"].values:
+              raise KeyError(f"Task ID: {task_id} not found. Are you sure you submitted the correct file?")
+
+          agent_answer = agent_answers.loc[agent_answers.task_id == task_id, "agent_answer"].values[0]
+          # num_steps = agent_answers.loc[agent_answers.task_id == task_id, "num_steps"].values[0]
+          score = question_scorer(agent_answer, correct_answer)
+
+          task_scores.append(
+              {
+                  "submission_id": submission_id,
+                  "task_id": task_id,
+                  "score": score,
+                  "level": level,
+                  "agent_answer": agent_answer,
+                  # "num_steps": num_steps,
+              }
+          )
+
+    return task_scores

pyproject.toml

@@ -1,13 +0,0 @@
-[tool.ruff]
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
-line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
-
-[tool.isort]
-profile = "black"
-line_length = 119
-
-[tool.black]
-line-length = 119

requirements.txt

@@ -1,16 +1,6 @@
-APScheduler
-black
 datasets
-gradio
-gradio[oauth]
-gradio_leaderboard==0.0.9
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
+huggingface-hub
 numpy
-pandas
-python-dateutil
-tqdm
-transformers
-tokenizers>=0.15.0
-sentencepiece
+fastapi==0.112.2
+gradio==5.0.0b1
+APScheduler

setup.py

@@ -0,0 +1,34 @@
+from setuptools import setup, find_packages
+
+# Read dependencies from requirements.txt
+def parse_requirements(filename):
+    not_needed_deps = [
+       "fastapi==0.112.2",
+       "gradio==5.0.0b1",
+       "APScheduler",
+    ]
+    dependencies = []
+    with open(filename) as f:
+        for line in f:
+            dep_name = line.strip()
+            if dep_name not in not_needed_deps:
+                dependencies.append(dep_name)
+    return dependencies
+
+setup(
+    name="DABstep Benchmark",
+    version="0.1.0",
+    description="DABstep: Data Agent Benchmark for Multi-Step Reasoning",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    author="Martin Iglesias, Alex Egg, Andreu Mora, Friso Kingma, Leandro von Werra",
+    author_email="martin.iglesiasgoyanes@adyen.com",
+    packages=find_packages(include=["dabstep_benchmark", "dabstep_benchmark.*"]),
+    include_package_data=True,
+    install_requires=parse_requirements("requirements.txt"),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+)

src/about.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-Intro text
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""

src/display/css_html_js.py

@@ -1,105 +0,0 @@
-custom_css = """
-
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#models-to-add-text {
-    font-size: 18px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#leaderboard-table-lite {
-    margin-top: 15px
-}
-
-#search-bar-table-box > div:first-child {
-    background: none;
-    border: none;
-}
- 
-#search-bar {
-    padding: 0px;
-}
-
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-table td:first-child,
-table th:first-child {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
-}
-
-#scale-logo .download {
-    display: none;
-}
-#filter_type{
-    border: 0;
-    padding-left: 0;
-    padding-top: 0;
-}
-#filter_type label {
-    display: flex;
-}
-#filter_type label > span{
-    margin-top: var(--spacing-lg);
-    margin-right: 0.5em;
-}
-#filter_type label > .wrap{
-    width: 103px;
-}
-#filter_type label > .wrap .wrap-inner{  
-    padding: 2px;
-}
-#filter_type label > .wrap .wrap-inner input{
-    width: 1px
-}
-#filter-columns-type{
-    border:0;
-    padding:0.5;
-}
-#filter-columns-size{
-    border:0;
-    padding:0.5;
-}
-#box-filter > .form{
-    border: 0
-}
-"""
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,110 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/envs.py

@@ -1,25 +0,0 @@
-import os
-
-from huggingface_hub import HfApi
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
-
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,196 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/populate.py

@@ -1,58 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )