Update data of ARC-AGI and Simple Bench; Add Codeforces and PlanBench

app.py

@@ -159,81 +159,12 @@ def create_size_for_performance_plot(category_to_display: str,
             gr.Dropdown(choices=list(elo_ratings_for_category.keys()), value=model_to_compare, interactive=True))
 
 
-def create_arc_agi_plot() -> go.Figure:
-    arc_agi_leaderboard = []
-    with open("arc_agi_leaderboard.jsonl", 'r') as file:
-        for line in file:
-            arc_agi_leaderboard.append(json.loads(line))
-
-    models = []
-    with open("models.jsonl", 'r') as file:
-        for line in file:
-            models.append(json.loads(line))
-
-    data = []
-    for entry in arc_agi_leaderboard:
-        model_name = entry['model']
-        score = entry['score']
-        model_info = next((m for m in models if m['Name'] == model_name), None)
-        if model_info:
-            release_date = datetime.strptime(model_info['Release Date'], "%Y-%m-%d")
-            data.append({'model': model_name, 'score': score, 'release_date': release_date})
-        else:
-            print(f"[WARNING] Model '{model_name}' not found in models.jsonl")
-
-    data.sort(key=lambda x: x['release_date'])
-
-    x_dates = [d['release_date'] for d in data]
-    y_scores = []
-    max_score = 0
-    for entry in data:
-        if entry['score'] > max_score:
-            max_score = entry['score']
-        y_scores.append(max_score)
-
-    fig = go.Figure()
-
-    fig.add_trace(go.Scatter(
-        x=x_dates,
-        y=y_scores,
-        mode='lines',
-        line=dict(shape='hv', width=2),
-        name='ARC-AGI Score'
-    ))
-
-    for i, entry in enumerate(data):
-        if i == 0 or y_scores[i] > y_scores[i - 1]:
-            fig.add_trace(go.Scatter(
-                x=[entry['release_date']],
-                y=[entry['score']],
-                mode='markers+text',
-                marker=dict(size=10),
-                text=[entry['model']],
-                textposition="top center",
-                name=entry['model']
-            ))
-
-    fig.update_layout(
-        title='ARC-AGI Score Progression Over Time',
-        xaxis_title='Release Date',
-        yaxis_title='ARC-AGI Score',
-        hovermode='x unified',
-        xaxis=dict(
-            range=[date(2024, 5, 13), date(2024, 9, 17)],
-            type='date'
-        ),
-        yaxis=dict(
-            range=[0, 100]
-        ),
-        height=800
-    )
-
-    return fig
-
-
-def create_simple_bench_plot() -> go.Figure:
+def create_simple_plot(data_path: str,
+                       name: str,
+                       start_date: datetime, end_date: datetime,
+                       min_value: int = 0, max_value: int = 100) -> go.Figure:
     simple_bench_leaderboard = []
-    with open("simple_bench_leaderboard.jsonl", 'r') as file:
+    with open(data_path, 'r') as file:
         for line in file:
             simple_bench_leaderboard.append(json.loads(line))
 
@@ -270,7 +201,7 @@ def create_simple_bench_plot() -> go.Figure:
         y=y_scores,
         mode='lines',
         line=dict(shape='hv', width=2),
-        name='Simple Bench Score'
+        name='Best Score to Date'
     ))
 
     for i, entry in enumerate(data):
@@ -286,16 +217,16 @@ def create_simple_bench_plot() -> go.Figure:
             ))
 
     fig.update_layout(
-        title='Simple Bench Score Progression Over Time',
+        title=f'{name} Over Time',
         xaxis_title='Release Date',
-        yaxis_title='Simple Bench Score',
+        yaxis_title=name,
         hovermode='x unified',
         xaxis=dict(
-            range=[date(2023, 6, 13), date(2024, 8, 14)],
+            range=[start_date, end_date],
             type='date'
         ),
         yaxis=dict(
-            range=[0, 100]
+            range=[min_value, max_value]
         ),
         height=800
     )
@@ -333,22 +264,28 @@ with gr.Blocks() as demo:
         with gr.Tab("API Cost for Specific Performance Level", interactive=False):
             api_cost_for_performance_plot: gr.Plot = gr.Plot()
     with gr.Tab("System Performance Over Time"):
-        with gr.Tab("ARC-AGI") as arc_agi_tab:
+        with gr.Tab("ARC-AGI-Pub") as arc_agi_tab:
             arc_agi_plot: gr.Plot = gr.Plot()
         with gr.Tab("Simple Bench") as simple_bench_tab:
             simple_bench_plot: gr.Plot = gr.Plot()
+        with gr.Tab("PlanBench") as planbench_tab:
+            planbench_plot: gr.Plot = gr.Plot()
+            planbench_markdown: gr.Markdown = gr.Markdown(
+                value="""Source: [Valmeekam et al. 2024](https://arxiv.org/abs/2409.13373)"""
+            )
+        with gr.Tab("Codeforces") as codeforces_tab:
+            with gr.Tab("General-Purpose Systems"):
+                codeforces_plot: gr.Plot = gr.Plot()
         with gr.Tab("BigCodeBench", interactive=False):
             bigcodebench_plot: gr.Plot = gr.Plot()
-        with gr.Tab("Codeforces", interactive=False):
-            codeforces_plot: gr.Plot = gr.Plot()
         with gr.Tab("GAIA", interactive=False):
             gaia_plot: gr.Plot = gr.Plot()
         with gr.Tab("GPQA", interactive=False):
             gpqa_plot: gr.Plot = gr.Plot()
         with gr.Tab("HumanEval", interactive=False):
             humaneval_plot: gr.Plot = gr.Plot()
-        with gr.Tab("LMSYS", interactive=False):
-            lmsys_plot: gr.Plot = gr.Plot()
+        with gr.Tab("Chatbot Arena", interactive=False):
+            chatbot_arena_plot: gr.Plot = gr.Plot()
         with gr.Tab("MATH", interactive=False):
             math_plot: gr.Plot = gr.Plot()
         with gr.Tab("OpenCompass", interactive=False):
@@ -374,8 +311,23 @@ with gr.Blocks() as demo:
                                       outputs=[size_for_performance_plot,
                                                size_for_performance_category_dropdown,
                                                size_for_performance_comparison_model_dropdown])
-    arc_agi_tab.select(fn=create_arc_agi_plot, outputs=arc_agi_plot)
-    simple_bench_tab.select(fn=create_simple_bench_plot, outputs=simple_bench_plot)
+    arc_agi_tab.select(fn=create_simple_plot,
+                       inputs=[gr.State("arc_agi_leaderboard.jsonl"), gr.State("ARC-AGI-Pub (Public Eval) Score"),
+                               gr.State(date(2024, 5, 13)), gr.State(date(2024, 12, 20))],
+                       outputs=arc_agi_plot)
+    simple_bench_tab.select(fn=create_simple_plot,
+                            inputs=[gr.State("simple_bench_leaderboard.jsonl"), gr.State("Simple Bench Score"),
+                                    gr.State(date(2023, 6, 13)), gr.State(date(2024, 8, 14))],
+                            outputs=simple_bench_plot)
+    codeforces_tab.select(fn=create_simple_plot,
+                          inputs=[gr.State("codeforces_leaderboard.jsonl"), gr.State("Codeforces (Elo Rating)"),
+                                  gr.State(date(2024, 5, 13)), gr.State(date(2024, 12, 20)),
+                                  gr.State(800), gr.State(3000)],
+                          outputs=codeforces_plot)
+    planbench_tab.select(fn=create_simple_plot,
+                         inputs=[gr.State("planbench_leaderboard.jsonl"), gr.State("PlanBench (Mystery Blocksworld, 0-shot) Score"),
+                                 gr.State(date(2023, 3, 14)), gr.State(date(2024, 9, 23))],
+                         outputs=planbench_plot)
 
 
 if __name__ == "__main__":

arc_agi_leaderboard.jsonl

@@ -1,3 +1,4 @@
+{"model": "o3", "score": 82.8}
 {"model": "o1-preview-2024-09-12", "score": 21}
 {"model": "claude-3-5-sonnet-20240620", "score": 21}
 {"model": "o1-mini-2024-09-12", "score": 13}

codeforces_leaderboard.jsonl

@@ -0,0 +1,6 @@
+{"model": "o3", "score": 2727}
+{"model": "o3-mini", "score": 2073}
+{"model": "o1", "score": 1673}
+{"model": "o1-mini", "score": 1650}
+{"model": "o1-preview", "score": 1258}
+{"model": "gpt-4o", "score": 808}

models.jsonl

@@ -1,5 +1,10 @@
+{"Name": "o3", "Release Date": "2024-12-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "o3-mini", "Release Date": "2024-12-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "o1", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "o1-preview-2024-09-12", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "o1-preview", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "o1-mini-2024-09-12", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "o1-mini", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "deepseek-v2.5", "Release Date": "2024-09-05", "Total Parameters": 236, "Active Parameters": 236, "API Cost": 0}
 {"Name": "qwen-plus-0828", "Release Date": "2024-08-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "qwen-plus-0828", "Release Date": "2024-08-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
@@ -11,6 +16,7 @@
 {"Name": "command-r-08-2024", "Release Date": "2024-08-19", "Total Parameters": 32, "Active Parameters": 32, "API Cost": 0}
 {"Name": "gemini-1.5-pro-exp-0801", "Release Date": "2024-08-01", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "grok-2-2024-08-13", "Release Date": "2024-08-13", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gpt-4o", "Release Date": "2024-05-13", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gpt-4o-2024-05-13", "Release Date": "2024-05-13", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gpt-4o-mini-2024-07-18", "Release Date": "2024-07-18", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemma-2-9b-it-simpo", "Release Date": "2024-07-17", "Total Parameters": 9, "Active Parameters": 9, "API Cost": 0}
@@ -19,6 +25,7 @@
 {"Name": "gemini-advanced-0514", "Release Date": "2024-05-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "llama-3.1-405b-instruct-bf16", "Release Date": "2024-07-23", "Total Parameters": 405, "Active Parameters": 405, "API Cost": 0}
 {"Name": "llama-3.1-405b-instruct-fp8", "Release Date": "2024-07-23", "Total Parameters": 405, "Active Parameters": 405, "API Cost": 0}
+{"Name": "llama-3.1-405b", "Release Date": "2024-07-23", "Total Parameters": 405, "Active Parameters": 405, "API Cost": 0}
 {"Name": "gpt-4o-2024-08-06", "Release Date": "2024-08-06", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemini-1.5-pro-api-0514", "Release Date": "2024-05-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gemini-1.5-pro-api-0409-preview", "Release Date": "2024-04-09", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
@@ -48,6 +55,8 @@
 {"Name": "command-r-plus", "Release Date": "2024-04-04", "Total Parameters": 104, "Active Parameters": 104, "API Cost": 0}
 {"Name": "gemma-2-9b-it", "Release Date": "2024-06-27", "Total Parameters": 9, "Active Parameters": 9, "API Cost": 0}
 {"Name": "qwen2-72b-instruct", "Release Date": "2024-06-07", "Total Parameters": 72, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gpt-4", "Release Date": "2023-03-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gpt-4-0314", "Release Date": "2023-03-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "gpt-4-0314", "Release Date": "2023-03-14", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "qwen-max-0428", "Release Date": "2024-04-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "glm-4-0116", "Release Date": "2024-01-16", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
@@ -143,4 +152,5 @@
 {"Name": "fastchat-t5-3b", "Release Date": "2023-04-27", "Total Parameters": 3, "Active Parameters": 3, "API Cost": 0}
 {"Name": "stablelm-tuned-alpha-7b", "Release Date": "2023-04-20", "Total Parameters": 7, "Active Parameters": 7, "API Cost": 0}
 {"Name": "dolly-v2-12b", "Release Date": "2023-04-12", "Total Parameters": 12, "Active Parameters": 12, "API Cost": 0}
-{"Name": "llama-13b", "Release Date": "2023-02-27", "Total Parameters": 13, "Active Parameters": 13, "API Cost": 0}
+{"Name": "llama-13b", "Release Date": "2023-02-27", "Total Parameters": 13, "Active Parameters": 13, "API Cost": 0}
+{"Name": "gpt-3.5", "Release Date": "2022-11-30", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}

planbench_leaderboard.jsonl

@@ -0,0 +1,3 @@
+{"model": "o1-preview-2024-09-12", "score": 52.8}
+{"model": "llama-3.1-405b", "score": 0.8}
+{"model": "gpt-4", "score": 0.16}