update

app.py

@@ -100,7 +100,8 @@ def init_leaderboard(dataframe):
     )
 
 # model_result_path = "./src/results/models_2024-10-20-23:34:57.242641.json"
-model_result_path = "./src/results/models_2024-10-24-08:08:59.127307.json"
+# model_result_path = "./src/results/models_2024-10-24-08:08:59.127307.json"
+model_result_path = "./src/results/models_2024-11-08-08:36:00.464224.json"
 # model_leaderboard_df = get_model_leaderboard_df(model_result_path)
 
 
@@ -192,7 +193,8 @@ with demo:
             
             TEXT = (
                 f'<p style="font-size:{INTRODUCTION_TEXT_FONT_SIZE}px;">'
-                '<b>Total #models: 57 (Last updated: 2024-10-21)</b>'
+                # '<b>Total #models: 57 (Last updated: 2024-10-21)</b>'
+                '<b>Total #models: 62 (Last updated: 2024-11-08)</b>'
                 '</p>'
                 f'<p style="font-size:{INTRODUCTION_TEXT_FONT_SIZE}px;">'
                 'This page prvovides a comprehensive overview of model ranks across various dimensions, based on their averaged ranks or scores.'
@@ -218,6 +220,9 @@ with demo:
                             AutoEvalColumn.rank_reason_logical.name,
                             AutoEvalColumn.rank_reason_social.name,
                             AutoEvalColumn.rank_chemistry.name,
+                            AutoEvalColumn.rank_biology.name,
+                            AutoEvalColumn.rank_physics.name,
+                            
                             AutoEvalColumn.rank_overall.name,
                             # AutoEvalColumn.rank_cpp.name,
                             ],
@@ -242,6 +247,9 @@ with demo:
                             AutoEvalColumn.score_reason_logical.name,
                             AutoEvalColumn.score_reason_social.name,
                             AutoEvalColumn.score_chemistry.name,
+                            AutoEvalColumn.score_biology.name,
+                            AutoEvalColumn.score_physics.name,
+
                             AutoEvalColumn.score_overall.name,
                             # AutoEvalColumn.score_cpp.name,
                             
@@ -278,11 +286,19 @@ with demo:
 
             TEXT = (
                 f'<p style="font-size:{INTRODUCTION_TEXT_FONT_SIZE}px;">'
-                '<b>Total #models: 57 (Last updated: 2024-10-21)</b>'
+                'Algebra, Geometry, and Probability are the current three main math domains in the leaderboard. '
+                'To mitigate the potential impact of data contimination, we have carefully selected the datasets from various sources. '
+                'We prioritize <b>recent math datasets</b> and focus on <b>college and beyond level</b> math questions. '
+                'The current datasets include</b>'
+                '<a href="https://arxiv.org/abs/2103.03874">MATH</a>, '
+                '<a href="htt ps://github.com/openai/prm800k/tree/main/prm800k/math_splits">MATH-500</a>, '
+                '<a href="https://omni-math.github.io/">Omni</a>, '
+                '<a href="https://arxiv.org/abs/1905.13319">MathQA</a>, '
+                '<a href="https://arxiv.org/abs/2405.12209">MathBench</a>, '
+                '<a href="https://arxiv.org/abs/2307.10635">SciBench</a>, and more! '                
                 '</p>'
                 f'<p style="font-size:{INTRODUCTION_TEXT_FONT_SIZE}px;">'
-                'This page prvovides a comprehensive overview of model ranks across various dimensions, based on their averaged ranks or scores.'
-                '(Missing values are due to the slow or problemtic model responses to be fixed soom.)'
+                'We plan to include more math domains, such as calculus, number theory, and more in the future. '
                 '</p>'
                 # '<p style="font-size:{INTRODUCTION_TEXT_FONT_SIZE}px;">'
                 # 'We present '
@@ -534,18 +550,19 @@ with demo:
                         get_model_leaderboard_df(
                             model_result_path,
                             benchmark_cols=[
-                                AutoEvalColumn.model.name, 
-                                
-                                AutoEvalColumn.license.name,
-                                AutoEvalColumn.organization.name,
-                                AutoEvalColumn.knowledge_cutoff.name,
- 
-                                AutoEvalColumn.rank_chemistry.name,
+                                AutoEvalColumn.model.name,                                 
+                                # AutoEvalColumn.license.name,
+                                # AutoEvalColumn.organization.name,
+                                # AutoEvalColumn.knowledge_cutoff.name,
+                                AutoEvalColumn.rank_chemistry.name, 
+                                AutoEvalColumn.rank_biology.name,
+                                AutoEvalColumn.rank_physics.name,
                                 ],
-                            rank_col=['sort_by_rank', 4, 5, 'Science'],
+                            rank_col=['sort_by_rank', 1, 4, 'Science'],
                         )
                     )
-
+                    
+                    
                 with gr.TabItem("⭐ Sort by Score", elem_id="science_overview_sort_by_score_subtab", id=1, elem_classes="subtab"): 
                     leaderboard = overall_leaderboard(
                         get_model_leaderboard_df(
@@ -553,14 +570,15 @@ with demo:
                             benchmark_cols=[
                                 AutoEvalColumn.model.name, 
 
-                                AutoEvalColumn.license.name,
-                                AutoEvalColumn.organization.name,
-                                AutoEvalColumn.knowledge_cutoff.name,
+                                # AutoEvalColumn.license.name,
+                                # AutoEvalColumn.organization.name,
+                                # AutoEvalColumn.knowledge_cutoff.name,
 
-                                AutoEvalColumn.score_chemistry.name,
-                                # AutoEvalColumn.rank_chemistry.name,
+                                AutoEvalColumn.score_chemistry.name, 
+                                AutoEvalColumn.score_biology.name,
+                                AutoEvalColumn.score_physics.name,
                                 ],
-                            rank_col=['sort_by_score', 4, 5], # two numbers are index to select the columns to average and sort
+                            rank_col=['sort_by_score', 1, 4, 'Science'], # two numbers are index to select the columns to average and sort
                         )
                     )
                     
@@ -583,18 +601,50 @@ with demo:
                     )
                 )
 
-            with gr.TabItem("⚛️ Physics", elem_id="physics_subtab", id=2, elem_classes="subtab"):   
-                CURRENT_TEXT = """
-                # Coming soon!
-                """
-                gr.Markdown(CURRENT_TEXT, elem_classes="markdown-text")
-
 
             with gr.TabItem("🧬 Biology", elem_id="biology_subtab", id=3, elem_classes="subtab"):   
-                CURRENT_TEXT = """
-                # Coming soon!
-                """
-                gr.Markdown(CURRENT_TEXT, elem_classes="markdown-text")
+                # CURRENT_TEXT = """
+                # # Coming soon!
+                # """
+                # gr.Markdown(CURRENT_TEXT, elem_classes="markdown-text")
+                leaderboard = overall_leaderboard(
+                    get_model_leaderboard_df(
+                        model_result_path,
+                        benchmark_cols=[
+                            AutoEvalColumn.rank_biology.name,
+                            AutoEvalColumn.model.name, 
+                            AutoEvalColumn.score_biology.name,
+                            # AutoEvalColumn.sd_reason_social.name,
+                            AutoEvalColumn.license.name,
+                            AutoEvalColumn.organization.name,
+                            AutoEvalColumn.knowledge_cutoff.name,
+                            ],
+                        rank_col=[AutoEvalColumn.rank_biology.name],
+                    )
+                )
+                
+                
+            with gr.TabItem("⚛️ Physics", elem_id="physics_subtab", id=2, elem_classes="subtab"):   
+                # CURRENT_TEXT = """
+                # # Coming soon!
+                # """
+                # gr.Markdown(CURRENT_TEXT, elem_classes="markdown-text")
+                leaderboard = overall_leaderboard(
+                    get_model_leaderboard_df(
+                        model_result_path,
+                        benchmark_cols=[
+                            AutoEvalColumn.rank_physics.name,
+                            AutoEvalColumn.model.name, 
+                            AutoEvalColumn.score_physics.name,
+                            # AutoEvalColumn.sd_reason_social.name,
+                            AutoEvalColumn.license.name,
+                            AutoEvalColumn.organization.name,
+                            AutoEvalColumn.knowledge_cutoff.name,
+                            ],
+                        rank_col=[AutoEvalColumn.rank_physics.name],
+                    )
+                )
+
 
 
         with gr.TabItem("</> Coding", elem_id="coding-table", id=5):

src/display/utils.py

@@ -101,6 +101,7 @@ auto_eval_column_dict.append(["sd_biology", ColumnContent, field(default_factory
 auto_eval_column_dict.append(["rank_biology", ColumnContent, field(default_factory=lambda: ColumnContent("Rank (Biology)", "number", True))])
 
 
+
 auto_eval_column_dict.append(["score_cpp", ColumnContent, field(default_factory=lambda: ColumnContent("Score (C++)", "number", True))])
 auto_eval_column_dict.append(["sd_cpp", ColumnContent, field(default_factory=lambda: ColumnContent("Std dev (C++)", "number", True))])
 auto_eval_column_dict.append(["rank_cpp", ColumnContent, field(default_factory=lambda: ColumnContent("Rank (C++)", "number", True))])

src/leaderboard/read_evals.py

@@ -188,6 +188,15 @@ class ModelResult:
             AutoEvalColumn.score_chemistry.name: self.results.get("Chemistry").get("Average Score", None) if self.results.get("Chemistry") else None,
             AutoEvalColumn.sd_chemistry.name: self.results.get("Chemistry").get("Standard Deviation", None) if self.results.get("Chemistry") else None,
             AutoEvalColumn.rank_chemistry.name: self.results.get("Chemistry").get("Rank", None) if self.results.get("Chemistry") else None,
+
+            AutoEvalColumn.score_biology.name: self.results.get("Biology").get("Average Score", None) if self.results.get("Biology") else None,
+            AutoEvalColumn.sd_biology.name: self.results.get("Biology").get("Standard Deviation", None) if self.results.get("Biology") else None,
+            AutoEvalColumn.rank_biology.name: self.results.get("Biology").get("Rank", None) if self.results.get("Biology") else None,
+
+            AutoEvalColumn.score_physics.name: self.results.get("Physics").get("Average Score", None) if self.results.get("Physics") else None,
+            AutoEvalColumn.sd_physics.name: self.results.get("Physics").get("Standard Deviation", None) if self.results.get("Physics") else None,
+            AutoEvalColumn.rank_physics.name: self.results.get("Physics").get("Rank", None) if self.results.get("Physics") else None,
+            
             
             AutoEvalColumn.score_cpp.name: self.results.get("CPP").get("Average Score", None) if self.results.get("CPP") else None,
             AutoEvalColumn.sd_cpp.name: self.results.get("CPP").get("Standard Deviation", None) if self.results.get("CPP") else None,