update results & separate results organization

app.py

@@ -55,7 +55,8 @@ with gr.Blocks() as block:
                 )
 
             # Define different captions for each table
-            default_caption = "**Table 1: MEGA-Bench full results.** The number in the parentheses is the number of tasks of each keyword. <br> The Core set contains $N_{\\text{core}} = 440$ tasks evaluated by rule-based metrics, and the Open-ended set contains $N_{\\text{open}} = 65$ tasks evaluated by a VLM judge (we use GPT-4o-0806). <br> $\\text{Overall} \\ = \\ \\frac{\\max(\\text{Core w/o CoT}, \\ \\text{Core w/ CoT}) \\ \\cdot \\ N_{\\text{core}} \\ + \\ \\text{Open-ended} \\ \\cdot \\ N_{\\text{open}}}{N_{\\text{core}} \\ + \\ N_{\\text{open}}}$"
+            default_caption = "**Table 1: MEGA-Bench full results.** The number in the parentheses is the number of tasks of each keyword. <br> The Core set contains $N_{\\text{core}} = 440$ tasks evaluated by rule-based metrics, and the Open-ended set contains $N_{\\text{open}} = 65$ tasks evaluated by a VLM judge (we use GPT-4o-0806). <br> Different from the results in our paper, we only use the Core results with CoT prompting here for clarity and compatibility with the released data. <br> $\\text{Overall} \\ = \\ \\frac{\\text{Core} \\ \\cdot \\ N_{\\text{core}} \\ + \\ \\text{Open-ended} \\ \\cdot \\ N_{\\text{open}}}{N_{\\text{core}} \\ + \\ N_{\\text{open}}}$ " 
+
             single_image_caption = "**Table 2: MEGA-Bench Single-image setting results.** The number in the parentheses is the number of tasks in each keyword. <br> This subset contains 273 single-image tasks from the Core set and 42 single-image tasks from the Open-ended set. For open-source models, we drop the image input in the 1-shot demonstration example so that the entire query contains a single image only. <br> Compared to the default table, some models with only single-image support are added."
 
             caption_component = gr.Markdown(

constants.py

@@ -28,7 +28,7 @@ We aim to provide cost-effective and accurate evaluation for multimodal models,
 
 ## 📊🔍 Results & Takeaways from Evaluating Top Models
 
-- GPT-4o (0513) and Claude 3.5 Sonnet (1022) lead the benchmark. Claude 3.5 Sonnet (1022) improves over Claude 3.5 Sonnet (0622) obviously in planning tasks (application dimension) and UI/Infographics inputs (input format dimension).
+- GPT-4o (0513) and Claude 3.5 Sonnet (1022) lead the benchmark. Claude 3.5 Sonnet (1022) improves over Claude 3.5 Sonnet (0620) obviously in planning tasks (application dimension) and UI/Infographics inputs (input format dimension).
 - Qwen2-VL stands out among open-source models, and its flagship model gets close to some proprietary flagship models
 - Chain-of-Thought (CoT) prompting improves proprietary models but has limited impact on open-source models
 - Gemini 1.5 Flash performs the best among all the evaluated efficiency models, but struggles with UI and document tasks

static/eval_results/Default/Aquila_VL_2B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.159970161379836,
+            "micro_mean_score": 0.15844711671722148
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.24567572098570653,
+            "micro_mean_score": 0.2704213241616509
+        },
+        "overall_score": 0.17100157004197775
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.1796551584774396
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.1263506560912463
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.1775085349123463
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.2114933522881099
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.16251700109869488
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.26453155444796583
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.3729498746867168
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.19090788408036002
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.16500679466160564
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.03972686819521137
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.07035116566014021
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.11915109312705179
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.18915652635850314
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.21939978337316163
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.17643260913333875
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.2438396314831894
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.08989401697906672
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.12241197113963243
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.10758402844431432
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.19372082302321905
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.19201243810115767
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.23278612647548963
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.21664527852608348
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.12138133030990172
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.01221681479628382
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.17994400163273605
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.21939978337316163
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.18212149746318507
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.21563163558700174
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.0981320856519089
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.0557399538308785
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.1351126472094214
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.2025034827431662
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.29326275059361956
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.22529225586731416
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.23810497886903373
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.17867138975396438
+            }
+        }
+    }
+}

static/eval_results/Default/Aria/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.289073788209904,
+            "micro_mean_score": 0.2859007507765791
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.5103725263180767,
+            "micro_mean_score": 0.5349957007738607
+        },
+        "overall_score": 0.31755778420402525
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.3153649050553317
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.34425736922415495
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.3921740378709932
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.37623282710622424
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.271674311347156
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.46313777834281344
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.5692180451127821
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.3152064038837139
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.23851147782276536
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.11246568298589892
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.28561724084490353
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.2505346698796475
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.3040414715952029
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.41865640360591405
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.3622713579911698
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.35872259826035346
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.1509096092007215
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.2846987779732631
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.2899384042262363
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.27412885527802433
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.3117275816801635
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.4523860109667709
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.310055869988487
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.18301681783824644
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.26651659725352617
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.34236220565522313
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.41865640360591405
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.19142683154129833
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.2596336265133595
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.3929243812973524
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.1403503245041943
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.25367910605102256
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.3494812758481046
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.3662927672998609
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.28616079233761366
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.3953949223279651
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.26097385403450996
+            }
+        }
+    }
+}

static/eval_results/Default/Claude_3.5/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.5040975742801586,
+            "micro_mean_score": 0.5002259116666758
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.6373907158949892,
+            "micro_mean_score": 0.6569647463456579
+        },
+        "overall_score": 0.5212541172602853
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.5405089647404562
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.6082834220752651
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.5745077617490254
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.5450038475783499
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.4767692987630454
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.5756126284078804
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.6969774436090224
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.5278843049497918
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.4082144793870471
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.23803578664609892
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.5691641481808987
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.4795267886975966
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.525848282456283
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.508735695828719
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.5699094130430454
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.5096772701625744
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.4429640420975014
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.5066797418318023
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.4971460788134188
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.5278127103234661
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.4490020843308984
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.5838224169821388
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.5456152399978661
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.46300075585789874
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.5414381873407914
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.5373019912310933
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.508735695828719
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.4422556748863689
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.49311554035078103
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.6663170946790707
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.3382015835012861
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.5194010220575684
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.532329797132399
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.5808831682303479
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.513474611293123
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.5507075880782885
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.47461998432626556
+            }
+        }
+    }
+}

static/eval_results/Default/Claude_3.5_new/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.5259191914020757,
+            "micro_mean_score": 0.5230785894131227
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.6563419761104125,
+            "micro_mean_score": 0.6724419604471196
+        },
+        "overall_score": 0.5427062825031487
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.5690045172520449
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.6220681231036606
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.6077980666415158
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.5511440615639541
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.4885536652013625
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.5908204006544897
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.6569473684210526
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.5486763511384175
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.4315385951907387
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.2909419331017877
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.6048192628845258
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.48924295292319175
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.556418710368288
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.4946691340754988
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.5558756390298104
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.5425198547046186
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.44210335381541843
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.5187252051932875
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.5071121107460066
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.5387340524651681
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.4824302644151348
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.6242798397166945
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.5782691045270721
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.4630277507828528
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.5914338446093256
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.5636254729390459
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.4946691340754988
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.4828123870640382
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.48756636014597515
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.6590137441693218
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.39901670035164916
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.5166853031535193
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.5561634744977417
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.6123769274172342
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.5512015158810595
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.565796566886933
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.4763267502912362
+            }
+        }
+    }
+}

static/eval_results/Default/GPT_4o/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.5265030595065238,
+            "micro_mean_score": 0.5236338521693411
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.6478225794744895,
+            "micro_mean_score": 0.665391229578676
+        },
+        "overall_score": 0.5421184432647768
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.5630758211022604
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.6216411634729735
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.616018277142757
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.5823101249498799
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.44177544539510955
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.6345458069232931
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.6795263157894738
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.5514924675940659
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.39435038953269674
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.22934807257231926
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.608083455060831
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.491325251564869
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.4999089647103332
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.5315979872161023
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.5641404607063637
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.5613545677222056
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.47760591698367955
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.5388690453811203
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.48037685656449847
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.5994159671881645
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.44606605087301393
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.6274371950293718
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.5448877153826162
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.4751133786848073
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.5343350103400748
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.5672657028463585
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.5315979872161023
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.4500928191484624
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.4908653289106883
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.7056027785545881
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.33202130899313653
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.5032849161169843
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.5510350848991218
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.6095778863474799
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.5283797185155754
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.6135723164021851
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.44047720383044436
+            }
+        }
+    }
+}

static/eval_results/Default/GPT_4o_mini/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.40767494558789397,
+            "micro_mean_score": 0.40431644154143376
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.586537827213665,
+            "micro_mean_score": 0.6133276010318144
+        },
+        "overall_score": 0.43069690064863675
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.4492982787524939
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.49026056071002017
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.5168957112681365
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.46731791428406805
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.3406008235342885
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.5572925295284307
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.6902380952380953
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.4189154010048976
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.2943206715105082
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.19422793560945503
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.47202628409684394
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.3624496929166193
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.38946844562183286
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.45508480503584553
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.47569921440672464
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.465175334092545
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.29410984789062117
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.41242028190533997
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.3906415365938764
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.44244772638735347
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.3629944944697668
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.5713834131825314
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.39874839531459466
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.3359977324263039
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.4305788513381019
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.46343334374251277
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.45508480503584553
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.24651576711552803
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.36981497185070983
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.5666618234843734
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.2420320329702607
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.3458483931206892
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.43590838051817093
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.5176671720617656
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.3554299482098288
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.5399167524341886
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.32918280841495845
+            }
+        }
+    }
+}

static/eval_results/Default/Gemini_1.5_flash_002/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.4189319021967416,
+            "micro_mean_score": 0.41567515414375245
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.5691365176285039,
+            "micro_mean_score": 0.5987532244196045
+        },
+        "overall_score": 0.4382651695295427
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.46355333176347063
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.4431807648811706
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.4975887290434539
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.49409642663278297
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.38033540105052427
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.5621166766717235
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.6570726817042606
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.4480877005302385
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.3338006749329557
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.16197013296986068
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.3971534837718938
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.3448204918940882
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.43525833484767545
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.4837362543956792
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.5111257660425502
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.49366013155105076
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.4001983820478609
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.386988040250785
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.3884226428206387
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.4425893080900246
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.42223626366392253
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.5390305634303021
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.472066557554629
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.3666950113378685
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.44571360028283974
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.45400479933257654
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.4837362543956792
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.35161402777057993
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.3839609821519984
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.4822341581959653
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.26434115361219657
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.3677547363031234
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.4640301382180305
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.5348199655361041
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.4890240042560499
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.5126038207415967
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.384818434165593
+            }
+        }
+    }
+}

static/eval_results/Default/Gemini_1.5_pro_002/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.4822473962867704,
+            "micro_mean_score": 0.4764805563057179
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.5858190649927173,
+            "micro_mean_score": 0.6104901117798793
+        },
+        "overall_score": 0.4955784031499121
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.5202055934299538
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.5017043129027509
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.5532599716027446
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.546753787203128
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.425969084163906
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.5751012914154264
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.6982330827067671
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.513647745999633
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.3845337030093212
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.23899503258223884
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.4625032188638111
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.4292353723689881
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.4869625906903554
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.5028718355967439
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.5584779204331461
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.55005349042813
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.4292127751495457
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.44896309957892694
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.44418591808616864
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.5146447350354234
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.4688623462674191
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.5580414823700747
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.5538255562099124
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.39066515495086923
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.5370278962809547
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.5034399620483027
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.5028718355967439
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.4885398161821004
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.45544217378728585
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.5421439953094952
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.3335324339429373
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.43465181771633377
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.5250631828331306
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.5821004797173627
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.5124355410095621
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.5722329455291694
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.41210885517904977
+            }
+        }
+    }
+}

static/eval_results/Default/Idefics3/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.08956972487602757,
+            "micro_mean_score": 0.08982225274252693
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.3210866162255635,
+            "micro_mean_score": 0.35649183147033553
+        },
+        "overall_score": 0.11936892871309657
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.123378776179585
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.09602065544451607
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.1661543932339007
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.13018902877020821
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.11200133210641629
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.1837120314657304
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.2364085213032582
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.15239546294916975
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.08255834173646705
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.03149369112824262
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.06151607584357764
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.10124344675801887
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.14147248511867794
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.15942387460900312
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.17458268378399872
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.13442937440893113
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.02766884416043467
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.15513016850044997
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.03757596375966502
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.05386631116442094
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.0760949224506388
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.2987797010800956
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.10403841600436024
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.0661753590325019
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.09190674791720088
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.12345439179884048
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.15942387460900312
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.11382786944230487
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.10803808254834846
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.11450308988278819
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.04671278220005028
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.0978814644137225
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.13283830731528018
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.09697463995668018
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.1840497279921703
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.1605667124060194
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.09835465288235297
+            }
+        }
+    }
+}

static/eval_results/Default/InternVL2_2B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.13141974398938763,
+            "micro_mean_score": 0.13063500716262516
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.23864417043743646,
+            "micro_mean_score": 0.24901117798796224
+        },
+        "overall_score": 0.14522090778963154
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.14491178903291552
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.12126906675624163
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.16912754929321935
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.18542274192083463
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.13923308734553164
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.23992252224543772
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.3420927318295739
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.14807577209152425
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.13036555933925006
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.01727799227799228
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.057021136657850864
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.10504085961245285
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.1625198552182714
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.18999779001767986
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.1487677475708977
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.2011727338536935
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.11886936592818943
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.1131404778887607
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.05739750616837997
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.15465451663650032
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.16044698450090833
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.21429521387724249
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.2128614316540013
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.03658352229780801
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.05757839721254354
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.15225683687839608
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.18999779001767986
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.17677460549936644
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.158165588340436
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.08722661966805
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.04102853815875594
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.11264043251709285
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.17001758160301803
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.3332891958712894
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.1686125516807394
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.21169137106199268
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.10975764217070672
+            }
+        }
+    }
+}

static/eval_results/Default/InternVL2_5_2B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.17806821966478364,
+            "micro_mean_score": 0.17708809739236367
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.2738430375585404,
+            "micro_mean_score": 0.2905417024935512
+        },
+        "overall_score": 0.19039567147289096
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.19614682488147464
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.18910947570579717
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.20543964378430513
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.23636598588530347
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.15691382827270517
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.28604169870255614
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.4248446115288219
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.18745928331343714
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.15097551654513372
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.030568378443583684
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.13898447520398388
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.13154711942685113
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.18343540213068474
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.20755556526976354
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.15983467048343838
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.26888883087046195
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.12906517409932386
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.14702422379343882
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.15324148486802894
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.19977956414542175
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.1665590610582109
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.2529339759528222
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.23420071687554841
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.09651832955404382
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.0784280378818194
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.21260786581183966
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.20755556526976354
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.138285387531761
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.20214332169825855
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.18128339685489062
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.053153113565753
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.12416116984428181
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.22449772657901465
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.3762336977650326
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.19222024833691936
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.25056132494721467
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.15596334442569906
+            }
+        }
+    }
+}

static/eval_results/Default/InternVL2_5_78B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.44132952988532753,
+            "micro_mean_score": 0.4397079059379812
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.5538024772749066,
+            "micro_mean_score": 0.5776870163370592
+        },
+        "overall_score": 0.4558062458859664
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.46893853078050696
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.5220829627238773
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.4933134095077618
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.477971701185214
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.3936387335462224
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.5610278744213835
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.6072907268170428
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.44533550848682696
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.3548055654857457
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.22852234519925363
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.4910486370158392
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.39410061025954557
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.43424133240430957
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.5300255483670417
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.4793195260560365
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.4622918421665308
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.3729954065847296
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.4226567593431527
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.4149806887502539
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.4904285184890861
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.4348674018783908
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.5124942746906233
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.4717682857925982
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.20496909081092754
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.4184724897299287
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.4951997132559491
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.5300255483670417
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.286105084660728
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.39635000103107665
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.5401547630322637
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.26403470419652064
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.3933356676003734
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.5168098196770042
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.47731479110938463
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.4388571290145052
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.5034762755043025
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.37742798395328586
+            }
+        }
+    }
+}

static/eval_results/Default/InternVL2_76B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.3562710424410931,
+            "micro_mean_score": 0.35129859801162616
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.5192997443033639,
+            "micro_mean_score": 0.5421324161650903
+        },
+        "overall_score": 0.3772549347599992
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.38193012983650343
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.41315219763443384
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.43665980552577693
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.4265623936500962
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.2975890791763991
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.5257990949897898
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.5779473684210527
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.33287081421166276
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.2949505390920417
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.17036496432397477
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.3634339625985008
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.31396468806559114
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.3473756113126343
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.395893002855977
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.44982107744035305
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.42875248733027654
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.2868239162778749
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.3630499545707523
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.3476691827105281
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.3943337471922549
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.29244088978470345
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.45822072478616577
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.3879326330400817
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.20309901738473166
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.34771123515123364
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.4145693044465943
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.395893002855977
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.24403942809507134
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.3153417935059416
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.4306947454508794
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.2132321995754061
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.2953329718984368
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.42202934355552685
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.47409276729986083
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.30014798153766264
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.4625649385962016
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.2868813944130515
+            }
+        }
+    }
+}

static/eval_results/Default/InternVL2_8B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.25956581776451815,
+            "micro_mean_score": 0.2546984460483302
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1165,
+            "macro_mean_score": 0.3978571701460552,
+            "micro_mean_score": 0.4108583690987125
+        },
+        "overall_score": 0.2773656948037259
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.2817247716997634
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.280559214034858
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2511,
+                "tasks": [],
+                "average_score": 0.32020728060179815
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2469,
+                "tasks": [],
+                "average_score": 0.325593535916075
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.24118253695139918
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.39684007367798446
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.4700852130325815
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.27052668526005397
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2439,
+                "tasks": [],
+                "average_score": 0.23189345356483618
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.08260405712900723
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.22800928556370195
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.2013779290163996
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.2804429603269583
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 700,
+                "tasks": [],
+                "average_score": 0.34791358240562653
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.2942163420306113
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.3388056726588417
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.10933317885944857
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.250804626773504
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.2522493284864019
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.27414636444623874
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.22381302045502052
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1456,
+                "tasks": [],
+                "average_score": 0.3537549824897016
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.30261189962428353
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.15434618291761149
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.19872104324302098
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.30088711082969344
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 700,
+                "tasks": [],
+                "average_score": 0.34791358240562653
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.17725087609332119
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.2532272454839157
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.29129840423784176
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.12166926715781588
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.24700310231619527
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2315,
+                "tasks": [],
+                "average_score": 0.3214666523378005
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.3995660275981844
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.24614711281861912
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.3393895915929317
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.22078333222564453
+            }
+        }
+    }
+}

static/eval_results/Default/Llama_3_2_11B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.15999641916771298,
+            "micro_mean_score": 0.15809331016967038
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.3173342406187366,
+            "micro_mean_score": 0.3487962166809973
+        },
+        "overall_score": 0.1802478219287358
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.1907604552173455
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.14328677752263275
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.19646404502647707
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.22399113135844315
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.13303760019716085
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.323153603297999
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.4260501253132832
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.1770852858056774
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.15366454315378308
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.06563884729522687
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.11886347847341794
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.11489351406848371
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.1693681214060816
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.2123769209846321
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.2520175802062012
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.2485354956932213
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.06418655520777307
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.12417283740525839
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.16374180545556977
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.1576236804437753
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.15014439824913947
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.3003142292328822
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.19270157739425633
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.1463246409674981
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.0732004839476103
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.1960107191983825
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.2123769209846321
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.1351857051327849
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.18586695387250338
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.17288724679416761
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.08100042975820579
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.0575426944971537
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.19899465185565898
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.254316961351997
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.162801811963855
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.28055776664538923
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.13937853323074623
+            }
+        }
+    }
+}

static/eval_results/Default/Mammoth_VL/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.264052880412689,
+            "micro_mean_score": 0.2626894374387823
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.37992668750165337,
+            "micro_mean_score": 0.40120378331900275
+        },
+        "overall_score": 0.27896733083008046
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.30194776127683565
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.2365295791606494
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.2993927028494267
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.3366347826116991
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.2408454736444444
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.37895522991264047
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.48003508771929826
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.27232427744946475
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.24522937191710698
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.11457024299726488
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.18941525254390731
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.1718334741390191
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.28108187023954245
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.3391119999611432
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.36434285930327387
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.36915384448504296
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.15940750469262005
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.2456942956200745
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.21586513216389874
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.29359048024032264
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.2646677074112521
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.34733130661096645
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.3286125236284589
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.16358654572940287
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.25463059203015115
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.2919119209789575
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.3391119999611432
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.20016011839130254
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.2679179451692527
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.23600902063965679
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.15326915093278803
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.20668466311255687
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.33348955971237954
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.3759170425350556
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.23894961766260706
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.351703435685048
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.26074348700688493
+            }
+        }
+    }
+}

static/eval_results/Default/MiniCPM_v2.6/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.22955895202146906,
+            "micro_mean_score": 0.22560399396899078
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.41728623355613875,
+            "micro_mean_score": 0.43452278589853827
+        },
+        "overall_score": 0.2537218694467236
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.2604967101191775
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.2500331562865158
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.3003169369011028
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.31808748114668184
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.18281637763548025
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.40732197204308807
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.48798245614035085
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.23723675736151562
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.1968926733821904
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.08735883237069725
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.21195711598986072
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.18639148159043903
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.21578309681746147
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.3527537836840162
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.3096882575625531
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.3176880312524649
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.0755920550038197
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.23506388020592064
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.1781127776443048
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.2551275278138797
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.20833171754655547
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.36473950920880716
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.293386806641223
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.13955971277399848
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.23596215721092323
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.26319603880798287
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.3527537836840162
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.17888270664238365
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.22288558250834017
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.2666989364424082
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.11693267119342445
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.15342045420318667
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.29243044121840894
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.3777897246686755
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.25714862989687987
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.33187729423141027
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.16493399805627715
+            }
+        }
+    }
+}

static/eval_results/Default/NVLM/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.21589726765847422,
+            "micro_mean_score": 0.21406043849932396
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.3478114310231307,
+            "micro_mean_score": 0.3947549441100602
+        },
+        "overall_score": 0.23287631838857856
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.21591473223174515
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.27426258729618225
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.284874072963892
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.2134087963800149
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.2525993645909815
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.4029543142569604
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.4317142857142857
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.2442484196551863
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.1424318574406695
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.046798309600525674
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.19655048708297065
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.18621338396242557
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.2922667531642391
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.0
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.3447361496776569
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.29674507895195534
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.09716389574493003
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.19684666506287793
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.2199792859352912
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.25164831125437204
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.2396831363622878
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.3215948035793096
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.1853526865291571
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.0
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.0
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.3352056263801705
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.0
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.038244047619047615
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.2100484481849172
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.15704252277801936
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.06688589450465973
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.2292747206409446
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.2689383226748064
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.18857142857142856
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.23682040748983965
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.3656649917873737
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.26866914106442213
+            }
+        }
+    }
+}

static/eval_results/Default/Phi-3.5-vision/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.22995297916629392,
+            "micro_mean_score": 0.22708502951025372
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.3947914647737769,
+            "micro_mean_score": 0.42459157351676696
+        },
+        "overall_score": 0.2511698139474551
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.2550326045763433
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.24395249720074527
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.2858236369733704
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.29876274710122536
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.21972896566746963
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.37513466171380355
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.4713934837092732
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.25475240046465697
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.20386233377001492
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.06657701969095552
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.16556787388989183
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.17989790940001513
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.2671646581690049
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.24920333780186898
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.3057560384411286
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.3341992361416253
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.12884156381685322
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.20494682188374266
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.21180084406324556
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.2609992615064841
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.2149689274645855
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.365192668303297
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.2593652357274648
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.10107709750566891
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.11861055655587921
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.2824151476986241
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.24920333780186898
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.1980440594073205
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.2636292373854696
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.20747122167273002
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.08602953103518936
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.20136893467064246
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.30979039348232706
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.3495072422622861
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.25858403958844717
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.3357218088688187
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.21140555087788399
+            }
+        }
+    }
+}

static/eval_results/Default/Pixtral_12B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.31362045151669854,
+            "micro_mean_score": 0.3100986209078182
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.4566234428542061,
+            "micro_mean_score": 0.4870593293207223
+        },
+        "overall_score": 0.33202677713439754
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.34184129499032456
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.37667712211439836
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.37896441862738645
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.37077191302051077
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.2843861774995234
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.4098150360139686
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.533077694235589
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.3372902862054838
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.25372282838901716
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.09524894246403817
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.2972619996610934
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.28304049684103855
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.33523333364720703
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.3988260865341648
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.39117521970978353
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.35583482417594536
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.21897822147396953
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.3436473210057542
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.28979044279399635
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.33530850344530555
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.30160980000905374
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.4166613092238044
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.30796171250186904
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.22871315192743763
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.21669652626580332
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.36087312117067055
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.3988260865341648
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.24616927284658197
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.2900329121369093
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.42652313209316933
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.1209559708312353
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.25678368121442124
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.37605128363484847
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.4576088857728113
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.3464929909487855
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.3858431845580602
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.2549787156825223
+            }
+        }
+    }
+}

static/eval_results/Default/Qwen2_VL_2B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.20877163406364055,
+            "micro_mean_score": 0.20561526268932287
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.3154302566225611,
+            "micro_mean_score": 0.33856405846947557
+        },
+        "overall_score": 0.22249997162072932
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.22236161923122505
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.23701014663017753
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.25669221785292334
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.26526414975225454
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.17623548305581763
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.31250702198481506
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.4140676691729323
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.20802820480076603
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.17320633068307653
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.06209506566980099
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.190837839372028
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.16287824421269087
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.19640906475019812
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.2520741776922928
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.24883076673424442
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.2877316297453947
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.13398525561847363
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.1624451002757208
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.20960092816529263
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.19986806708136184
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.2201024015934558
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.30248748033122763
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.256631742010999
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.07681405895691609
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.10526691703628158
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.25018977062352593
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.2520741776922928
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.17435940889565366
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.21286783416184518
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.2521972668785968
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.06967138760493456
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.16996250112948405
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.27603334911345223
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.31002436092347696
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.21061929716065056
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.2656728023444808
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.16356158787929762
+            }
+        }
+    }
+}

static/eval_results/Default/Qwen2_VL_72B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.4542376574527161,
+            "micro_mean_score": 0.4501201906164793
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1163,
+            "macro_mean_score": 0.5639771804231668,
+            "micro_mean_score": 0.5835339638865004
+        },
+        "overall_score": 0.4683625465479226
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.48669152179713876
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.5291932917937967
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2509,
+                "tasks": [],
+                "average_score": 0.53654503409075
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2467,
+                "tasks": [],
+                "average_score": 0.4931554892760308
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.3908023665629473
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.5668846347262286
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.6121127819548872
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.4493794346300551
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2437,
+                "tasks": [],
+                "average_score": 0.33622171962424363
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.21642754068858566
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.5263730250833892
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.42759570727857965
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.4228561177227288
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.4780253686541936
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.5070774860945021
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.4807292191169126
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.38847545874852984
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.4359156358804688
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.43781407268698613
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.49080138099759946
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.42481004254128113
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1454,
+                "tasks": [],
+                "average_score": 0.5132810622684265
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.5062248706593999
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.3063303099017385
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.523959576707116
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.4879791577413812
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 698,
+                "tasks": [],
+                "average_score": 0.4780253686541936
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.34846161336322395
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.44101149919132854
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.5663587858366833
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.3067825586087303
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.4121566368482877
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2313,
+                "tasks": [],
+                "average_score": 0.5176521211872086
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.5030444649397028
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.45616267568458396
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.5047683071464567
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.3553838743540432
+            }
+        }
+    }
+}

static/eval_results/Default/Qwen2_VL_7B/summary_results.json

@@ -0,0 +1,251 @@
+{
+    "model_summary": {
+        "core": {
+            "num_eval_tasks": 440,
+            "num_eval_samples": 6539,
+            "macro_mean_score": 0.3293449599230247,
+            "micro_mean_score": 0.325331493515679
+        },
+        "open": {
+            "num_eval_tasks": 65,
+            "num_eval_samples": 1170,
+            "macro_mean_score": 0.43955105763038577,
+            "micro_mean_score": 0.45508547008546996
+        },
+        "overall_score": 0.34352990319228904
+    },
+    "keyword_stats": {
+        "skills": {
+            "Object Recognition and Classification": {
+                "count": 303,
+                "num_samples": 4755,
+                "tasks": [],
+                "average_score": 0.3506773570484231
+            },
+            "Text Recognition (OCR)": {
+                "count": 137,
+                "num_samples": 2239,
+                "tasks": [],
+                "average_score": 0.38363163370919123
+            },
+            "Language Understanding and Generation": {
+                "count": 154,
+                "num_samples": 2511,
+                "tasks": [],
+                "average_score": 0.3882785389756705
+            },
+            "Scene and Event Understanding": {
+                "count": 154,
+                "num_samples": 2469,
+                "tasks": [],
+                "average_score": 0.38292659892379843
+            },
+            "Mathematical and Logical Reasoning": {
+                "count": 109,
+                "num_samples": 1910,
+                "tasks": [],
+                "average_score": 0.2730765188348748
+            },
+            "Commonsense and Social Reasoning": {
+                "count": 51,
+                "num_samples": 855,
+                "tasks": [],
+                "average_score": 0.4625711182912848
+            },
+            "Ethical and Safety Reasoning": {
+                "count": 15,
+                "num_samples": 245,
+                "tasks": [],
+                "average_score": 0.5287318295739348
+            },
+            "Domain-Specific Knowledge and Skills": {
+                "count": 77,
+                "num_samples": 1386,
+                "tasks": [],
+                "average_score": 0.32297080808954215
+            },
+            "Spatial and Temporal Reasoning": {
+                "count": 152,
+                "num_samples": 2439,
+                "tasks": [],
+                "average_score": 0.2561357336105554
+            },
+            "Planning and Decision Making": {
+                "count": 37,
+                "num_samples": 577,
+                "tasks": [],
+                "average_score": 0.12651411144309255
+            }
+        },
+        "input_format": {
+            "User Interface Screenshots": {
+                "count": 93,
+                "num_samples": 1517,
+                "tasks": [],
+                "average_score": 0.35229497847636093
+            },
+            "Text-Based Images and Documents": {
+                "count": 82,
+                "num_samples": 1294,
+                "tasks": [],
+                "average_score": 0.2881996369284258
+            },
+            "Diagrams and Data Visualizations": {
+                "count": 101,
+                "num_samples": 1718,
+                "tasks": [],
+                "average_score": 0.3162917354476226
+            },
+            "Videos": {
+                "count": 43,
+                "num_samples": 700,
+                "tasks": [],
+                "average_score": 0.3555910609857979
+            },
+            "Artistic and Creative Content": {
+                "count": 32,
+                "num_samples": 541,
+                "tasks": [],
+                "average_score": 0.3513518594470202
+            },
+            "Photographs": {
+                "count": 143,
+                "num_samples": 2248,
+                "tasks": [],
+                "average_score": 0.39509504888372243
+            },
+            "3D Models and Aerial Imagery": {
+                "count": 11,
+                "num_samples": 169,
+                "tasks": [],
+                "average_score": 0.19173322639974366
+            }
+        },
+        "output_format": {
+            "contextual_formatted_text": {
+                "count": 98,
+                "num_samples": 1514,
+                "tasks": [],
+                "average_score": 0.3118818521697947
+            },
+            "structured_output": {
+                "count": 110,
+                "num_samples": 1714,
+                "tasks": [],
+                "average_score": 0.3323478338046426
+            },
+            "exact_text": {
+                "count": 83,
+                "num_samples": 1278,
+                "tasks": [],
+                "average_score": 0.31975345327634014
+            },
+            "numerical_data": {
+                "count": 49,
+                "num_samples": 862,
+                "tasks": [],
+                "average_score": 0.3207400992620562
+            },
+            "open_ended_output": {
+                "count": 80,
+                "num_samples": 1456,
+                "tasks": [],
+                "average_score": 0.39680785337230745
+            },
+            "multiple_choice": {
+                "count": 85,
+                "num_samples": 1363,
+                "tasks": [],
+                "average_score": 0.38069986029874947
+            }
+        },
+        "input_num": {
+            "6-8 images": {
+                "count": 21,
+                "num_samples": 314,
+                "tasks": [],
+                "average_score": 0.21448412698412703
+            },
+            "9-image or more": {
+                "count": 41,
+                "num_samples": 623,
+                "tasks": [],
+                "average_score": 0.34991843422677277
+            },
+            "1-image": {
+                "count": 315,
+                "num_samples": 5228,
+                "tasks": [],
+                "average_score": 0.36487656334089386
+            },
+            "video": {
+                "count": 43,
+                "num_samples": 700,
+                "tasks": [],
+                "average_score": 0.3555910609857979
+            },
+            "4-5 images": {
+                "count": 34,
+                "num_samples": 520,
+                "tasks": [],
+                "average_score": 0.23950364354876252
+            },
+            "2-3 images": {
+                "count": 51,
+                "num_samples": 802,
+                "tasks": [],
+                "average_score": 0.31886513111201115
+            }
+        },
+        "app": {
+            "Information_Extraction": {
+                "count": 72,
+                "num_samples": 1124,
+                "tasks": [],
+                "average_score": 0.3972495309304478
+            },
+            "Planning": {
+                "count": 78,
+                "num_samples": 1239,
+                "tasks": [],
+                "average_score": 0.18098305857595157
+            },
+            "Coding": {
+                "count": 31,
+                "num_samples": 474,
+                "tasks": [],
+                "average_score": 0.30887234822244314
+            },
+            "Perception": {
+                "count": 145,
+                "num_samples": 2315,
+                "tasks": [],
+                "average_score": 0.39256038521661607
+            },
+            "Metrics": {
+                "count": 20,
+                "num_samples": 309,
+                "tasks": [],
+                "average_score": 0.44924313486983725
+            },
+            "Science": {
+                "count": 29,
+                "num_samples": 574,
+                "tasks": [],
+                "average_score": 0.2880278656037017
+            },
+            "Knowledge": {
+                "count": 97,
+                "num_samples": 1605,
+                "tasks": [],
+                "average_score": 0.4015531477048036
+            },
+            "Mathematics": {
+                "count": 33,
+                "num_samples": 547,
+                "tasks": [],
+                "average_score": 0.24179792538224956
+            }
+        }
+    }
+}

static/eval_results/Default/all_summary.json

@@ -1,525 +0,0 @@
-{
-    "GPT_4o": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.5203440930873326,
-            "micro_mean_score": 0.514302640282204
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.5265030595065238,
-            "micro_mean_score": 0.5236338521693411
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.6478225794744895,
-            "micro_mean_score": 0.665391229578676
-        },
-        "overall_score": 0.5421184432647768
-    },
-    "Gemini_1.5_pro_002": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.4699992918320008,
-            "micro_mean_score": 0.4651116133689296
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.4822473962867704,
-            "micro_mean_score": 0.4764805563057179
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.5858190649927173,
-            "micro_mean_score": 0.6104901117798793
-        },
-        "overall_score": 0.4955784031499121
-    },
-    "Gemini_1.5_flash_002": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.41898948981774853,
-            "micro_mean_score": 0.4127376993779598
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.4189319021967416,
-            "micro_mean_score": 0.41567515414375245
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.5691365176285039,
-            "micro_mean_score": 0.5987532244196045
-        },
-        "overall_score": 0.43831534488249924
-    },
-    "Claude_3.5": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.48800427486796155,
-            "micro_mean_score": 0.4814327812005499
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.5040975742801586,
-            "micro_mean_score": 0.5002259116666758
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.6373907158949892,
-            "micro_mean_score": 0.6569647463456579
-        },
-        "overall_score": 0.5212541172602853
-    },
-    "Claude_3.5_new": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.4919657684484185,
-            "micro_mean_score": 0.4874520567007144
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.5259191914020757,
-            "micro_mean_score": 0.5230785894131227
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.6563419761104125,
-            "micro_mean_score": 0.6724419604471196
-        },
-        "overall_score": 0.5427062825031487
-    },
-    "GPT_4o_mini": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.39854757130003565,
-            "micro_mean_score": 0.3936551517403452
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.40767494558789397,
-            "micro_mean_score": 0.40431644154143376
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.586537827213665,
-            "micro_mean_score": 0.6133276010318144
-        },
-        "overall_score": 0.43069690064863675
-    },
-    "Qwen2_VL_72B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.46406654108789214,
-            "micro_mean_score": 0.4584702152011697
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.4542376574527161,
-            "micro_mean_score": 0.4501201906164793
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.5639771804231668,
-            "micro_mean_score": 0.5835339638865004
-        },
-        "overall_score": 0.4769263263488681
-    },
-    "Qwen2_VL_7B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.3480020832611913,
-            "micro_mean_score": 0.3441858958345098
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.3293449599230247,
-            "micro_mean_score": 0.325331493515679
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1170,
-            "macro_mean_score": 0.43955105763038577,
-            "micro_mean_score": 0.45508547008546996
-        },
-        "overall_score": 0.3597856146156421
-    },
-    "llava_onevision_72B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.3199332158220174,
-            "micro_mean_score": 0.31770770553892647
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.2974368415462532,
-            "micro_mean_score": 0.2956217833156672
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.4599484231632498,
-            "micro_mean_score": 0.4850386930352536
-        },
-        "overall_score": 0.33795497518277007
-    },
-    "llava_onevision_7B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.22409531510496777,
-            "micro_mean_score": 0.22238854298563537
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.21362697219149712,
-            "micro_mean_score": 0.21073910058505504
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.33979975321921935,
-            "micro_mean_score": 0.36474634565778147
-        },
-        "overall_score": 0.23898796555531696
-    },
-    "InternVL2_76B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.3502244283768534,
-            "micro_mean_score": 0.3456783051732046
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.3562710424410931,
-            "micro_mean_score": 0.35129859801162616
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.5192997443033639,
-            "micro_mean_score": 0.5421324161650903
-        },
-        "overall_score": 0.3772549347599992
-    },
-    "InternVL2_8B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.25956581776451815,
-            "micro_mean_score": 0.2546984460483302
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.24090301358258295,
-            "micro_mean_score": 0.23819084111520938
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1165,
-            "macro_mean_score": 0.3978571701460552,
-            "micro_mean_score": 0.4108583690987125
-        },
-        "overall_score": 0.2773656948037259
-    },
-    "MiniCPM_v2.6": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.2287645706203155,
-            "micro_mean_score": 0.2249087742955901
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.22955895202146906,
-            "micro_mean_score": 0.22560399396899078
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.41728623355613875,
-            "micro_mean_score": 0.43452278589853827
-        },
-        "overall_score": 0.2537218694467236
-    },
-    "Phi-3.5-vision": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.23271251159409778,
-            "micro_mean_score": 0.2296262323791101
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.22995297916629392,
-            "micro_mean_score": 0.22708502951025372
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.3947914647737769,
-            "micro_mean_score": 0.42459157351676696
-        },
-        "overall_score": 0.25357415903306635
-    },
-    "Pixtral_12B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.31905695620134694,
-            "micro_mean_score": 0.31556607913724777
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.31362045151669854,
-            "micro_mean_score": 0.3100986209078182
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.4566234428542061,
-            "micro_mean_score": 0.4870593293207223
-        },
-        "overall_score": 0.33676353369131895
-    },
-    "Llama_3_2_11B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.10044261716549671,
-            "micro_mean_score": 0.09980638766828835
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.15999641916771298,
-            "micro_mean_score": 0.15809331016967038
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.3173342406187366,
-            "micro_mean_score": 0.3487962166809973
-        },
-        "overall_score": 0.1802478219287358
-    },
-    "Idefics3": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.11118980301103833,
-            "micro_mean_score": 0.11201785633274061
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.08956972487602757,
-            "micro_mean_score": 0.08982225274252693
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.3210866162255635,
-            "micro_mean_score": 0.35649183147033553
-        },
-        "overall_score": 0.138206224513898
-    },
-    "Aria": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.30485930718699694,
-            "micro_mean_score": 0.3016713629035311
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.289073788209904,
-            "micro_mean_score": 0.2859007507765791
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.5103725263180767,
-            "micro_mean_score": 0.5349957007738607
-        },
-        "overall_score": 0.3313115037088191
-    },
-    "NVLM": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.2420528895703979,
-            "micro_mean_score": 0.23838419989257642
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.21589726765847422,
-            "micro_mean_score": 0.21406043849932396
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.3478114310231307,
-            "micro_mean_score": 0.3947549441100602
-        },
-        "overall_score": 0.25566537510391796
-    },
-    "InternVL2_2B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.09089701489596874,
-            "micro_mean_score": 0.09036328295381871
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.13141974398938763,
-            "micro_mean_score": 0.13063500716262516
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.23864417043743646,
-            "micro_mean_score": 0.24901117798796224
-        },
-        "overall_score": 0.14522090778963154
-    },
-    "Qwen2_VL_2B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.16448220309703876,
-            "micro_mean_score": 0.1610710186451323
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.20877163406364055,
-            "micro_mean_score": 0.20561526268932287
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.3154302566225611,
-            "micro_mean_score": 0.33856405846947557
-        },
-        "overall_score": 0.22249997162072932
-    },
-    "Aquila_VL_2B": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.16317824309838627,
-            "micro_mean_score": 0.16198837245148487
-        },
-        "core_cot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.159970161379836,
-            "micro_mean_score": 0.15844711671722148
-        },
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.24567572098570653,
-            "micro_mean_score": 0.2704213241616509
-        },
-        "overall_score": 0.17379673035120966
-    },
-    "Mammoth_VL": {
-        "core_noncot": {
-            "num_eval_tasks": 440,
-            "num_eval_samples": 6539,
-            "num_not_eval_samples": 0,
-            "macro_mean_score": 0.264052880412689,
-            "micro_mean_score": 0.2626894374387823
-        },
-        "core_cot": null,
-        "open": {
-            "num_eval_tasks": 65,
-            "num_eval_samples": 1163,
-            "macro_mean_score": 0.37992668750165337,
-            "micro_mean_score": 0.40120378331900275
-        },
-        "overall_score": 0.27896733083008046
-    }
-}