Hugging Face's logo

Spaces:
TIGER-Lab
/
MEGA-Bench
Running

App Files Community
1
cccjc commited on
Commit
e062c74
·
1 Parent(s): dd2b9e8

GLM-4.1V-thinking

Browse files
Files changed (3) hide show
  1. constants.py +4 -0
  2. static/eval_results/Default/GLM_4.1V_Thinking/summary_and_keyword_stats.json +249 -0
  3. static/eval_results/Default/GLM_4.1V_Thinking/task_results.json +3537 -0
constants.py CHANGED
@@ -156,6 +156,7 @@ MODEL_NAME_MAP = {
156
  "InternVL3_78B": "InternVL3-78B",
157
  "Ovis2_8B": "Ovis2-8B",
158
  "Valley_Eagle_7B": "Valley-Eagle-7B",
 
159
  }
160
 
161
  DIMENSION_NAME_MAP = {
@@ -268,6 +269,7 @@ MODEL_URLS = {
268
  "Seed1.5-VL": "https://github.com/ByteDance-Seed/Seed1.5-VL",
269
  "Ovis2_8B": "https://huggingface.co/AIDC-AI/Ovis2-8B",
270
  "Valley_Eagle_7B": "https://huggingface.co/bytedance-research/Valley-Eagle-7B",
 
271
  }
272
 
273
  # Define the base MODEL_GROUPS structure
@@ -326,6 +328,7 @@ BASE_MODEL_GROUPS = {
326
  "InternVL3_14B",
327
  "Ovis2_8B",
328
  "Valley_Eagle_7B",
 
329
  ],
330
  "Proprietary Flagship models": [
331
  "Claude_3.5_new",
@@ -384,5 +387,6 @@ BASE_MODEL_GROUPS = {
384
  "InternVL3_14B",
385
  "Ovis2_8B",
386
  "Valley_Eagle_7B",
 
387
  ],
388
  }
 
156
  "InternVL3_78B": "InternVL3-78B",
157
  "Ovis2_8B": "Ovis2-8B",
158
  "Valley_Eagle_7B": "Valley-Eagle-7B",
159
+ "GLM_4.1V_Thinking": "GLM-4.1V-Thinking"
160
  }
161
 
162
  DIMENSION_NAME_MAP = {
 
269
  "Seed1.5-VL": "https://github.com/ByteDance-Seed/Seed1.5-VL",
270
  "Ovis2_8B": "https://huggingface.co/AIDC-AI/Ovis2-8B",
271
  "Valley_Eagle_7B": "https://huggingface.co/bytedance-research/Valley-Eagle-7B",
272
+ "GLM_4.1V_Thinking": "https://github.com/THUDM/GLM-4.1V-Thinking",
273
  }
274
 
275
  # Define the base MODEL_GROUPS structure
 
328
  "InternVL3_14B",
329
  "Ovis2_8B",
330
  "Valley_Eagle_7B",
331
+ "GLM_4.1V_Thinking",
332
  ],
333
  "Proprietary Flagship models": [
334
  "Claude_3.5_new",
 
387
  "InternVL3_14B",
388
  "Ovis2_8B",
389
  "Valley_Eagle_7B",
390
+ "GLM_4.1V_Thinking",
391
  ],
392
  }
static/eval_results/Default/GLM_4.1V_Thinking/summary_and_keyword_stats.json ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_summary": {
3
+ "core": {
4
+ "num_eval_tasks": 440,
5
+ "num_eval_samples": 6531,
6
+ "macro_mean_score": 0.4486926203304986
7
+ },
8
+ "open": {
9
+ "num_eval_tasks": 65,
10
+ "num_eval_samples": 1158,
11
+ "macro_mean_score": 0.6108150206588362
12
+ },
13
+ "overall_score": 0.46955985997672023
14
+ },
15
+ "keyword_stats": {
16
+ "skills": {
17
+ "Text Recognition (OCR)": {
18
+ "count": 137,
19
+ "num_samples": 2215,
20
+ "tasks": [],
21
+ "average_score": 0.5172106384499557
22
+ },
23
+ "Language Understanding and Generation": {
24
+ "count": 154,
25
+ "num_samples": 2483,
26
+ "tasks": [],
27
+ "average_score": 0.5237662937025751
28
+ },
29
+ "Object Recognition and Classification": {
30
+ "count": 303,
31
+ "num_samples": 4712,
32
+ "tasks": [],
33
+ "average_score": 0.4953188385475589
34
+ },
35
+ "Scene and Event Understanding": {
36
+ "count": 154,
37
+ "num_samples": 2456,
38
+ "tasks": [],
39
+ "average_score": 0.5105629002516974
40
+ },
41
+ "Commonsense and Social Reasoning": {
42
+ "count": 51,
43
+ "num_samples": 850,
44
+ "tasks": [],
45
+ "average_score": 0.5560890513552442
46
+ },
47
+ "Mathematical and Logical Reasoning": {
48
+ "count": 109,
49
+ "num_samples": 1907,
50
+ "tasks": [],
51
+ "average_score": 0.45661405612330974
52
+ },
53
+ "Spatial and Temporal Reasoning": {
54
+ "count": 152,
55
+ "num_samples": 2430,
56
+ "tasks": [],
57
+ "average_score": 0.35774817817177057
58
+ },
59
+ "Planning and Decision Making": {
60
+ "count": 37,
61
+ "num_samples": 569,
62
+ "tasks": [],
63
+ "average_score": 0.16432465676580915
64
+ },
65
+ "Domain-Specific Knowledge and Skills": {
66
+ "count": 77,
67
+ "num_samples": 1383,
68
+ "tasks": [],
69
+ "average_score": 0.42985988743711107
70
+ },
71
+ "Ethical and Safety Reasoning": {
72
+ "count": 15,
73
+ "num_samples": 241,
74
+ "tasks": [],
75
+ "average_score": 0.5999448621553886
76
+ }
77
+ },
78
+ "input_format": {
79
+ "User Interface Screenshots": {
80
+ "count": 93,
81
+ "num_samples": 1487,
82
+ "tasks": [],
83
+ "average_score": 0.5318353838949659
84
+ },
85
+ "Videos": {
86
+ "count": 43,
87
+ "num_samples": 698,
88
+ "tasks": [],
89
+ "average_score": 0.42670039925394876
90
+ },
91
+ "Artistic and Creative Content": {
92
+ "count": 32,
93
+ "num_samples": 539,
94
+ "tasks": [],
95
+ "average_score": 0.4294089306737516
96
+ },
97
+ "Photographs": {
98
+ "count": 143,
99
+ "num_samples": 2239,
100
+ "tasks": [],
101
+ "average_score": 0.4897370296181892
102
+ },
103
+ "Diagrams and Data Visualizations": {
104
+ "count": 101,
105
+ "num_samples": 1715,
106
+ "tasks": [],
107
+ "average_score": 0.5037553614522623
108
+ },
109
+ "Text-Based Images and Documents": {
110
+ "count": 82,
111
+ "num_samples": 1286,
112
+ "tasks": [],
113
+ "average_score": 0.3644006697811522
114
+ },
115
+ "3D Models and Aerial Imagery": {
116
+ "count": 11,
117
+ "num_samples": 167,
118
+ "tasks": [],
119
+ "average_score": 0.4350267247634551
120
+ }
121
+ },
122
+ "output_format": {
123
+ "open_ended_output": {
124
+ "count": 80,
125
+ "num_samples": 1444,
126
+ "tasks": [],
127
+ "average_score": 0.5514161725392729
128
+ },
129
+ "contextual_formatted_text": {
130
+ "count": 98,
131
+ "num_samples": 1497,
132
+ "tasks": [],
133
+ "average_score": 0.44313348433053745
134
+ },
135
+ "structured_output": {
136
+ "count": 110,
137
+ "num_samples": 1710,
138
+ "tasks": [],
139
+ "average_score": 0.4067253221794801
140
+ },
141
+ "numerical_data": {
142
+ "count": 49,
143
+ "num_samples": 860,
144
+ "tasks": [],
145
+ "average_score": 0.5012861564098088
146
+ },
147
+ "multiple_choice": {
148
+ "count": 85,
149
+ "num_samples": 1347,
150
+ "tasks": [],
151
+ "average_score": 0.5010447004582862
152
+ },
153
+ "exact_text": {
154
+ "count": 83,
155
+ "num_samples": 1273,
156
+ "tasks": [],
157
+ "average_score": 0.4541656310594156
158
+ }
159
+ },
160
+ "input_num": {
161
+ "1-image": {
162
+ "count": 315,
163
+ "num_samples": 5207,
164
+ "tasks": [],
165
+ "average_score": 0.5009527875194596
166
+ },
167
+ "video": {
168
+ "count": 43,
169
+ "num_samples": 698,
170
+ "tasks": [],
171
+ "average_score": 0.42670039925394876
172
+ },
173
+ "4-5 images": {
174
+ "count": 34,
175
+ "num_samples": 520,
176
+ "tasks": [],
177
+ "average_score": 0.30537346326745196
178
+ },
179
+ "6-8 images": {
180
+ "count": 21,
181
+ "num_samples": 309,
182
+ "tasks": [],
183
+ "average_score": 0.41089380196523057
184
+ },
185
+ "2-3 images": {
186
+ "count": 51,
187
+ "num_samples": 802,
188
+ "tasks": [],
189
+ "average_score": 0.3990376069267054
190
+ },
191
+ "9-image or more": {
192
+ "count": 41,
193
+ "num_samples": 595,
194
+ "tasks": [],
195
+ "average_score": 0.5272463050260753
196
+ }
197
+ },
198
+ "app": {
199
+ "Information_Extraction": {
200
+ "count": 72,
201
+ "num_samples": 1099,
202
+ "tasks": [],
203
+ "average_score": 0.5840742162440538
204
+ },
205
+ "Knowledge": {
206
+ "count": 97,
207
+ "num_samples": 1602,
208
+ "tasks": [],
209
+ "average_score": 0.5107183342025416
210
+ },
211
+ "Perception": {
212
+ "count": 145,
213
+ "num_samples": 2305,
214
+ "tasks": [],
215
+ "average_score": 0.5237620974413176
216
+ },
217
+ "Planning": {
218
+ "count": 78,
219
+ "num_samples": 1222,
220
+ "tasks": [],
221
+ "average_score": 0.2653984706794158
222
+ },
223
+ "Science": {
224
+ "count": 29,
225
+ "num_samples": 574,
226
+ "tasks": [],
227
+ "average_score": 0.4503686780767413
228
+ },
229
+ "Metrics": {
230
+ "count": 20,
231
+ "num_samples": 309,
232
+ "tasks": [],
233
+ "average_score": 0.40954569202797303
234
+ },
235
+ "Coding": {
236
+ "count": 31,
237
+ "num_samples": 474,
238
+ "tasks": [],
239
+ "average_score": 0.350875350140056
240
+ },
241
+ "Mathematics": {
242
+ "count": 33,
243
+ "num_samples": 546,
244
+ "tasks": [],
245
+ "average_score": 0.507860033330701
246
+ }
247
+ }
248
+ }
249
+ }
static/eval_results/Default/GLM_4.1V_Thinking/task_results.json ADDED
@@ -0,0 +1,3537 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "name": "GUI_Chat_Easy",
4
+ "score": 0.7230769230769231,
5
+ "eval_type": "llm",
6
+ "num_demo": 1,
7
+ "num_query": 26
8
+ },
9
+ {
10
+ "name": "GUI_Chat_Hard",
11
+ "score": 0.5807692307692307,
12
+ "eval_type": "llm",
13
+ "num_demo": 1,
14
+ "num_query": 26
15
+ },
16
+ {
17
+ "name": "activitynetqa",
18
+ "score": 0.5368421052631579,
19
+ "eval_type": "llm",
20
+ "num_demo": 1,
21
+ "num_query": 19
22
+ },
23
+ {
24
+ "name": "art_explanation",
25
+ "score": 0.603448275862069,
26
+ "eval_type": "llm",
27
+ "num_demo": 1,
28
+ "num_query": 29
29
+ },
30
+ {
31
+ "name": "ascii_art_30",
32
+ "score": 0.0,
33
+ "eval_type": "llm",
34
+ "num_demo": 1,
35
+ "num_query": 14
36
+ },
37
+ {
38
+ "name": "bar_chart_interpretation",
39
+ "score": 0.5724137931034482,
40
+ "eval_type": "llm",
41
+ "num_demo": 1,
42
+ "num_query": 29
43
+ },
44
+ {
45
+ "name": "bridge_strategies_advanced",
46
+ "score": 0.12142857142857143,
47
+ "eval_type": "llm",
48
+ "num_demo": 1,
49
+ "num_query": 14
50
+ },
51
+ {
52
+ "name": "bridge_strategies_expert",
53
+ "score": 0.2571428571428572,
54
+ "eval_type": "llm",
55
+ "num_demo": 1,
56
+ "num_query": 14
57
+ },
58
+ {
59
+ "name": "bridge_strategies_worldclass",
60
+ "score": 0.12142857142857144,
61
+ "eval_type": "llm",
62
+ "num_demo": 1,
63
+ "num_query": 14
64
+ },
65
+ {
66
+ "name": "defeasible_reasoning",
67
+ "score": 0.8103448275862069,
68
+ "eval_type": "llm",
69
+ "num_demo": 1,
70
+ "num_query": 29
71
+ },
72
+ {
73
+ "name": "doc_vqa",
74
+ "score": 0.7250000000000001,
75
+ "eval_type": "llm",
76
+ "num_demo": 1,
77
+ "num_query": 16
78
+ },
79
+ {
80
+ "name": "docci_image_description_long",
81
+ "score": 0.8285714285714286,
82
+ "eval_type": "llm",
83
+ "num_demo": 1,
84
+ "num_query": 14
85
+ },
86
+ {
87
+ "name": "electrocardiogram",
88
+ "score": 0.45714285714285713,
89
+ "eval_type": "llm",
90
+ "num_demo": 1,
91
+ "num_query": 14
92
+ },
93
+ {
94
+ "name": "figurative_speech_explanation",
95
+ "score": 0.789655172413793,
96
+ "eval_type": "llm",
97
+ "num_demo": 1,
98
+ "num_query": 29
99
+ },
100
+ {
101
+ "name": "funny_image_title",
102
+ "score": 0.5928571428571427,
103
+ "eval_type": "llm",
104
+ "num_demo": 1,
105
+ "num_query": 14
106
+ },
107
+ {
108
+ "name": "funqa_unexpected_action_creative_video",
109
+ "score": 0.4133333333333334,
110
+ "eval_type": "llm",
111
+ "num_demo": 1,
112
+ "num_query": 15
113
+ },
114
+ {
115
+ "name": "funqa_unexpected_action_humor_video",
116
+ "score": 0.37333333333333335,
117
+ "eval_type": "llm",
118
+ "num_demo": 1,
119
+ "num_query": 15
120
+ },
121
+ {
122
+ "name": "funqa_unexpected_action_magic_video",
123
+ "score": 0.6,
124
+ "eval_type": "llm",
125
+ "num_demo": 1,
126
+ "num_query": 15
127
+ },
128
+ {
129
+ "name": "generated_video_artifacts",
130
+ "score": 0.43125,
131
+ "eval_type": "llm",
132
+ "num_demo": 1,
133
+ "num_query": 16
134
+ },
135
+ {
136
+ "name": "graph_interpretation",
137
+ "score": 0.8379310344827585,
138
+ "eval_type": "llm",
139
+ "num_demo": 1,
140
+ "num_query": 29
141
+ },
142
+ {
143
+ "name": "guess_image_generation_prompt",
144
+ "score": 0.836842105263158,
145
+ "eval_type": "llm",
146
+ "num_demo": 1,
147
+ "num_query": 19
148
+ },
149
+ {
150
+ "name": "humor_explanation",
151
+ "score": 0.6533333333333334,
152
+ "eval_type": "llm",
153
+ "num_demo": 1,
154
+ "num_query": 15
155
+ },
156
+ {
157
+ "name": "image_captioning_with_additional_requirements",
158
+ "score": 0.9000000000000001,
159
+ "eval_type": "llm",
160
+ "num_demo": 1,
161
+ "num_query": 14
162
+ },
163
+ {
164
+ "name": "image_humor_understanding",
165
+ "score": 0.8172413793103449,
166
+ "eval_type": "llm",
167
+ "num_demo": 1,
168
+ "num_query": 29
169
+ },
170
+ {
171
+ "name": "iq_test_open_ended",
172
+ "score": 0.7586206896551724,
173
+ "eval_type": "llm",
174
+ "num_demo": 1,
175
+ "num_query": 29
176
+ },
177
+ {
178
+ "name": "meme_explain",
179
+ "score": 0.7857142857142857,
180
+ "eval_type": "llm",
181
+ "num_demo": 1,
182
+ "num_query": 14
183
+ },
184
+ {
185
+ "name": "multi_lingual_Ruozhiba_expalnation_Arabic",
186
+ "score": 0.5857142857142856,
187
+ "eval_type": "llm",
188
+ "num_demo": 1,
189
+ "num_query": 14
190
+ },
191
+ {
192
+ "name": "multi_lingual_Ruozhiba_expalnation_English",
193
+ "score": 0.6857142857142858,
194
+ "eval_type": "llm",
195
+ "num_demo": 1,
196
+ "num_query": 14
197
+ },
198
+ {
199
+ "name": "multi_lingual_Ruozhiba_expalnation_French",
200
+ "score": 0.6428571428571427,
201
+ "eval_type": "llm",
202
+ "num_demo": 1,
203
+ "num_query": 14
204
+ },
205
+ {
206
+ "name": "multi_lingual_Ruozhiba_expalnation_Japanese",
207
+ "score": 0.6,
208
+ "eval_type": "llm",
209
+ "num_demo": 1,
210
+ "num_query": 14
211
+ },
212
+ {
213
+ "name": "multi_lingual_Ruozhiba_expalnation_Russian",
214
+ "score": 0.5999999999999999,
215
+ "eval_type": "llm",
216
+ "num_demo": 1,
217
+ "num_query": 14
218
+ },
219
+ {
220
+ "name": "multi_lingual_Ruozhiba_expalnation_Spanish",
221
+ "score": 0.6428571428571427,
222
+ "eval_type": "llm",
223
+ "num_demo": 1,
224
+ "num_query": 14
225
+ },
226
+ {
227
+ "name": "multi_lingual_manual_explanation_scooter_Arabic",
228
+ "score": 0.37142857142857144,
229
+ "eval_type": "llm",
230
+ "num_demo": 0,
231
+ "num_query": 14
232
+ },
233
+ {
234
+ "name": "multi_lingual_manual_explanation_scooter_Chinese",
235
+ "score": 0.5071428571428572,
236
+ "eval_type": "llm",
237
+ "num_demo": 0,
238
+ "num_query": 14
239
+ },
240
+ {
241
+ "name": "multi_lingual_manual_explanation_scooter_French",
242
+ "score": 0.45714285714285713,
243
+ "eval_type": "llm",
244
+ "num_demo": 0,
245
+ "num_query": 14
246
+ },
247
+ {
248
+ "name": "multi_lingual_manual_explanation_scooter_Russian",
249
+ "score": 0.5,
250
+ "eval_type": "llm",
251
+ "num_demo": 0,
252
+ "num_query": 14
253
+ },
254
+ {
255
+ "name": "multi_lingual_manual_explanation_scooter_Spanish",
256
+ "score": 0.4928571428571429,
257
+ "eval_type": "llm",
258
+ "num_demo": 0,
259
+ "num_query": 14
260
+ },
261
+ {
262
+ "name": "nextqa_oe",
263
+ "score": 0.6,
264
+ "eval_type": "llm",
265
+ "num_demo": 1,
266
+ "num_query": 19
267
+ },
268
+ {
269
+ "name": "ocr_open_ended_qa",
270
+ "score": 0.8689655172413792,
271
+ "eval_type": "llm",
272
+ "num_demo": 1,
273
+ "num_query": 29
274
+ },
275
+ {
276
+ "name": "paper_review_writing",
277
+ "score": 0.6533333333333332,
278
+ "eval_type": "llm",
279
+ "num_demo": 1,
280
+ "num_query": 15
281
+ },
282
+ {
283
+ "name": "red_teaming_captcha",
284
+ "score": 0.10000000000000003,
285
+ "eval_type": "llm",
286
+ "num_demo": 1,
287
+ "num_query": 19
288
+ },
289
+ {
290
+ "name": "red_teaming_celebrity",
291
+ "score": 0.6500000000000001,
292
+ "eval_type": "llm",
293
+ "num_demo": 0,
294
+ "num_query": 20
295
+ },
296
+ {
297
+ "name": "red_teaming_jailbreak",
298
+ "score": 0.51,
299
+ "eval_type": "llm",
300
+ "num_demo": 0,
301
+ "num_query": 20
302
+ },
303
+ {
304
+ "name": "red_teaming_politics",
305
+ "score": 0.74,
306
+ "eval_type": "llm",
307
+ "num_demo": 0,
308
+ "num_query": 20
309
+ },
310
+ {
311
+ "name": "red_teaming_racial",
312
+ "score": 0.78,
313
+ "eval_type": "llm",
314
+ "num_demo": 0,
315
+ "num_query": 20
316
+ },
317
+ {
318
+ "name": "red_teaming_visual_order_A",
319
+ "score": 0.9,
320
+ "eval_type": "llm",
321
+ "num_demo": 1,
322
+ "num_query": 19
323
+ },
324
+ {
325
+ "name": "red_teaming_visual_order_B",
326
+ "score": 0.8894736842105264,
327
+ "eval_type": "llm",
328
+ "num_demo": 1,
329
+ "num_query": 19
330
+ },
331
+ {
332
+ "name": "red_teaming_visualmisleading",
333
+ "score": 0.8263157894736844,
334
+ "eval_type": "llm",
335
+ "num_demo": 1,
336
+ "num_query": 19
337
+ },
338
+ {
339
+ "name": "sceneqa_scene_transition_video",
340
+ "score": 0.4214285714285714,
341
+ "eval_type": "llm",
342
+ "num_demo": 1,
343
+ "num_query": 14
344
+ },
345
+ {
346
+ "name": "scibench_w_solution_open_ended",
347
+ "score": 0.41,
348
+ "eval_type": "llm",
349
+ "num_demo": 1,
350
+ "num_query": 25
351
+ },
352
+ {
353
+ "name": "science_figure_explanation",
354
+ "score": 0.8275862068965518,
355
+ "eval_type": "llm",
356
+ "num_demo": 1,
357
+ "num_query": 29
358
+ },
359
+ {
360
+ "name": "table2latex_complex",
361
+ "score": 0.6777777777777777,
362
+ "eval_type": "llm",
363
+ "num_demo": 1,
364
+ "num_query": 9
365
+ },
366
+ {
367
+ "name": "table_understanding_fetaqa",
368
+ "score": 0.6357142857142859,
369
+ "eval_type": "llm",
370
+ "num_demo": 1,
371
+ "num_query": 14
372
+ },
373
+ {
374
+ "name": "traffic_accident_analysis",
375
+ "score": 0.7357142857142858,
376
+ "eval_type": "llm",
377
+ "num_demo": 1,
378
+ "num_query": 14
379
+ },
380
+ {
381
+ "name": "tweets_captioning",
382
+ "score": 0.5428571428571428,
383
+ "eval_type": "llm",
384
+ "num_demo": 1,
385
+ "num_query": 14
386
+ },
387
+ {
388
+ "name": "unusual_images",
389
+ "score": 0.8172413793103449,
390
+ "eval_type": "llm",
391
+ "num_demo": 1,
392
+ "num_query": 29
393
+ },
394
+ {
395
+ "name": "vibe_eval_phrase",
396
+ "score": 0.7000000000000001,
397
+ "eval_type": "llm",
398
+ "num_demo": 1,
399
+ "num_query": 14
400
+ },
401
+ {
402
+ "name": "video2notes",
403
+ "score": 0.5714285714285714,
404
+ "eval_type": "llm",
405
+ "num_demo": 1,
406
+ "num_query": 14
407
+ },
408
+ {
409
+ "name": "video_content_follow_up",
410
+ "score": 0.7928571428571429,
411
+ "eval_type": "llm",
412
+ "num_demo": 1,
413
+ "num_query": 14
414
+ },
415
+ {
416
+ "name": "video_detail_description",
417
+ "score": 0.6578947368421053,
418
+ "eval_type": "llm",
419
+ "num_demo": 1,
420
+ "num_query": 19
421
+ },
422
+ {
423
+ "name": "video_qa",
424
+ "score": 0.7928571428571428,
425
+ "eval_type": "llm",
426
+ "num_demo": 1,
427
+ "num_query": 14
428
+ },
429
+ {
430
+ "name": "video_short_title",
431
+ "score": 0.6142857142857142,
432
+ "eval_type": "llm",
433
+ "num_demo": 1,
434
+ "num_query": 14
435
+ },
436
+ {
437
+ "name": "video_summary",
438
+ "score": 0.6285714285714284,
439
+ "eval_type": "llm",
440
+ "num_demo": 1,
441
+ "num_query": 14
442
+ },
443
+ {
444
+ "name": "visualization_with_code",
445
+ "score": 0.5785714285714286,
446
+ "eval_type": "llm",
447
+ "num_demo": 1,
448
+ "num_query": 14
449
+ },
450
+ {
451
+ "name": "wikihow_complex_task_completion",
452
+ "score": 0.5666666666666667,
453
+ "eval_type": "llm",
454
+ "num_demo": 1,
455
+ "num_query": 9
456
+ },
457
+ {
458
+ "name": "2d_image_jigsaw_puzzle_easy",
459
+ "score": 0.1619047619047619,
460
+ "eval_type": "rule",
461
+ "num_demo": 1,
462
+ "num_query": 14
463
+ },
464
+ {
465
+ "name": "3d_fragments_understanding",
466
+ "score": 0.14285714285714285,
467
+ "eval_type": "rule",
468
+ "num_demo": 1,
469
+ "num_query": 14
470
+ },
471
+ {
472
+ "name": "3d_indoor_scene_text_bbox_prediction",
473
+ "score": 0.2743696026501071,
474
+ "eval_type": "rule",
475
+ "num_demo": 0,
476
+ "num_query": 14
477
+ },
478
+ {
479
+ "name": "3d_indoor_scene_text_bbox_selection",
480
+ "score": 0.7142857142857143,
481
+ "eval_type": "rule",
482
+ "num_demo": 0,
483
+ "num_query": 14
484
+ },
485
+ {
486
+ "name": "Ad_count_detection",
487
+ "score": 0.6071428571428571,
488
+ "eval_type": "rule",
489
+ "num_demo": 1,
490
+ "num_query": 14
491
+ },
492
+ {
493
+ "name": "GUI_Act_Mobile_swipe",
494
+ "score": 0.5002159519380676,
495
+ "eval_type": "rule",
496
+ "num_demo": 1,
497
+ "num_query": 13
498
+ },
499
+ {
500
+ "name": "GUI_Act_Mobile_tap",
501
+ "score": 0.21428571428571427,
502
+ "eval_type": "rule",
503
+ "num_demo": 1,
504
+ "num_query": 14
505
+ },
506
+ {
507
+ "name": "GUI_Act_Web_Multi",
508
+ "score": 0.42857142857142855,
509
+ "eval_type": "rule",
510
+ "num_demo": 1,
511
+ "num_query": 14
512
+ },
513
+ {
514
+ "name": "GUI_Act_Web_Single",
515
+ "score": 0.0,
516
+ "eval_type": "rule",
517
+ "num_demo": 1,
518
+ "num_query": 14
519
+ },
520
+ {
521
+ "name": "IAM_line_ocr_and_locate",
522
+ "score": 0.5600541516009605,
523
+ "eval_type": "rule",
524
+ "num_demo": 1,
525
+ "num_query": 14
526
+ },
527
+ {
528
+ "name": "MMMU_physics_chemistry_selected",
529
+ "score": 0.7142857142857143,
530
+ "eval_type": "rule",
531
+ "num_demo": 1,
532
+ "num_query": 14
533
+ },
534
+ {
535
+ "name": "MMMU_pro_exam_screenshot",
536
+ "score": 0.48484848484848486,
537
+ "eval_type": "rule",
538
+ "num_demo": 1,
539
+ "num_query": 99
540
+ },
541
+ {
542
+ "name": "MMSoc_HatefulMemes",
543
+ "score": 0.7142857142857143,
544
+ "eval_type": "rule",
545
+ "num_demo": 1,
546
+ "num_query": 14
547
+ },
548
+ {
549
+ "name": "MMSoc_Memotion",
550
+ "score": 0.611764705882353,
551
+ "eval_type": "rule",
552
+ "num_demo": 1,
553
+ "num_query": 17
554
+ },
555
+ {
556
+ "name": "MMSoc_Misinformation_GossipCop",
557
+ "score": 0.7142857142857143,
558
+ "eval_type": "rule",
559
+ "num_demo": 1,
560
+ "num_query": 14
561
+ },
562
+ {
563
+ "name": "MMSoc_Misinformation_PolitiFact",
564
+ "score": 0.7857142857142857,
565
+ "eval_type": "rule",
566
+ "num_demo": 1,
567
+ "num_query": 14
568
+ },
569
+ {
570
+ "name": "TRANCE_physics_reasoning_basic",
571
+ "score": 0.8823529411764706,
572
+ "eval_type": "rule",
573
+ "num_demo": 1,
574
+ "num_query": 17
575
+ },
576
+ {
577
+ "name": "TRANCE_physics_reasoning_event",
578
+ "score": 0.21428571428571427,
579
+ "eval_type": "rule",
580
+ "num_demo": 1,
581
+ "num_query": 14
582
+ },
583
+ {
584
+ "name": "TRANCE_physics_reasoning_view",
585
+ "score": 0.07142857142857142,
586
+ "eval_type": "rule",
587
+ "num_demo": 1,
588
+ "num_query": 14
589
+ },
590
+ {
591
+ "name": "TV_show_info_parsing",
592
+ "score": 0.753968253968254,
593
+ "eval_type": "rule",
594
+ "num_demo": 1,
595
+ "num_query": 14
596
+ },
597
+ {
598
+ "name": "action_sequence_understanding",
599
+ "score": 0.5,
600
+ "eval_type": "rule",
601
+ "num_demo": 1,
602
+ "num_query": 14
603
+ },
604
+ {
605
+ "name": "actor_recognition_in_Movie",
606
+ "score": 0.5,
607
+ "eval_type": "rule",
608
+ "num_demo": 1,
609
+ "num_query": 14
610
+ },
611
+ {
612
+ "name": "adapted_cvbench_count",
613
+ "score": 0.7142857142857143,
614
+ "eval_type": "rule",
615
+ "num_demo": 1,
616
+ "num_query": 14
617
+ },
618
+ {
619
+ "name": "adapted_cvbench_depth",
620
+ "score": 0.8571428571428571,
621
+ "eval_type": "rule",
622
+ "num_demo": 1,
623
+ "num_query": 14
624
+ },
625
+ {
626
+ "name": "adapted_cvbench_distance",
627
+ "score": 0.6428571428571429,
628
+ "eval_type": "rule",
629
+ "num_demo": 1,
630
+ "num_query": 14
631
+ },
632
+ {
633
+ "name": "adapted_cvbench_relation",
634
+ "score": 0.2857142857142857,
635
+ "eval_type": "rule",
636
+ "num_demo": 1,
637
+ "num_query": 14
638
+ },
639
+ {
640
+ "name": "algebra",
641
+ "score": 0.2857142857142857,
642
+ "eval_type": "rule",
643
+ "num_demo": 1,
644
+ "num_query": 14
645
+ },
646
+ {
647
+ "name": "ancient_map_understanding",
648
+ "score": 0.9285714285714286,
649
+ "eval_type": "rule",
650
+ "num_demo": 1,
651
+ "num_query": 14
652
+ },
653
+ {
654
+ "name": "animal_pose_estimation",
655
+ "score": 0.3256744538032678,
656
+ "eval_type": "rule",
657
+ "num_demo": 1,
658
+ "num_query": 14
659
+ },
660
+ {
661
+ "name": "annoying_word_search",
662
+ "score": 0.0,
663
+ "eval_type": "rule",
664
+ "num_demo": 1,
665
+ "num_query": 14
666
+ },
667
+ {
668
+ "name": "app_interactive_operations_alipay",
669
+ "score": 0.6470588235294118,
670
+ "eval_type": "rule",
671
+ "num_demo": 0,
672
+ "num_query": 17
673
+ },
674
+ {
675
+ "name": "app_interactive_operations_amazon",
676
+ "score": 0.6428571428571429,
677
+ "eval_type": "rule",
678
+ "num_demo": 0,
679
+ "num_query": 14
680
+ },
681
+ {
682
+ "name": "app_interactive_operations_excel",
683
+ "score": 0.42857142857142855,
684
+ "eval_type": "rule",
685
+ "num_demo": 0,
686
+ "num_query": 14
687
+ },
688
+ {
689
+ "name": "app_interactive_operations_instagram",
690
+ "score": 0.7142857142857143,
691
+ "eval_type": "rule",
692
+ "num_demo": 0,
693
+ "num_query": 14
694
+ },
695
+ {
696
+ "name": "app_interactive_operations_iphone_settings",
697
+ "score": 0.7857142857142857,
698
+ "eval_type": "rule",
699
+ "num_demo": 0,
700
+ "num_query": 14
701
+ },
702
+ {
703
+ "name": "app_interactive_operations_leetcode",
704
+ "score": 0.6428571428571429,
705
+ "eval_type": "rule",
706
+ "num_demo": 0,
707
+ "num_query": 14
708
+ },
709
+ {
710
+ "name": "app_interactive_operations_notes",
711
+ "score": 0.42857142857142855,
712
+ "eval_type": "rule",
713
+ "num_demo": 0,
714
+ "num_query": 14
715
+ },
716
+ {
717
+ "name": "app_interactive_operations_ppt",
718
+ "score": 0.7142857142857143,
719
+ "eval_type": "rule",
720
+ "num_demo": 0,
721
+ "num_query": 14
722
+ },
723
+ {
724
+ "name": "app_interactive_operations_tiktok",
725
+ "score": 0.7142857142857143,
726
+ "eval_type": "rule",
727
+ "num_demo": 0,
728
+ "num_query": 14
729
+ },
730
+ {
731
+ "name": "app_interactive_operations_twitter",
732
+ "score": 0.7142857142857143,
733
+ "eval_type": "rule",
734
+ "num_demo": 0,
735
+ "num_query": 14
736
+ },
737
+ {
738
+ "name": "app_interactive_operations_word",
739
+ "score": 0.35714285714285715,
740
+ "eval_type": "rule",
741
+ "num_demo": 0,
742
+ "num_query": 14
743
+ },
744
+ {
745
+ "name": "app_interactive_operations_youtube",
746
+ "score": 0.8571428571428571,
747
+ "eval_type": "rule",
748
+ "num_demo": 0,
749
+ "num_query": 14
750
+ },
751
+ {
752
+ "name": "app_interactive_operations_zoom",
753
+ "score": 0.4666666666666667,
754
+ "eval_type": "rule",
755
+ "num_demo": 0,
756
+ "num_query": 15
757
+ },
758
+ {
759
+ "name": "app_layout_understanding_alipay",
760
+ "score": 0.6470588235294118,
761
+ "eval_type": "rule",
762
+ "num_demo": 1,
763
+ "num_query": 17
764
+ },
765
+ {
766
+ "name": "app_layout_understanding_amazon",
767
+ "score": 0.35714285714285715,
768
+ "eval_type": "rule",
769
+ "num_demo": 1,
770
+ "num_query": 14
771
+ },
772
+ {
773
+ "name": "app_layout_understanding_excel",
774
+ "score": 0.42857142857142855,
775
+ "eval_type": "rule",
776
+ "num_demo": 1,
777
+ "num_query": 14
778
+ },
779
+ {
780
+ "name": "app_layout_understanding_instagram",
781
+ "score": 0.7857142857142857,
782
+ "eval_type": "rule",
783
+ "num_demo": 1,
784
+ "num_query": 14
785
+ },
786
+ {
787
+ "name": "app_layout_understanding_iphone_settings",
788
+ "score": 0.7857142857142857,
789
+ "eval_type": "rule",
790
+ "num_demo": 1,
791
+ "num_query": 14
792
+ },
793
+ {
794
+ "name": "app_layout_understanding_leetcode",
795
+ "score": 0.2857142857142857,
796
+ "eval_type": "rule",
797
+ "num_demo": 1,
798
+ "num_query": 14
799
+ },
800
+ {
801
+ "name": "app_layout_understanding_notes",
802
+ "score": 0.42857142857142855,
803
+ "eval_type": "rule",
804
+ "num_demo": 1,
805
+ "num_query": 14
806
+ },
807
+ {
808
+ "name": "app_layout_understanding_ppt",
809
+ "score": 0.6428571428571429,
810
+ "eval_type": "rule",
811
+ "num_demo": 1,
812
+ "num_query": 14
813
+ },
814
+ {
815
+ "name": "app_layout_understanding_tiktok",
816
+ "score": 0.5714285714285714,
817
+ "eval_type": "rule",
818
+ "num_demo": 1,
819
+ "num_query": 14
820
+ },
821
+ {
822
+ "name": "app_layout_understanding_twitter",
823
+ "score": 0.7142857142857143,
824
+ "eval_type": "rule",
825
+ "num_demo": 1,
826
+ "num_query": 14
827
+ },
828
+ {
829
+ "name": "app_layout_understanding_word",
830
+ "score": 0.6428571428571429,
831
+ "eval_type": "rule",
832
+ "num_demo": 1,
833
+ "num_query": 14
834
+ },
835
+ {
836
+ "name": "app_layout_understanding_youtube",
837
+ "score": 0.7142857142857143,
838
+ "eval_type": "rule",
839
+ "num_demo": 1,
840
+ "num_query": 14
841
+ },
842
+ {
843
+ "name": "app_layout_understanding_zoom",
844
+ "score": 0.6,
845
+ "eval_type": "rule",
846
+ "num_demo": 1,
847
+ "num_query": 15
848
+ },
849
+ {
850
+ "name": "arc_agi",
851
+ "score": 0.0,
852
+ "eval_type": "rule",
853
+ "num_demo": 1,
854
+ "num_query": 14
855
+ },
856
+ {
857
+ "name": "arxiv_vqa",
858
+ "score": 0.7857142857142857,
859
+ "eval_type": "rule",
860
+ "num_demo": 1,
861
+ "num_query": 14
862
+ },
863
+ {
864
+ "name": "ascii_art_understanding",
865
+ "score": 0.5,
866
+ "eval_type": "rule",
867
+ "num_demo": 1,
868
+ "num_query": 14
869
+ },
870
+ {
871
+ "name": "autonomous_driving_scene_analysis",
872
+ "score": 0.9285714285714286,
873
+ "eval_type": "rule",
874
+ "num_demo": 1,
875
+ "num_query": 14
876
+ },
877
+ {
878
+ "name": "autorater_3d_model_texturing",
879
+ "score": 0.6428571428571429,
880
+ "eval_type": "rule",
881
+ "num_demo": 1,
882
+ "num_query": 14
883
+ },
884
+ {
885
+ "name": "autorater_aesthetics",
886
+ "score": 0.6428571428571429,
887
+ "eval_type": "rule",
888
+ "num_demo": 1,
889
+ "num_query": 14
890
+ },
891
+ {
892
+ "name": "autorater_artifact",
893
+ "score": 0.21428571428571427,
894
+ "eval_type": "rule",
895
+ "num_demo": 1,
896
+ "num_query": 14
897
+ },
898
+ {
899
+ "name": "autorater_artifact_reason",
900
+ "score": 0.3333333333333333,
901
+ "eval_type": "rule",
902
+ "num_demo": 0,
903
+ "num_query": 15
904
+ },
905
+ {
906
+ "name": "autorater_control",
907
+ "score": 0.7857142857142857,
908
+ "eval_type": "rule",
909
+ "num_demo": 1,
910
+ "num_query": 14
911
+ },
912
+ {
913
+ "name": "autorater_mask",
914
+ "score": 0.2857142857142857,
915
+ "eval_type": "rule",
916
+ "num_demo": 1,
917
+ "num_query": 14
918
+ },
919
+ {
920
+ "name": "autorater_motion_guided_editing",
921
+ "score": 0.14285714285714285,
922
+ "eval_type": "rule",
923
+ "num_demo": 1,
924
+ "num_query": 14
925
+ },
926
+ {
927
+ "name": "autorater_semantics",
928
+ "score": 0.7857142857142857,
929
+ "eval_type": "rule",
930
+ "num_demo": 1,
931
+ "num_query": 14
932
+ },
933
+ {
934
+ "name": "autorater_subject",
935
+ "score": 0.5714285714285714,
936
+ "eval_type": "rule",
937
+ "num_demo": 1,
938
+ "num_query": 14
939
+ },
940
+ {
941
+ "name": "autorater_unmask",
942
+ "score": 0.21428571428571427,
943
+ "eval_type": "rule",
944
+ "num_demo": 1,
945
+ "num_query": 14
946
+ },
947
+ {
948
+ "name": "av_human_multiview_counting",
949
+ "score": 0.13333333333333333,
950
+ "eval_type": "rule",
951
+ "num_demo": 1,
952
+ "num_query": 15
953
+ },
954
+ {
955
+ "name": "av_multicamera_tracking_predict_bbox",
956
+ "score": 0.01836734693877553,
957
+ "eval_type": "rule",
958
+ "num_demo": 1,
959
+ "num_query": 14
960
+ },
961
+ {
962
+ "name": "av_vehicle_multiview_counting",
963
+ "score": 0.13333333333333333,
964
+ "eval_type": "rule",
965
+ "num_demo": 1,
966
+ "num_query": 15
967
+ },
968
+ {
969
+ "name": "av_view_identification",
970
+ "score": 0.17777777777777776,
971
+ "eval_type": "rule",
972
+ "num_demo": 1,
973
+ "num_query": 15
974
+ },
975
+ {
976
+ "name": "average_humidity_estimate_plot",
977
+ "score": 0.724,
978
+ "eval_type": "rule",
979
+ "num_demo": 1,
980
+ "num_query": 15
981
+ },
982
+ {
983
+ "name": "background_change",
984
+ "score": 0.7142857142857143,
985
+ "eval_type": "rule",
986
+ "num_demo": 1,
987
+ "num_query": 14
988
+ },
989
+ {
990
+ "name": "ball_cup_swap_3",
991
+ "score": 0.21428571428571427,
992
+ "eval_type": "rule",
993
+ "num_demo": 1,
994
+ "num_query": 14
995
+ },
996
+ {
997
+ "name": "biology_exams_v",
998
+ "score": 0.42857142857142855,
999
+ "eval_type": "rule",
1000
+ "num_demo": 1,
1001
+ "num_query": 14
1002
+ },
1003
+ {
1004
+ "name": "bongard_problem",
1005
+ "score": 0.23684210526315788,
1006
+ "eval_type": "rule",
1007
+ "num_demo": 1,
1008
+ "num_query": 19
1009
+ },
1010
+ {
1011
+ "name": "booking_web_rating",
1012
+ "score": 1.0,
1013
+ "eval_type": "rule",
1014
+ "num_demo": 0,
1015
+ "num_query": 14
1016
+ },
1017
+ {
1018
+ "name": "booking_web_recommendation",
1019
+ "score": 0.8092403628117913,
1020
+ "eval_type": "rule",
1021
+ "num_demo": 0,
1022
+ "num_query": 14
1023
+ },
1024
+ {
1025
+ "name": "brand_logo_recognition_and_elaboration",
1026
+ "score": 0.68,
1027
+ "eval_type": "rule",
1028
+ "num_demo": 1,
1029
+ "num_query": 25
1030
+ },
1031
+ {
1032
+ "name": "calendar_schedule_suggestion",
1033
+ "score": 0.2857142857142857,
1034
+ "eval_type": "rule",
1035
+ "num_demo": 0,
1036
+ "num_query": 14
1037
+ },
1038
+ {
1039
+ "name": "cam_traj_to_video_selection",
1040
+ "score": 0.42857142857142855,
1041
+ "eval_type": "rule",
1042
+ "num_demo": 1,
1043
+ "num_query": 14
1044
+ },
1045
+ {
1046
+ "name": "character_recognition_in_TV_shows",
1047
+ "score": 0.42857142857142855,
1048
+ "eval_type": "rule",
1049
+ "num_demo": 1,
1050
+ "num_query": 14
1051
+ },
1052
+ {
1053
+ "name": "chart_vqa",
1054
+ "score": 0.7142857142857143,
1055
+ "eval_type": "rule",
1056
+ "num_demo": 1,
1057
+ "num_query": 14
1058
+ },
1059
+ {
1060
+ "name": "cheapest_flight_identification",
1061
+ "score": 0.35714285714285715,
1062
+ "eval_type": "rule",
1063
+ "num_demo": 1,
1064
+ "num_query": 14
1065
+ },
1066
+ {
1067
+ "name": "chemistry_exams_v",
1068
+ "score": 0.14285714285714285,
1069
+ "eval_type": "rule",
1070
+ "num_demo": 1,
1071
+ "num_query": 14
1072
+ },
1073
+ {
1074
+ "name": "chess_find_legal_moves",
1075
+ "score": 0.01263043897090046,
1076
+ "eval_type": "rule",
1077
+ "num_demo": 1,
1078
+ "num_query": 14
1079
+ },
1080
+ {
1081
+ "name": "chess_puzzle_single_step",
1082
+ "score": 0.0,
1083
+ "eval_type": "rule",
1084
+ "num_demo": 1,
1085
+ "num_query": 15
1086
+ },
1087
+ {
1088
+ "name": "chess_puzzles_checkmate",
1089
+ "score": 0.0,
1090
+ "eval_type": "rule",
1091
+ "num_demo": 1,
1092
+ "num_query": 14
1093
+ },
1094
+ {
1095
+ "name": "chess_puzzles_crushing",
1096
+ "score": 0.0,
1097
+ "eval_type": "rule",
1098
+ "num_demo": 1,
1099
+ "num_query": 14
1100
+ },
1101
+ {
1102
+ "name": "chess_puzzles_equality",
1103
+ "score": 0.0,
1104
+ "eval_type": "rule",
1105
+ "num_demo": 1,
1106
+ "num_query": 15
1107
+ },
1108
+ {
1109
+ "name": "chess_sygyzy_endgames",
1110
+ "score": 0.09867851446798814,
1111
+ "eval_type": "rule",
1112
+ "num_demo": 1,
1113
+ "num_query": 14
1114
+ },
1115
+ {
1116
+ "name": "chess_winner_identification",
1117
+ "score": 0.3333333333333333,
1118
+ "eval_type": "rule",
1119
+ "num_demo": 1,
1120
+ "num_query": 15
1121
+ },
1122
+ {
1123
+ "name": "chinese_idiom_recognition",
1124
+ "score": 0.2857142857142857,
1125
+ "eval_type": "rule",
1126
+ "num_demo": 1,
1127
+ "num_query": 14
1128
+ },
1129
+ {
1130
+ "name": "circuit_diagram_understanding",
1131
+ "score": 0.4666666666666667,
1132
+ "eval_type": "rule",
1133
+ "num_demo": 1,
1134
+ "num_query": 15
1135
+ },
1136
+ {
1137
+ "name": "clevr_arithmetic",
1138
+ "score": 0.7894736842105263,
1139
+ "eval_type": "rule",
1140
+ "num_demo": 1,
1141
+ "num_query": 19
1142
+ },
1143
+ {
1144
+ "name": "clevrer_moving_direction_video",
1145
+ "score": 0.1875,
1146
+ "eval_type": "rule",
1147
+ "num_demo": 1,
1148
+ "num_query": 16
1149
+ },
1150
+ {
1151
+ "name": "clevrer_object_existence_video",
1152
+ "score": 0.875,
1153
+ "eval_type": "rule",
1154
+ "num_demo": 1,
1155
+ "num_query": 16
1156
+ },
1157
+ {
1158
+ "name": "clevrer_physics",
1159
+ "score": 0.4,
1160
+ "eval_type": "rule",
1161
+ "num_demo": 1,
1162
+ "num_query": 20
1163
+ },
1164
+ {
1165
+ "name": "clevrer_video_moving_object_count",
1166
+ "score": 0.6666666666666666,
1167
+ "eval_type": "rule",
1168
+ "num_demo": 1,
1169
+ "num_query": 21
1170
+ },
1171
+ {
1172
+ "name": "clevrer_video_moving_object_property_recognition",
1173
+ "score": 1.0,
1174
+ "eval_type": "rule",
1175
+ "num_demo": 1,
1176
+ "num_query": 16
1177
+ },
1178
+ {
1179
+ "name": "clip_stable_diffusion_generate",
1180
+ "score": 0.42857142857142855,
1181
+ "eval_type": "rule",
1182
+ "num_demo": 1,
1183
+ "num_query": 14
1184
+ },
1185
+ {
1186
+ "name": "coco_object_detection_by_query_property",
1187
+ "score": 0.4483204700365899,
1188
+ "eval_type": "rule",
1189
+ "num_demo": 1,
1190
+ "num_query": 14
1191
+ },
1192
+ {
1193
+ "name": "coco_ood_global_image_retrieval_by_query_property",
1194
+ "score": 0.7505952380952381,
1195
+ "eval_type": "rule",
1196
+ "num_demo": 0,
1197
+ "num_query": 14
1198
+ },
1199
+ {
1200
+ "name": "coco_person_detection",
1201
+ "score": 0.7107148123741729,
1202
+ "eval_type": "rule",
1203
+ "num_demo": 1,
1204
+ "num_query": 14
1205
+ },
1206
+ {
1207
+ "name": "code_add_tag",
1208
+ "score": 0.3333333333333333,
1209
+ "eval_type": "rule",
1210
+ "num_demo": 1,
1211
+ "num_query": 15
1212
+ },
1213
+ {
1214
+ "name": "code_error_line_identification",
1215
+ "score": 0.35714285714285715,
1216
+ "eval_type": "rule",
1217
+ "num_demo": 1,
1218
+ "num_query": 14
1219
+ },
1220
+ {
1221
+ "name": "code_execution",
1222
+ "score": 0.4375,
1223
+ "eval_type": "rule",
1224
+ "num_demo": 1,
1225
+ "num_query": 16
1226
+ },
1227
+ {
1228
+ "name": "code_match_problem",
1229
+ "score": 0.14285714285714285,
1230
+ "eval_type": "rule",
1231
+ "num_demo": 1,
1232
+ "num_query": 14
1233
+ },
1234
+ {
1235
+ "name": "code_output_result",
1236
+ "score": 0.07142857142857142,
1237
+ "eval_type": "rule",
1238
+ "num_demo": 1,
1239
+ "num_query": 14
1240
+ },
1241
+ {
1242
+ "name": "code_programming_extremely_hard",
1243
+ "score": 0.0,
1244
+ "eval_type": "rule",
1245
+ "num_demo": 1,
1246
+ "num_query": 16
1247
+ },
1248
+ {
1249
+ "name": "code_programming_test_advanced",
1250
+ "score": 0.05555555555555555,
1251
+ "eval_type": "rule",
1252
+ "num_demo": 1,
1253
+ "num_query": 18
1254
+ },
1255
+ {
1256
+ "name": "code_programming_test_easy",
1257
+ "score": 0.16666666666666666,
1258
+ "eval_type": "rule",
1259
+ "num_demo": 1,
1260
+ "num_query": 24
1261
+ },
1262
+ {
1263
+ "name": "code_programming_test_hard",
1264
+ "score": 0.0,
1265
+ "eval_type": "rule",
1266
+ "num_demo": 1,
1267
+ "num_query": 14
1268
+ },
1269
+ {
1270
+ "name": "code_retrieval",
1271
+ "score": 0.35714285714285715,
1272
+ "eval_type": "rule",
1273
+ "num_demo": 1,
1274
+ "num_query": 14
1275
+ },
1276
+ {
1277
+ "name": "code_solution_compare",
1278
+ "score": 0.2857142857142857,
1279
+ "eval_type": "rule",
1280
+ "num_demo": 1,
1281
+ "num_query": 14
1282
+ },
1283
+ {
1284
+ "name": "code_translation_Python",
1285
+ "score": 0.0,
1286
+ "eval_type": "rule",
1287
+ "num_demo": 1,
1288
+ "num_query": 16
1289
+ },
1290
+ {
1291
+ "name": "code_translation_advanced",
1292
+ "score": 0.0,
1293
+ "eval_type": "rule",
1294
+ "num_demo": 1,
1295
+ "num_query": 14
1296
+ },
1297
+ {
1298
+ "name": "code_translation_easy",
1299
+ "score": 0.0,
1300
+ "eval_type": "rule",
1301
+ "num_demo": 1,
1302
+ "num_query": 14
1303
+ },
1304
+ {
1305
+ "name": "code_translation_hard",
1306
+ "score": 0.0,
1307
+ "eval_type": "rule",
1308
+ "num_demo": 1,
1309
+ "num_query": 14
1310
+ },
1311
+ {
1312
+ "name": "code_visualization_output_understanding",
1313
+ "score": 0.3,
1314
+ "eval_type": "rule",
1315
+ "num_demo": 1,
1316
+ "num_query": 10
1317
+ },
1318
+ {
1319
+ "name": "comic_page_ordering",
1320
+ "score": 0.0,
1321
+ "eval_type": "rule",
1322
+ "num_demo": 1,
1323
+ "num_query": 14
1324
+ },
1325
+ {
1326
+ "name": "contain_contain_images",
1327
+ "score": 0.9333333333333333,
1328
+ "eval_type": "rule",
1329
+ "num_demo": 0,
1330
+ "num_query": 15
1331
+ },
1332
+ {
1333
+ "name": "contain_contain_length",
1334
+ "score": 0.8666666666666667,
1335
+ "eval_type": "rule",
1336
+ "num_demo": 0,
1337
+ "num_query": 15
1338
+ },
1339
+ {
1340
+ "name": "contain_length",
1341
+ "score": 0.5333333333333333,
1342
+ "eval_type": "rule",
1343
+ "num_demo": 0,
1344
+ "num_query": 15
1345
+ },
1346
+ {
1347
+ "name": "contain_position_images",
1348
+ "score": 0.6666666666666666,
1349
+ "eval_type": "rule",
1350
+ "num_demo": 0,
1351
+ "num_query": 15
1352
+ },
1353
+ {
1354
+ "name": "contain_position_length",
1355
+ "score": 0.7333333333333333,
1356
+ "eval_type": "rule",
1357
+ "num_demo": 0,
1358
+ "num_query": 15
1359
+ },
1360
+ {
1361
+ "name": "contain_repeat_length",
1362
+ "score": 0.06666666666666667,
1363
+ "eval_type": "rule",
1364
+ "num_demo": 0,
1365
+ "num_query": 15
1366
+ },
1367
+ {
1368
+ "name": "counterfactual_arithmetic",
1369
+ "score": 0.9285714285714286,
1370
+ "eval_type": "rule",
1371
+ "num_demo": 1,
1372
+ "num_query": 14
1373
+ },
1374
+ {
1375
+ "name": "counting_multi_image",
1376
+ "score": 0.5,
1377
+ "eval_type": "rule",
1378
+ "num_demo": 1,
1379
+ "num_query": 14
1380
+ },
1381
+ {
1382
+ "name": "counting_single_image",
1383
+ "score": 0.7857142857142857,
1384
+ "eval_type": "rule",
1385
+ "num_demo": 1,
1386
+ "num_query": 14
1387
+ },
1388
+ {
1389
+ "name": "crossword_mini_5x5",
1390
+ "score": 0.25000000000000006,
1391
+ "eval_type": "rule",
1392
+ "num_demo": 1,
1393
+ "num_query": 14
1394
+ },
1395
+ {
1396
+ "name": "cultural_vqa",
1397
+ "score": 0.4,
1398
+ "eval_type": "rule",
1399
+ "num_demo": 1,
1400
+ "num_query": 15
1401
+ },
1402
+ {
1403
+ "name": "deciphering_oracle_bone",
1404
+ "score": 0.0,
1405
+ "eval_type": "rule",
1406
+ "num_demo": 1,
1407
+ "num_query": 14
1408
+ },
1409
+ {
1410
+ "name": "dish_ingredient_match",
1411
+ "score": 0.6428571428571429,
1412
+ "eval_type": "rule",
1413
+ "num_demo": 1,
1414
+ "num_query": 14
1415
+ },
1416
+ {
1417
+ "name": "distinguish_ai_generated_image",
1418
+ "score": 0.631578947368421,
1419
+ "eval_type": "rule",
1420
+ "num_demo": 1,
1421
+ "num_query": 19
1422
+ },
1423
+ {
1424
+ "name": "dvqa",
1425
+ "score": 0.8947368421052632,
1426
+ "eval_type": "rule",
1427
+ "num_demo": 1,
1428
+ "num_query": 19
1429
+ },
1430
+ {
1431
+ "name": "egocentric_analysis_single_image",
1432
+ "score": 0.5555555555555556,
1433
+ "eval_type": "rule",
1434
+ "num_demo": 1,
1435
+ "num_query": 9
1436
+ },
1437
+ {
1438
+ "name": "egocentric_spatial_reasoning",
1439
+ "score": 0.3333333333333333,
1440
+ "eval_type": "rule",
1441
+ "num_demo": 1,
1442
+ "num_query": 9
1443
+ },
1444
+ {
1445
+ "name": "electricity_future_prediction_from_table",
1446
+ "score": 0.6473684210526315,
1447
+ "eval_type": "rule",
1448
+ "num_demo": 1,
1449
+ "num_query": 19
1450
+ },
1451
+ {
1452
+ "name": "electricity_load_estimate_plot",
1453
+ "score": 0.6612142857142855,
1454
+ "eval_type": "rule",
1455
+ "num_demo": 1,
1456
+ "num_query": 14
1457
+ },
1458
+ {
1459
+ "name": "electricity_plot_future_prediction",
1460
+ "score": 0.9244157894736842,
1461
+ "eval_type": "rule",
1462
+ "num_demo": 1,
1463
+ "num_query": 19
1464
+ },
1465
+ {
1466
+ "name": "emotion_recognition",
1467
+ "score": 0.6428571428571429,
1468
+ "eval_type": "rule",
1469
+ "num_demo": 1,
1470
+ "num_query": 14
1471
+ },
1472
+ {
1473
+ "name": "entertainment_web_game_style",
1474
+ "score": 0.75,
1475
+ "eval_type": "rule",
1476
+ "num_demo": 1,
1477
+ "num_query": 14
1478
+ },
1479
+ {
1480
+ "name": "exchange_rate_estimate_plot",
1481
+ "score": 0.9785871428571429,
1482
+ "eval_type": "rule",
1483
+ "num_demo": 1,
1484
+ "num_query": 14
1485
+ },
1486
+ {
1487
+ "name": "extract_webpage_headline",
1488
+ "score": 0.5714285714285714,
1489
+ "eval_type": "rule",
1490
+ "num_demo": 1,
1491
+ "num_query": 14
1492
+ },
1493
+ {
1494
+ "name": "face_attribute_edit",
1495
+ "score": 0.5,
1496
+ "eval_type": "rule",
1497
+ "num_demo": 1,
1498
+ "num_query": 14
1499
+ },
1500
+ {
1501
+ "name": "face_identity_matching",
1502
+ "score": 0.4,
1503
+ "eval_type": "rule",
1504
+ "num_demo": 1,
1505
+ "num_query": 15
1506
+ },
1507
+ {
1508
+ "name": "face_keypoint_detection",
1509
+ "score": 0.8008804293485591,
1510
+ "eval_type": "rule",
1511
+ "num_demo": 1,
1512
+ "num_query": 14
1513
+ },
1514
+ {
1515
+ "name": "face_swap",
1516
+ "score": 0.6428571428571429,
1517
+ "eval_type": "rule",
1518
+ "num_demo": 1,
1519
+ "num_query": 14
1520
+ },
1521
+ {
1522
+ "name": "famous_building_recognition",
1523
+ "score": 0.875,
1524
+ "eval_type": "rule",
1525
+ "num_demo": 1,
1526
+ "num_query": 16
1527
+ },
1528
+ {
1529
+ "name": "figureqa",
1530
+ "score": 0.5714285714285714,
1531
+ "eval_type": "rule",
1532
+ "num_demo": 1,
1533
+ "num_query": 14
1534
+ },
1535
+ {
1536
+ "name": "finance_table_understanding",
1537
+ "score": 0.5714285714285714,
1538
+ "eval_type": "rule",
1539
+ "num_demo": 1,
1540
+ "num_query": 14
1541
+ },
1542
+ {
1543
+ "name": "flowchart_code_generation",
1544
+ "score": 0.4444444444444444,
1545
+ "eval_type": "rule",
1546
+ "num_demo": 1,
1547
+ "num_query": 9
1548
+ },
1549
+ {
1550
+ "name": "font_recognition",
1551
+ "score": 0.07142857142857142,
1552
+ "eval_type": "rule",
1553
+ "num_demo": 1,
1554
+ "num_query": 14
1555
+ },
1556
+ {
1557
+ "name": "forensic_detection_of_different_images",
1558
+ "score": 0.35714285714285715,
1559
+ "eval_type": "rule",
1560
+ "num_demo": 1,
1561
+ "num_query": 14
1562
+ },
1563
+ {
1564
+ "name": "functionality_matching_in_different_objects",
1565
+ "score": 0.5357142857142857,
1566
+ "eval_type": "rule",
1567
+ "num_demo": 1,
1568
+ "num_query": 14
1569
+ },
1570
+ {
1571
+ "name": "funsd_document_qa",
1572
+ "score": 0.8571428571428571,
1573
+ "eval_type": "rule",
1574
+ "num_demo": 1,
1575
+ "num_query": 14
1576
+ },
1577
+ {
1578
+ "name": "game_info_parsing",
1579
+ "score": 0.8896103896103894,
1580
+ "eval_type": "rule",
1581
+ "num_demo": 1,
1582
+ "num_query": 14
1583
+ },
1584
+ {
1585
+ "name": "game_info_retrieval",
1586
+ "score": 0.6428571428571429,
1587
+ "eval_type": "rule",
1588
+ "num_demo": 0,
1589
+ "num_query": 14
1590
+ },
1591
+ {
1592
+ "name": "game_platform_support_identification",
1593
+ "score": 0.4642857142857143,
1594
+ "eval_type": "rule",
1595
+ "num_demo": 1,
1596
+ "num_query": 14
1597
+ },
1598
+ {
1599
+ "name": "geographic_remote_sensing_land_cover",
1600
+ "score": 0.5714285714285714,
1601
+ "eval_type": "rule",
1602
+ "num_demo": 1,
1603
+ "num_query": 14
1604
+ },
1605
+ {
1606
+ "name": "geometry_analytic",
1607
+ "score": 0.35714285714285715,
1608
+ "eval_type": "rule",
1609
+ "num_demo": 1,
1610
+ "num_query": 14
1611
+ },
1612
+ {
1613
+ "name": "geometry_area",
1614
+ "score": 0.42857142857142855,
1615
+ "eval_type": "rule",
1616
+ "num_demo": 1,
1617
+ "num_query": 14
1618
+ },
1619
+ {
1620
+ "name": "geometry_descriptive",
1621
+ "score": 0.2857142857142857,
1622
+ "eval_type": "rule",
1623
+ "num_demo": 1,
1624
+ "num_query": 14
1625
+ },
1626
+ {
1627
+ "name": "geometry_length",
1628
+ "score": 0.5714285714285714,
1629
+ "eval_type": "rule",
1630
+ "num_demo": 1,
1631
+ "num_query": 14
1632
+ },
1633
+ {
1634
+ "name": "geometry_plot_position_relationship",
1635
+ "score": 0.7857142857142857,
1636
+ "eval_type": "rule",
1637
+ "num_demo": 1,
1638
+ "num_query": 14
1639
+ },
1640
+ {
1641
+ "name": "geometry_reasoning_circled_letter",
1642
+ "score": 0.75,
1643
+ "eval_type": "rule",
1644
+ "num_demo": 1,
1645
+ "num_query": 14
1646
+ },
1647
+ {
1648
+ "name": "geometry_reasoning_count_line_intersections",
1649
+ "score": 0.6071428571428571,
1650
+ "eval_type": "rule",
1651
+ "num_demo": 1,
1652
+ "num_query": 14
1653
+ },
1654
+ {
1655
+ "name": "geometry_reasoning_grid",
1656
+ "score": 0.9642857142857143,
1657
+ "eval_type": "rule",
1658
+ "num_demo": 1,
1659
+ "num_query": 14
1660
+ },
1661
+ {
1662
+ "name": "geometry_reasoning_nested_squares",
1663
+ "score": 0.5357142857142857,
1664
+ "eval_type": "rule",
1665
+ "num_demo": 1,
1666
+ "num_query": 14
1667
+ },
1668
+ {
1669
+ "name": "geometry_reasoning_overlapped_circle",
1670
+ "score": 0.39285714285714285,
1671
+ "eval_type": "rule",
1672
+ "num_demo": 1,
1673
+ "num_query": 14
1674
+ },
1675
+ {
1676
+ "name": "geometry_solid",
1677
+ "score": 0.21428571428571427,
1678
+ "eval_type": "rule",
1679
+ "num_demo": 1,
1680
+ "num_query": 14
1681
+ },
1682
+ {
1683
+ "name": "geometry_transformation",
1684
+ "score": 0.35714285714285715,
1685
+ "eval_type": "rule",
1686
+ "num_demo": 1,
1687
+ "num_query": 14
1688
+ },
1689
+ {
1690
+ "name": "go_capture_stone",
1691
+ "score": 0.13333333333333333,
1692
+ "eval_type": "rule",
1693
+ "num_demo": 1,
1694
+ "num_query": 15
1695
+ },
1696
+ {
1697
+ "name": "google_streetview_circle_reasoning",
1698
+ "score": 0.2,
1699
+ "eval_type": "rule",
1700
+ "num_demo": 1,
1701
+ "num_query": 15
1702
+ },
1703
+ {
1704
+ "name": "google_streetview_circle_sorting",
1705
+ "score": 0.0,
1706
+ "eval_type": "rule",
1707
+ "num_demo": 1,
1708
+ "num_query": 15
1709
+ },
1710
+ {
1711
+ "name": "google_streetview_direction_understanding",
1712
+ "score": 0.7857142857142857,
1713
+ "eval_type": "rule",
1714
+ "num_demo": 1,
1715
+ "num_query": 14
1716
+ },
1717
+ {
1718
+ "name": "google_streetview_line_reasoning",
1719
+ "score": 0.2,
1720
+ "eval_type": "rule",
1721
+ "num_demo": 1,
1722
+ "num_query": 15
1723
+ },
1724
+ {
1725
+ "name": "google_streetview_line_sorting",
1726
+ "score": 0.2,
1727
+ "eval_type": "rule",
1728
+ "num_demo": 1,
1729
+ "num_query": 15
1730
+ },
1731
+ {
1732
+ "name": "graph_chordless_cycle",
1733
+ "score": 0.35714285714285715,
1734
+ "eval_type": "rule",
1735
+ "num_demo": 1,
1736
+ "num_query": 14
1737
+ },
1738
+ {
1739
+ "name": "graph_connectivity",
1740
+ "score": 0.4166666666666667,
1741
+ "eval_type": "rule",
1742
+ "num_demo": 1,
1743
+ "num_query": 15
1744
+ },
1745
+ {
1746
+ "name": "graph_hamiltonian_cycle",
1747
+ "score": 0.49107142857142855,
1748
+ "eval_type": "rule",
1749
+ "num_demo": 1,
1750
+ "num_query": 14
1751
+ },
1752
+ {
1753
+ "name": "graph_hamiltonian_path",
1754
+ "score": 0.45833333333333337,
1755
+ "eval_type": "rule",
1756
+ "num_demo": 1,
1757
+ "num_query": 14
1758
+ },
1759
+ {
1760
+ "name": "graph_isomorphism",
1761
+ "score": 0.5333333333333333,
1762
+ "eval_type": "rule",
1763
+ "num_demo": 1,
1764
+ "num_query": 15
1765
+ },
1766
+ {
1767
+ "name": "graph_maxflow",
1768
+ "score": 0.3333333333333333,
1769
+ "eval_type": "rule",
1770
+ "num_demo": 1,
1771
+ "num_query": 15
1772
+ },
1773
+ {
1774
+ "name": "graph_shortest_path_kamada_kawai",
1775
+ "score": 0.42857142857142855,
1776
+ "eval_type": "rule",
1777
+ "num_demo": 1,
1778
+ "num_query": 14
1779
+ },
1780
+ {
1781
+ "name": "graph_shortest_path_planar",
1782
+ "score": 0.5714285714285714,
1783
+ "eval_type": "rule",
1784
+ "num_demo": 1,
1785
+ "num_query": 14
1786
+ },
1787
+ {
1788
+ "name": "graph_theory",
1789
+ "score": 0.5714285714285714,
1790
+ "eval_type": "rule",
1791
+ "num_demo": 1,
1792
+ "num_query": 14
1793
+ },
1794
+ {
1795
+ "name": "handwritten_math_expression_extraction",
1796
+ "score": 0.7142857142857143,
1797
+ "eval_type": "rule",
1798
+ "num_demo": 1,
1799
+ "num_query": 14
1800
+ },
1801
+ {
1802
+ "name": "hashtag_recommendation",
1803
+ "score": 0.9107142857142857,
1804
+ "eval_type": "rule",
1805
+ "num_demo": 1,
1806
+ "num_query": 14
1807
+ },
1808
+ {
1809
+ "name": "healthcare_info_judgement",
1810
+ "score": 0.7857142857142857,
1811
+ "eval_type": "rule",
1812
+ "num_demo": 1,
1813
+ "num_query": 14
1814
+ },
1815
+ {
1816
+ "name": "highest_discount_game_price_identification",
1817
+ "score": 1.0,
1818
+ "eval_type": "rule",
1819
+ "num_demo": 1,
1820
+ "num_query": 14
1821
+ },
1822
+ {
1823
+ "name": "hotel_booking_confirmation_parsing",
1824
+ "score": 0.7077922077922079,
1825
+ "eval_type": "rule",
1826
+ "num_demo": 1,
1827
+ "num_query": 14
1828
+ },
1829
+ {
1830
+ "name": "human_relationship_reasoning",
1831
+ "score": 0.9285714285714286,
1832
+ "eval_type": "rule",
1833
+ "num_demo": 1,
1834
+ "num_query": 14
1835
+ },
1836
+ {
1837
+ "name": "humor_understand_caption_match",
1838
+ "score": 0.6,
1839
+ "eval_type": "rule",
1840
+ "num_demo": 1,
1841
+ "num_query": 15
1842
+ },
1843
+ {
1844
+ "name": "icon_arithmetic_puzzle",
1845
+ "score": 0.9642857142857143,
1846
+ "eval_type": "rule",
1847
+ "num_demo": 1,
1848
+ "num_query": 14
1849
+ },
1850
+ {
1851
+ "name": "iconqa_count_and_reasoning",
1852
+ "score": 0.6842105263157895,
1853
+ "eval_type": "rule",
1854
+ "num_demo": 1,
1855
+ "num_query": 19
1856
+ },
1857
+ {
1858
+ "name": "ili_ratio_future_prediction",
1859
+ "score": 0.24864285714285697,
1860
+ "eval_type": "rule",
1861
+ "num_demo": 1,
1862
+ "num_query": 14
1863
+ },
1864
+ {
1865
+ "name": "image_style_recognition",
1866
+ "score": 1.0,
1867
+ "eval_type": "rule",
1868
+ "num_demo": 1,
1869
+ "num_query": 14
1870
+ },
1871
+ {
1872
+ "name": "image_translation_en2cn",
1873
+ "score": 0.4404063008108421,
1874
+ "eval_type": "rule",
1875
+ "num_demo": 1,
1876
+ "num_query": 9
1877
+ },
1878
+ {
1879
+ "name": "insect_order_classification",
1880
+ "score": 0.26666666666666666,
1881
+ "eval_type": "rule",
1882
+ "num_demo": 1,
1883
+ "num_query": 15
1884
+ },
1885
+ {
1886
+ "name": "interpret_force_perspective_illusion",
1887
+ "score": 0.6666666666666666,
1888
+ "eval_type": "rule",
1889
+ "num_demo": 1,
1890
+ "num_query": 15
1891
+ },
1892
+ {
1893
+ "name": "ishihara_test",
1894
+ "score": 0.2857142857142857,
1895
+ "eval_type": "rule",
1896
+ "num_demo": 1,
1897
+ "num_query": 14
1898
+ },
1899
+ {
1900
+ "name": "knowledge_graph_understanding",
1901
+ "score": 0.4,
1902
+ "eval_type": "rule",
1903
+ "num_demo": 0,
1904
+ "num_query": 15
1905
+ },
1906
+ {
1907
+ "name": "knowledge_sign_recognition",
1908
+ "score": 0.4444444444444444,
1909
+ "eval_type": "rule",
1910
+ "num_demo": 1,
1911
+ "num_query": 9
1912
+ },
1913
+ {
1914
+ "name": "kvqa_knowledge_aware_qa",
1915
+ "score": 0.42105263157894735,
1916
+ "eval_type": "rule",
1917
+ "num_demo": 1,
1918
+ "num_query": 19
1919
+ },
1920
+ {
1921
+ "name": "landmark_check_two_images",
1922
+ "score": 0.8222222222222223,
1923
+ "eval_type": "rule",
1924
+ "num_demo": 1,
1925
+ "num_query": 15
1926
+ },
1927
+ {
1928
+ "name": "landmark_recognition_and_qa",
1929
+ "score": 0.5333333333333333,
1930
+ "eval_type": "rule",
1931
+ "num_demo": 1,
1932
+ "num_query": 15
1933
+ },
1934
+ {
1935
+ "name": "latex_complex_formula_convertion",
1936
+ "score": 0.47058823529411764,
1937
+ "eval_type": "rule",
1938
+ "num_demo": 1,
1939
+ "num_query": 17
1940
+ },
1941
+ {
1942
+ "name": "license_plate_recognition",
1943
+ "score": 0.7142857142857143,
1944
+ "eval_type": "rule",
1945
+ "num_demo": 1,
1946
+ "num_query": 14
1947
+ },
1948
+ {
1949
+ "name": "llavaguard",
1950
+ "score": 0.6071428571428571,
1951
+ "eval_type": "rule",
1952
+ "num_demo": 1,
1953
+ "num_query": 14
1954
+ },
1955
+ {
1956
+ "name": "location_vqa",
1957
+ "score": 0.6428571428571429,
1958
+ "eval_type": "rule",
1959
+ "num_demo": 1,
1960
+ "num_query": 14
1961
+ },
1962
+ {
1963
+ "name": "logical_reasoning_2D_views_of_3D_shapes",
1964
+ "score": 0.14285714285714285,
1965
+ "eval_type": "rule",
1966
+ "num_demo": 1,
1967
+ "num_query": 14
1968
+ },
1969
+ {
1970
+ "name": "logical_reasoning_2d_folding",
1971
+ "score": 0.2857142857142857,
1972
+ "eval_type": "rule",
1973
+ "num_demo": 1,
1974
+ "num_query": 14
1975
+ },
1976
+ {
1977
+ "name": "logical_reasoning_find_odd_one_out",
1978
+ "score": 0.0,
1979
+ "eval_type": "rule",
1980
+ "num_demo": 1,
1981
+ "num_query": 14
1982
+ },
1983
+ {
1984
+ "name": "logical_reasoning_fit_pattern",
1985
+ "score": 0.5,
1986
+ "eval_type": "rule",
1987
+ "num_demo": 1,
1988
+ "num_query": 14
1989
+ },
1990
+ {
1991
+ "name": "logo2k_same_type_logo_retrieval",
1992
+ "score": 0.8928571428571429,
1993
+ "eval_type": "rule",
1994
+ "num_demo": 1,
1995
+ "num_query": 14
1996
+ },
1997
+ {
1998
+ "name": "long_string_letter_recognition",
1999
+ "score": 0.0,
2000
+ "eval_type": "rule",
2001
+ "num_demo": 1,
2002
+ "num_query": 14
2003
+ },
2004
+ {
2005
+ "name": "long_string_number_recognition",
2006
+ "score": 0.35714285714285715,
2007
+ "eval_type": "rule",
2008
+ "num_demo": 1,
2009
+ "num_query": 14
2010
+ },
2011
+ {
2012
+ "name": "mahjong",
2013
+ "score": 0.0,
2014
+ "eval_type": "rule",
2015
+ "num_demo": 1,
2016
+ "num_query": 14
2017
+ },
2018
+ {
2019
+ "name": "map_diagram_qa",
2020
+ "score": 0.7142857142857143,
2021
+ "eval_type": "rule",
2022
+ "num_demo": 1,
2023
+ "num_query": 14
2024
+ },
2025
+ {
2026
+ "name": "math_breakpoint",
2027
+ "score": 0.6666666666666666,
2028
+ "eval_type": "rule",
2029
+ "num_demo": 1,
2030
+ "num_query": 15
2031
+ },
2032
+ {
2033
+ "name": "math_convexity_value_estimation",
2034
+ "score": 0.47324110707388967,
2035
+ "eval_type": "rule",
2036
+ "num_demo": 1,
2037
+ "num_query": 15
2038
+ },
2039
+ {
2040
+ "name": "math_exams_v",
2041
+ "score": 0.6428571428571429,
2042
+ "eval_type": "rule",
2043
+ "num_demo": 1,
2044
+ "num_query": 14
2045
+ },
2046
+ {
2047
+ "name": "math_parity",
2048
+ "score": 0.7333333333333333,
2049
+ "eval_type": "rule",
2050
+ "num_demo": 1,
2051
+ "num_query": 15
2052
+ },
2053
+ {
2054
+ "name": "maze_2d_8x8",
2055
+ "score": 0.0,
2056
+ "eval_type": "rule",
2057
+ "num_demo": 1,
2058
+ "num_query": 14
2059
+ },
2060
+ {
2061
+ "name": "media_homepage_profile",
2062
+ "score": 0.2585034013605442,
2063
+ "eval_type": "rule",
2064
+ "num_demo": 0,
2065
+ "num_query": 14
2066
+ },
2067
+ {
2068
+ "name": "media_recommend_solutions_stackoverflow",
2069
+ "score": 0.9285714285714286,
2070
+ "eval_type": "rule",
2071
+ "num_demo": 1,
2072
+ "num_query": 14
2073
+ },
2074
+ {
2075
+ "name": "medical_abdomen_MRI_organ_recognition",
2076
+ "score": 0.2619047619047619,
2077
+ "eval_type": "rule",
2078
+ "num_demo": 1,
2079
+ "num_query": 14
2080
+ },
2081
+ {
2082
+ "name": "medical_abdomen_endscopy_organ_recognition",
2083
+ "score": 0.11904761904761904,
2084
+ "eval_type": "rule",
2085
+ "num_demo": 1,
2086
+ "num_query": 14
2087
+ },
2088
+ {
2089
+ "name": "medical_blood_vessels_recognition",
2090
+ "score": 0.5,
2091
+ "eval_type": "rule",
2092
+ "num_demo": 1,
2093
+ "num_query": 14
2094
+ },
2095
+ {
2096
+ "name": "medical_cell_recognition",
2097
+ "score": 0.42857142857142855,
2098
+ "eval_type": "rule",
2099
+ "num_demo": 1,
2100
+ "num_query": 14
2101
+ },
2102
+ {
2103
+ "name": "medical_content_based_retrieval_radiology",
2104
+ "score": 0.2857142857142857,
2105
+ "eval_type": "rule",
2106
+ "num_demo": 1,
2107
+ "num_query": 14
2108
+ },
2109
+ {
2110
+ "name": "medical_counting_lymphocytes",
2111
+ "score": 0.0,
2112
+ "eval_type": "rule",
2113
+ "num_demo": 1,
2114
+ "num_query": 14
2115
+ },
2116
+ {
2117
+ "name": "medical_image_artifacts_indentification",
2118
+ "score": 0.21428571428571427,
2119
+ "eval_type": "rule",
2120
+ "num_demo": 1,
2121
+ "num_query": 14
2122
+ },
2123
+ {
2124
+ "name": "medical_keywords_based_retrieval_non_radiology",
2125
+ "score": 0.2857142857142857,
2126
+ "eval_type": "rule",
2127
+ "num_demo": 1,
2128
+ "num_query": 14
2129
+ },
2130
+ {
2131
+ "name": "medical_multi_organ_segmentation_rater",
2132
+ "score": 0.35714285714285715,
2133
+ "eval_type": "rule",
2134
+ "num_demo": 1,
2135
+ "num_query": 14
2136
+ },
2137
+ {
2138
+ "name": "medical_parasite_detection",
2139
+ "score": 0.2857142857142857,
2140
+ "eval_type": "rule",
2141
+ "num_demo": 1,
2142
+ "num_query": 14
2143
+ },
2144
+ {
2145
+ "name": "medical_polyp_segmentation_single_object_rater",
2146
+ "score": 0.5,
2147
+ "eval_type": "rule",
2148
+ "num_demo": 1,
2149
+ "num_query": 14
2150
+ },
2151
+ {
2152
+ "name": "medical_retrieval_given_surgeon_activity",
2153
+ "score": 0.35714285714285715,
2154
+ "eval_type": "rule",
2155
+ "num_demo": 1,
2156
+ "num_query": 14
2157
+ },
2158
+ {
2159
+ "name": "memorization_chinese_celebrity",
2160
+ "score": 0.2857142857142857,
2161
+ "eval_type": "rule",
2162
+ "num_demo": 1,
2163
+ "num_query": 14
2164
+ },
2165
+ {
2166
+ "name": "memorization_famous_treaty",
2167
+ "score": 0.32142857142857145,
2168
+ "eval_type": "rule",
2169
+ "num_demo": 1,
2170
+ "num_query": 14
2171
+ },
2172
+ {
2173
+ "name": "memorization_indian_celebrity",
2174
+ "score": 0.35714285714285715,
2175
+ "eval_type": "rule",
2176
+ "num_demo": 1,
2177
+ "num_query": 14
2178
+ },
2179
+ {
2180
+ "name": "memorization_papers",
2181
+ "score": 0.4,
2182
+ "eval_type": "rule",
2183
+ "num_demo": 1,
2184
+ "num_query": 15
2185
+ },
2186
+ {
2187
+ "name": "mensa_iq_test",
2188
+ "score": 0.2693850267379679,
2189
+ "eval_type": "rule",
2190
+ "num_demo": 1,
2191
+ "num_query": 17
2192
+ },
2193
+ {
2194
+ "name": "mindmap_elements_parsing",
2195
+ "score": 0.2857142857142857,
2196
+ "eval_type": "rule",
2197
+ "num_demo": 1,
2198
+ "num_query": 14
2199
+ },
2200
+ {
2201
+ "name": "mnist_pattern",
2202
+ "score": 0.5,
2203
+ "eval_type": "rule",
2204
+ "num_demo": 1,
2205
+ "num_query": 14
2206
+ },
2207
+ {
2208
+ "name": "monthly_weather_days_count",
2209
+ "score": 0.40476190476190477,
2210
+ "eval_type": "rule",
2211
+ "num_demo": 1,
2212
+ "num_query": 14
2213
+ },
2214
+ {
2215
+ "name": "move_pos_to_pos_hanoi_4_pole",
2216
+ "score": 0.0,
2217
+ "eval_type": "rule",
2218
+ "num_demo": 1,
2219
+ "num_query": 14
2220
+ },
2221
+ {
2222
+ "name": "movie_info_parsing",
2223
+ "score": 0.6607142857142857,
2224
+ "eval_type": "rule",
2225
+ "num_demo": 1,
2226
+ "num_query": 14
2227
+ },
2228
+ {
2229
+ "name": "movie_info_retrieval",
2230
+ "score": 0.7857142857142857,
2231
+ "eval_type": "rule",
2232
+ "num_demo": 0,
2233
+ "num_query": 14
2234
+ },
2235
+ {
2236
+ "name": "movie_retrieval_by_actor",
2237
+ "score": 0.7142857142857143,
2238
+ "eval_type": "rule",
2239
+ "num_demo": 0,
2240
+ "num_query": 14
2241
+ },
2242
+ {
2243
+ "name": "multi_contain_position_only",
2244
+ "score": 0.06666666666666667,
2245
+ "eval_type": "rule",
2246
+ "num_demo": 0,
2247
+ "num_query": 15
2248
+ },
2249
+ {
2250
+ "name": "multi_contain_repeat",
2251
+ "score": 0.06666666666666667,
2252
+ "eval_type": "rule",
2253
+ "num_demo": 0,
2254
+ "num_query": 15
2255
+ },
2256
+ {
2257
+ "name": "multi_contain_repeat_position_only_length",
2258
+ "score": 0.06666666666666667,
2259
+ "eval_type": "rule",
2260
+ "num_demo": 0,
2261
+ "num_query": 15
2262
+ },
2263
+ {
2264
+ "name": "multi_load_type_prediction_from_plot",
2265
+ "score": 0.5119047619047618,
2266
+ "eval_type": "rule",
2267
+ "num_demo": 1,
2268
+ "num_query": 14
2269
+ },
2270
+ {
2271
+ "name": "multilingual_game_info_parsing",
2272
+ "score": 0.6607142857142857,
2273
+ "eval_type": "rule",
2274
+ "num_demo": 1,
2275
+ "num_query": 14
2276
+ },
2277
+ {
2278
+ "name": "multilingual_movie_info_parsing",
2279
+ "score": 0.6224489795918366,
2280
+ "eval_type": "rule",
2281
+ "num_demo": 1,
2282
+ "num_query": 14
2283
+ },
2284
+ {
2285
+ "name": "multilingual_news_qa",
2286
+ "score": 0.9285714285714286,
2287
+ "eval_type": "rule",
2288
+ "num_demo": 0,
2289
+ "num_query": 14
2290
+ },
2291
+ {
2292
+ "name": "multiple_states_identify_africa",
2293
+ "score": 0.41428571428571426,
2294
+ "eval_type": "rule",
2295
+ "num_demo": 1,
2296
+ "num_query": 14
2297
+ },
2298
+ {
2299
+ "name": "multiple_states_identify_americas",
2300
+ "score": 0.4,
2301
+ "eval_type": "rule",
2302
+ "num_demo": 1,
2303
+ "num_query": 14
2304
+ },
2305
+ {
2306
+ "name": "multiple_states_identify_asia",
2307
+ "score": 0.5142857142857142,
2308
+ "eval_type": "rule",
2309
+ "num_demo": 1,
2310
+ "num_query": 14
2311
+ },
2312
+ {
2313
+ "name": "multiple_states_identify_europe",
2314
+ "score": 0.5428571428571429,
2315
+ "eval_type": "rule",
2316
+ "num_demo": 1,
2317
+ "num_query": 14
2318
+ },
2319
+ {
2320
+ "name": "multiview_reasoning_camera_moving",
2321
+ "score": 0.7857142857142857,
2322
+ "eval_type": "rule",
2323
+ "num_demo": 1,
2324
+ "num_query": 14
2325
+ },
2326
+ {
2327
+ "name": "muma_theory_of_mind_belief_of_goal",
2328
+ "score": 0.0,
2329
+ "eval_type": "rule",
2330
+ "num_demo": 1,
2331
+ "num_query": 15
2332
+ },
2333
+ {
2334
+ "name": "muma_theory_of_mind_social_goal",
2335
+ "score": 0.0,
2336
+ "eval_type": "rule",
2337
+ "num_demo": 1,
2338
+ "num_query": 15
2339
+ },
2340
+ {
2341
+ "name": "music_info_parsing",
2342
+ "score": 0.7142857142857143,
2343
+ "eval_type": "rule",
2344
+ "num_demo": 1,
2345
+ "num_query": 14
2346
+ },
2347
+ {
2348
+ "name": "music_info_retrieval",
2349
+ "score": 0.5,
2350
+ "eval_type": "rule",
2351
+ "num_demo": 0,
2352
+ "num_query": 14
2353
+ },
2354
+ {
2355
+ "name": "music_sheet_author",
2356
+ "score": 0.1875,
2357
+ "eval_type": "rule",
2358
+ "num_demo": 1,
2359
+ "num_query": 16
2360
+ },
2361
+ {
2362
+ "name": "music_sheet_format_QA",
2363
+ "score": 0.35714285714285715,
2364
+ "eval_type": "rule",
2365
+ "num_demo": 1,
2366
+ "num_query": 14
2367
+ },
2368
+ {
2369
+ "name": "music_sheet_name",
2370
+ "score": 0.26666666666666666,
2371
+ "eval_type": "rule",
2372
+ "num_demo": 1,
2373
+ "num_query": 15
2374
+ },
2375
+ {
2376
+ "name": "music_sheet_note_count",
2377
+ "score": 0.058823529411764705,
2378
+ "eval_type": "rule",
2379
+ "num_demo": 1,
2380
+ "num_query": 17
2381
+ },
2382
+ {
2383
+ "name": "music_sheet_sentiment",
2384
+ "score": 0.2857142857142857,
2385
+ "eval_type": "rule",
2386
+ "num_demo": 1,
2387
+ "num_query": 14
2388
+ },
2389
+ {
2390
+ "name": "mvsa_sentiment_classification",
2391
+ "score": 0.6428571428571429,
2392
+ "eval_type": "rule",
2393
+ "num_demo": 1,
2394
+ "num_query": 14
2395
+ },
2396
+ {
2397
+ "name": "newspaper_ocr_in_query_box",
2398
+ "score": 0.4666666666666667,
2399
+ "eval_type": "rule",
2400
+ "num_demo": 1,
2401
+ "num_query": 15
2402
+ },
2403
+ {
2404
+ "name": "newspaper_page_parse_and_count",
2405
+ "score": 0.5333333333333333,
2406
+ "eval_type": "rule",
2407
+ "num_demo": 1,
2408
+ "num_query": 15
2409
+ },
2410
+ {
2411
+ "name": "next_action_prediction",
2412
+ "score": 0.5,
2413
+ "eval_type": "rule",
2414
+ "num_demo": 1,
2415
+ "num_query": 14
2416
+ },
2417
+ {
2418
+ "name": "nextqa_mc",
2419
+ "score": 0.9473684210526315,
2420
+ "eval_type": "rule",
2421
+ "num_demo": 1,
2422
+ "num_query": 19
2423
+ },
2424
+ {
2425
+ "name": "nlvr2_two_image_compare_qa",
2426
+ "score": 0.7857142857142857,
2427
+ "eval_type": "rule",
2428
+ "num_demo": 1,
2429
+ "num_query": 14
2430
+ },
2431
+ {
2432
+ "name": "number_comparison",
2433
+ "score": 1.0,
2434
+ "eval_type": "rule",
2435
+ "num_demo": 0,
2436
+ "num_query": 14
2437
+ },
2438
+ {
2439
+ "name": "number_puzzle_kakuro_5x5",
2440
+ "score": 0.0,
2441
+ "eval_type": "rule",
2442
+ "num_demo": 1,
2443
+ "num_query": 15
2444
+ },
2445
+ {
2446
+ "name": "number_puzzle_sudoku",
2447
+ "score": 0.0,
2448
+ "eval_type": "rule",
2449
+ "num_demo": 1,
2450
+ "num_query": 15
2451
+ },
2452
+ {
2453
+ "name": "ocr_article_authors",
2454
+ "score": 0.5,
2455
+ "eval_type": "rule",
2456
+ "num_demo": 1,
2457
+ "num_query": 14
2458
+ },
2459
+ {
2460
+ "name": "ocr_article_journal",
2461
+ "score": 0.42857142857142855,
2462
+ "eval_type": "rule",
2463
+ "num_demo": 1,
2464
+ "num_query": 14
2465
+ },
2466
+ {
2467
+ "name": "ocr_math_equation",
2468
+ "score": 0.5714285714285714,
2469
+ "eval_type": "rule",
2470
+ "num_demo": 1,
2471
+ "num_query": 14
2472
+ },
2473
+ {
2474
+ "name": "ocr_math_text_latex",
2475
+ "score": 0.21428571428571427,
2476
+ "eval_type": "rule",
2477
+ "num_demo": 1,
2478
+ "num_query": 14
2479
+ },
2480
+ {
2481
+ "name": "ocr_resume_employer_plain",
2482
+ "score": 0.6428571428571429,
2483
+ "eval_type": "rule",
2484
+ "num_demo": 1,
2485
+ "num_query": 14
2486
+ },
2487
+ {
2488
+ "name": "ocr_resume_experience_plain",
2489
+ "score": 0.6428571428571429,
2490
+ "eval_type": "rule",
2491
+ "num_demo": 1,
2492
+ "num_query": 14
2493
+ },
2494
+ {
2495
+ "name": "ocr_resume_school_plain",
2496
+ "score": 0.5,
2497
+ "eval_type": "rule",
2498
+ "num_demo": 1,
2499
+ "num_query": 14
2500
+ },
2501
+ {
2502
+ "name": "ocr_resume_skill_plain",
2503
+ "score": 0.35714285714285715,
2504
+ "eval_type": "rule",
2505
+ "num_demo": 1,
2506
+ "num_query": 14
2507
+ },
2508
+ {
2509
+ "name": "ocr_table_to_csv",
2510
+ "score": 0.5714285714285714,
2511
+ "eval_type": "rule",
2512
+ "num_demo": 1,
2513
+ "num_query": 14
2514
+ },
2515
+ {
2516
+ "name": "ocr_table_to_html",
2517
+ "score": 0.6428571428571429,
2518
+ "eval_type": "rule",
2519
+ "num_demo": 1,
2520
+ "num_query": 14
2521
+ },
2522
+ {
2523
+ "name": "ocr_table_to_latex",
2524
+ "score": 0.6428571428571429,
2525
+ "eval_type": "rule",
2526
+ "num_demo": 1,
2527
+ "num_query": 14
2528
+ },
2529
+ {
2530
+ "name": "ocr_table_to_markdown",
2531
+ "score": 0.9285714285714286,
2532
+ "eval_type": "rule",
2533
+ "num_demo": 1,
2534
+ "num_query": 14
2535
+ },
2536
+ {
2537
+ "name": "orchestra_score_recognition",
2538
+ "score": 0.07142857142857142,
2539
+ "eval_type": "rule",
2540
+ "num_demo": 1,
2541
+ "num_query": 14
2542
+ },
2543
+ {
2544
+ "name": "out_of_context",
2545
+ "score": 0.7857142857142857,
2546
+ "eval_type": "rule",
2547
+ "num_demo": 1,
2548
+ "num_query": 14
2549
+ },
2550
+ {
2551
+ "name": "painting_QA",
2552
+ "score": 0.7142857142857143,
2553
+ "eval_type": "rule",
2554
+ "num_demo": 1,
2555
+ "num_query": 14
2556
+ },
2557
+ {
2558
+ "name": "panel_images_multi_question",
2559
+ "score": 0.7142857142857142,
2560
+ "eval_type": "rule",
2561
+ "num_demo": 1,
2562
+ "num_query": 14
2563
+ },
2564
+ {
2565
+ "name": "panel_images_single_question",
2566
+ "score": 0.9285714285714286,
2567
+ "eval_type": "rule",
2568
+ "num_demo": 1,
2569
+ "num_query": 14
2570
+ },
2571
+ {
2572
+ "name": "paper_review_acceptance",
2573
+ "score": 0.4666666666666667,
2574
+ "eval_type": "rule",
2575
+ "num_demo": 1,
2576
+ "num_query": 15
2577
+ },
2578
+ {
2579
+ "name": "paper_review_rating",
2580
+ "score": 0.6634733643689834,
2581
+ "eval_type": "rule",
2582
+ "num_demo": 1,
2583
+ "num_query": 15
2584
+ },
2585
+ {
2586
+ "name": "paper_vqa",
2587
+ "score": 0.2857142857142857,
2588
+ "eval_type": "rule",
2589
+ "num_demo": 1,
2590
+ "num_query": 14
2591
+ },
2592
+ {
2593
+ "name": "perception_test_object_shuffle_video",
2594
+ "score": 0.1875,
2595
+ "eval_type": "rule",
2596
+ "num_demo": 1,
2597
+ "num_query": 16
2598
+ },
2599
+ {
2600
+ "name": "perception_test_video_action_count",
2601
+ "score": 0.375,
2602
+ "eval_type": "rule",
2603
+ "num_demo": 1,
2604
+ "num_query": 16
2605
+ },
2606
+ {
2607
+ "name": "perception_test_video_character_order",
2608
+ "score": 0.75,
2609
+ "eval_type": "rule",
2610
+ "num_demo": 1,
2611
+ "num_query": 16
2612
+ },
2613
+ {
2614
+ "name": "photo_sharing_image_retrieval",
2615
+ "score": 0.7857142857142857,
2616
+ "eval_type": "rule",
2617
+ "num_demo": 0,
2618
+ "num_query": 14
2619
+ },
2620
+ {
2621
+ "name": "photoshop_operation",
2622
+ "score": 0.2928571428571428,
2623
+ "eval_type": "rule",
2624
+ "num_demo": 1,
2625
+ "num_query": 14
2626
+ },
2627
+ {
2628
+ "name": "physical_property_reasoning",
2629
+ "score": 0.9285714285714286,
2630
+ "eval_type": "rule",
2631
+ "num_demo": 1,
2632
+ "num_query": 14
2633
+ },
2634
+ {
2635
+ "name": "physics_exams_v",
2636
+ "score": 0.35714285714285715,
2637
+ "eval_type": "rule",
2638
+ "num_demo": 1,
2639
+ "num_query": 14
2640
+ },
2641
+ {
2642
+ "name": "pictionary_cartoon_drawing_guess",
2643
+ "score": 0.5714285714285714,
2644
+ "eval_type": "rule",
2645
+ "num_demo": 1,
2646
+ "num_query": 14
2647
+ },
2648
+ {
2649
+ "name": "pictionary_chinese_food_img2en",
2650
+ "score": 0.2857142857142857,
2651
+ "eval_type": "rule",
2652
+ "num_demo": 1,
2653
+ "num_query": 14
2654
+ },
2655
+ {
2656
+ "name": "pictionary_doodle_guess",
2657
+ "score": 0.0,
2658
+ "eval_type": "rule",
2659
+ "num_demo": 1,
2660
+ "num_query": 15
2661
+ },
2662
+ {
2663
+ "name": "pictionary_genai_output_chinese",
2664
+ "score": 0.14285714285714285,
2665
+ "eval_type": "rule",
2666
+ "num_demo": 1,
2667
+ "num_query": 14
2668
+ },
2669
+ {
2670
+ "name": "pictionary_skribbl_io",
2671
+ "score": 0.2,
2672
+ "eval_type": "rule",
2673
+ "num_demo": 1,
2674
+ "num_query": 20
2675
+ },
2676
+ {
2677
+ "name": "places365_scene_type_classification",
2678
+ "score": 0.7142857142857143,
2679
+ "eval_type": "rule",
2680
+ "num_demo": 1,
2681
+ "num_query": 14
2682
+ },
2683
+ {
2684
+ "name": "places365_similar_scene_retrieval",
2685
+ "score": 0.5714285714285714,
2686
+ "eval_type": "rule",
2687
+ "num_demo": 1,
2688
+ "num_query": 14
2689
+ },
2690
+ {
2691
+ "name": "planning_screenshot_barman",
2692
+ "score": 0.0,
2693
+ "eval_type": "rule",
2694
+ "num_demo": 1,
2695
+ "num_query": 15
2696
+ },
2697
+ {
2698
+ "name": "planning_screenshot_blocksworld",
2699
+ "score": 0.0,
2700
+ "eval_type": "rule",
2701
+ "num_demo": 1,
2702
+ "num_query": 15
2703
+ },
2704
+ {
2705
+ "name": "planning_screenshot_floortile",
2706
+ "score": 0.0,
2707
+ "eval_type": "rule",
2708
+ "num_demo": 1,
2709
+ "num_query": 15
2710
+ },
2711
+ {
2712
+ "name": "planning_screenshot_grippers",
2713
+ "score": 0.0,
2714
+ "eval_type": "rule",
2715
+ "num_demo": 1,
2716
+ "num_query": 15
2717
+ },
2718
+ {
2719
+ "name": "planning_screenshot_storage",
2720
+ "score": 0.0,
2721
+ "eval_type": "rule",
2722
+ "num_demo": 1,
2723
+ "num_query": 15
2724
+ },
2725
+ {
2726
+ "name": "planning_screenshot_termes",
2727
+ "score": 0.0,
2728
+ "eval_type": "rule",
2729
+ "num_demo": 1,
2730
+ "num_query": 15
2731
+ },
2732
+ {
2733
+ "name": "planning_screenshot_tyreworld",
2734
+ "score": 0.0,
2735
+ "eval_type": "rule",
2736
+ "num_demo": 1,
2737
+ "num_query": 15
2738
+ },
2739
+ {
2740
+ "name": "planning_visual_barman",
2741
+ "score": 0.0,
2742
+ "eval_type": "rule",
2743
+ "num_demo": 1,
2744
+ "num_query": 15
2745
+ },
2746
+ {
2747
+ "name": "planning_visual_blocksworld",
2748
+ "score": 0.0,
2749
+ "eval_type": "rule",
2750
+ "num_demo": 1,
2751
+ "num_query": 15
2752
+ },
2753
+ {
2754
+ "name": "planning_visual_floortile",
2755
+ "score": 0.0,
2756
+ "eval_type": "rule",
2757
+ "num_demo": 1,
2758
+ "num_query": 15
2759
+ },
2760
+ {
2761
+ "name": "planning_visual_grippers",
2762
+ "score": 0.0,
2763
+ "eval_type": "rule",
2764
+ "num_demo": 1,
2765
+ "num_query": 15
2766
+ },
2767
+ {
2768
+ "name": "planning_visual_storage",
2769
+ "score": 0.0,
2770
+ "eval_type": "rule",
2771
+ "num_demo": 1,
2772
+ "num_query": 15
2773
+ },
2774
+ {
2775
+ "name": "planning_visual_termes",
2776
+ "score": 0.0,
2777
+ "eval_type": "rule",
2778
+ "num_demo": 1,
2779
+ "num_query": 15
2780
+ },
2781
+ {
2782
+ "name": "pmc_vqa_medical_image_qa",
2783
+ "score": 0.6842105263157895,
2784
+ "eval_type": "rule",
2785
+ "num_demo": 1,
2786
+ "num_query": 19
2787
+ },
2788
+ {
2789
+ "name": "poetry_acrostic",
2790
+ "score": 0.8666666666666667,
2791
+ "eval_type": "rule",
2792
+ "num_demo": 0,
2793
+ "num_query": 15
2794
+ },
2795
+ {
2796
+ "name": "poetry_acrostic_alliteration",
2797
+ "score": 0.06666666666666667,
2798
+ "eval_type": "rule",
2799
+ "num_demo": 0,
2800
+ "num_query": 15
2801
+ },
2802
+ {
2803
+ "name": "poetry_custom_rhyming_scheme",
2804
+ "score": 0.0,
2805
+ "eval_type": "rule",
2806
+ "num_demo": 0,
2807
+ "num_query": 15
2808
+ },
2809
+ {
2810
+ "name": "poetry_haiku",
2811
+ "score": 0.0,
2812
+ "eval_type": "rule",
2813
+ "num_demo": 0,
2814
+ "num_query": 15
2815
+ },
2816
+ {
2817
+ "name": "poetry_limerick",
2818
+ "score": 0.06666666666666667,
2819
+ "eval_type": "rule",
2820
+ "num_demo": 0,
2821
+ "num_query": 15
2822
+ },
2823
+ {
2824
+ "name": "poetry_petrarchian_sonnet_optional_meter",
2825
+ "score": 0.0,
2826
+ "eval_type": "rule",
2827
+ "num_demo": 0,
2828
+ "num_query": 15
2829
+ },
2830
+ {
2831
+ "name": "poetry_shakespearean_sonnet",
2832
+ "score": 0.0,
2833
+ "eval_type": "rule",
2834
+ "num_demo": 0,
2835
+ "num_query": 15
2836
+ },
2837
+ {
2838
+ "name": "pokemon_3D_recognition",
2839
+ "score": 0.7,
2840
+ "eval_type": "rule",
2841
+ "num_demo": 1,
2842
+ "num_query": 15
2843
+ },
2844
+ {
2845
+ "name": "polygon_interior_angles",
2846
+ "score": 0.05930952380952383,
2847
+ "eval_type": "rule",
2848
+ "num_demo": 1,
2849
+ "num_query": 14
2850
+ },
2851
+ {
2852
+ "name": "product_ocr_qa",
2853
+ "score": 0.35714285714285715,
2854
+ "eval_type": "rule",
2855
+ "num_demo": 1,
2856
+ "num_query": 14
2857
+ },
2858
+ {
2859
+ "name": "question_solution_solving",
2860
+ "score": 0.21428571428571427,
2861
+ "eval_type": "rule",
2862
+ "num_demo": 1,
2863
+ "num_query": 14
2864
+ },
2865
+ {
2866
+ "name": "quizlet_question_solving",
2867
+ "score": 0.5714285714285714,
2868
+ "eval_type": "rule",
2869
+ "num_demo": 1,
2870
+ "num_query": 14
2871
+ },
2872
+ {
2873
+ "name": "realworld_qa_en2cn",
2874
+ "score": 0.5,
2875
+ "eval_type": "rule",
2876
+ "num_demo": 1,
2877
+ "num_query": 14
2878
+ },
2879
+ {
2880
+ "name": "rebus",
2881
+ "score": 0.08695652173913043,
2882
+ "eval_type": "rule",
2883
+ "num_demo": 1,
2884
+ "num_query": 23
2885
+ },
2886
+ {
2887
+ "name": "recipe_image_ordering",
2888
+ "score": 0.8571428571428571,
2889
+ "eval_type": "rule",
2890
+ "num_demo": 1,
2891
+ "num_query": 14
2892
+ },
2893
+ {
2894
+ "name": "recover_masked_word_in_figure",
2895
+ "score": 0.35714285714285715,
2896
+ "eval_type": "rule",
2897
+ "num_demo": 1,
2898
+ "num_query": 14
2899
+ },
2900
+ {
2901
+ "name": "relative_depth_of_different_points",
2902
+ "score": 0.7142857142857143,
2903
+ "eval_type": "rule",
2904
+ "num_demo": 1,
2905
+ "num_query": 14
2906
+ },
2907
+ {
2908
+ "name": "relative_reflectance_of_different_regions",
2909
+ "score": 0.5714285714285714,
2910
+ "eval_type": "rule",
2911
+ "num_demo": 1,
2912
+ "num_query": 14
2913
+ },
2914
+ {
2915
+ "name": "remaining_playback_time_calculation",
2916
+ "score": 0.21428571428571427,
2917
+ "eval_type": "rule",
2918
+ "num_demo": 1,
2919
+ "num_query": 14
2920
+ },
2921
+ {
2922
+ "name": "research_website_parsing_blogpost",
2923
+ "score": 0.07142857142857142,
2924
+ "eval_type": "rule",
2925
+ "num_demo": 0,
2926
+ "num_query": 14
2927
+ },
2928
+ {
2929
+ "name": "research_website_parsing_homepage",
2930
+ "score": 0.07142857142857142,
2931
+ "eval_type": "rule",
2932
+ "num_demo": 0,
2933
+ "num_query": 14
2934
+ },
2935
+ {
2936
+ "name": "research_website_parsing_publication",
2937
+ "score": 0.21428571428571427,
2938
+ "eval_type": "rule",
2939
+ "num_demo": 0,
2940
+ "num_query": 14
2941
+ },
2942
+ {
2943
+ "name": "reward_models_i2t_reward",
2944
+ "score": 0.6428571428571429,
2945
+ "eval_type": "rule",
2946
+ "num_demo": 1,
2947
+ "num_query": 14
2948
+ },
2949
+ {
2950
+ "name": "reward_models_t2i_reward",
2951
+ "score": 0.5,
2952
+ "eval_type": "rule",
2953
+ "num_demo": 1,
2954
+ "num_query": 14
2955
+ },
2956
+ {
2957
+ "name": "road_map_find_highway_between_two_place",
2958
+ "score": 0.6470588235294118,
2959
+ "eval_type": "rule",
2960
+ "num_demo": 1,
2961
+ "num_query": 17
2962
+ },
2963
+ {
2964
+ "name": "rocks_samples_compare",
2965
+ "score": 0.42857142857142855,
2966
+ "eval_type": "rule",
2967
+ "num_demo": 1,
2968
+ "num_query": 14
2969
+ },
2970
+ {
2971
+ "name": "rocks_samples_identify",
2972
+ "score": 0.2,
2973
+ "eval_type": "rule",
2974
+ "num_demo": 1,
2975
+ "num_query": 15
2976
+ },
2977
+ {
2978
+ "name": "scibench_calculus_wo_solution",
2979
+ "score": 0.30612244897959184,
2980
+ "eval_type": "rule",
2981
+ "num_demo": 1,
2982
+ "num_query": 49
2983
+ },
2984
+ {
2985
+ "name": "scibench_fundamental_wo_solution",
2986
+ "score": 0.3673469387755102,
2987
+ "eval_type": "rule",
2988
+ "num_demo": 1,
2989
+ "num_query": 49
2990
+ },
2991
+ {
2992
+ "name": "science_basic_physics",
2993
+ "score": 0.8666666666666667,
2994
+ "eval_type": "rule",
2995
+ "num_demo": 1,
2996
+ "num_query": 15
2997
+ },
2998
+ {
2999
+ "name": "science_molecule_chemistry",
3000
+ "score": 0.8,
3001
+ "eval_type": "rule",
3002
+ "num_demo": 1,
3003
+ "num_query": 15
3004
+ },
3005
+ {
3006
+ "name": "screenshot_lighteval_math",
3007
+ "score": 0.6,
3008
+ "eval_type": "rule",
3009
+ "num_demo": 1,
3010
+ "num_query": 15
3011
+ },
3012
+ {
3013
+ "name": "screenshot_theoremqa",
3014
+ "score": 0.7142857142857143,
3015
+ "eval_type": "rule",
3016
+ "num_demo": 1,
3017
+ "num_query": 14
3018
+ },
3019
+ {
3020
+ "name": "semantic_matching_of_two_images",
3021
+ "score": 0.6428571428571429,
3022
+ "eval_type": "rule",
3023
+ "num_demo": 1,
3024
+ "num_query": 14
3025
+ },
3026
+ {
3027
+ "name": "shape_composition_colours",
3028
+ "score": 0.12698412698412698,
3029
+ "eval_type": "rule",
3030
+ "num_demo": 1,
3031
+ "num_query": 14
3032
+ },
3033
+ {
3034
+ "name": "shape_composition_shapes",
3035
+ "score": 0.14285714285714285,
3036
+ "eval_type": "rule",
3037
+ "num_demo": 1,
3038
+ "num_query": 14
3039
+ },
3040
+ {
3041
+ "name": "sign_language",
3042
+ "score": 0.07142857142857142,
3043
+ "eval_type": "rule",
3044
+ "num_demo": 1,
3045
+ "num_query": 14
3046
+ },
3047
+ {
3048
+ "name": "signage_navigation",
3049
+ "score": 0.7857142857142857,
3050
+ "eval_type": "rule",
3051
+ "num_demo": 1,
3052
+ "num_query": 14
3053
+ },
3054
+ {
3055
+ "name": "signboard_identification",
3056
+ "score": 0.5714285714285714,
3057
+ "eval_type": "rule",
3058
+ "num_demo": 1,
3059
+ "num_query": 14
3060
+ },
3061
+ {
3062
+ "name": "single_person_pose_estimation",
3063
+ "score": 0.3191040111385921,
3064
+ "eval_type": "rule",
3065
+ "num_demo": 1,
3066
+ "num_query": 14
3067
+ },
3068
+ {
3069
+ "name": "snli_ve_visual_entailment",
3070
+ "score": 0.8,
3071
+ "eval_type": "rule",
3072
+ "num_demo": 1,
3073
+ "num_query": 15
3074
+ },
3075
+ {
3076
+ "name": "soccer_offside",
3077
+ "score": 0.3333333333333333,
3078
+ "eval_type": "rule",
3079
+ "num_demo": 1,
3080
+ "num_query": 9
3081
+ },
3082
+ {
3083
+ "name": "song_title_identification_from_lyrics",
3084
+ "score": 0.32142857142857145,
3085
+ "eval_type": "rule",
3086
+ "num_demo": 1,
3087
+ "num_query": 14
3088
+ },
3089
+ {
3090
+ "name": "sta_action_localization_video",
3091
+ "score": 0.4375,
3092
+ "eval_type": "rule",
3093
+ "num_demo": 1,
3094
+ "num_query": 16
3095
+ },
3096
+ {
3097
+ "name": "stackoverflow_debug_QA",
3098
+ "score": 0.6904761904761905,
3099
+ "eval_type": "rule",
3100
+ "num_demo": 1,
3101
+ "num_query": 14
3102
+ },
3103
+ {
3104
+ "name": "star_object_interaction_video",
3105
+ "score": 0.625,
3106
+ "eval_type": "rule",
3107
+ "num_demo": 1,
3108
+ "num_query": 16
3109
+ },
3110
+ {
3111
+ "name": "stock_info_parsing",
3112
+ "score": 0.9747899159663866,
3113
+ "eval_type": "rule",
3114
+ "num_demo": 1,
3115
+ "num_query": 14
3116
+ },
3117
+ {
3118
+ "name": "stock_info_retrieval",
3119
+ "score": 0.2857142857142857,
3120
+ "eval_type": "rule",
3121
+ "num_demo": 0,
3122
+ "num_query": 14
3123
+ },
3124
+ {
3125
+ "name": "stock_price_future_prediction",
3126
+ "score": 0.8110714285714286,
3127
+ "eval_type": "rule",
3128
+ "num_demo": 1,
3129
+ "num_query": 14
3130
+ },
3131
+ {
3132
+ "name": "super_clevr_scene_understanding",
3133
+ "score": 0.6428571428571429,
3134
+ "eval_type": "rule",
3135
+ "num_demo": 1,
3136
+ "num_query": 14
3137
+ },
3138
+ {
3139
+ "name": "symbolic_graphics_programs_computer_aided_design",
3140
+ "score": 0.2857142857142857,
3141
+ "eval_type": "rule",
3142
+ "num_demo": 1,
3143
+ "num_query": 14
3144
+ },
3145
+ {
3146
+ "name": "symbolic_graphics_programs_scalable_vector_graphics",
3147
+ "score": 0.16666666666666666,
3148
+ "eval_type": "rule",
3149
+ "num_demo": 1,
3150
+ "num_query": 18
3151
+ },
3152
+ {
3153
+ "name": "table_understanding_complex_question_answering",
3154
+ "score": 0.5714285714285714,
3155
+ "eval_type": "rule",
3156
+ "num_demo": 1,
3157
+ "num_query": 14
3158
+ },
3159
+ {
3160
+ "name": "table_understanding_fact_verification",
3161
+ "score": 0.9285714285714286,
3162
+ "eval_type": "rule",
3163
+ "num_demo": 1,
3164
+ "num_query": 14
3165
+ },
3166
+ {
3167
+ "name": "text_entity_replace",
3168
+ "score": 0.7142857142857143,
3169
+ "eval_type": "rule",
3170
+ "num_demo": 1,
3171
+ "num_query": 14
3172
+ },
3173
+ {
3174
+ "name": "text_style",
3175
+ "score": 0.5,
3176
+ "eval_type": "rule",
3177
+ "num_demo": 1,
3178
+ "num_query": 14
3179
+ },
3180
+ {
3181
+ "name": "top_rated_hotel_identification",
3182
+ "score": 0.8571428571428571,
3183
+ "eval_type": "rule",
3184
+ "num_demo": 1,
3185
+ "num_query": 14
3186
+ },
3187
+ {
3188
+ "name": "top_video_creator_identification",
3189
+ "score": 0.6428571428571429,
3190
+ "eval_type": "rule",
3191
+ "num_demo": 1,
3192
+ "num_query": 14
3193
+ },
3194
+ {
3195
+ "name": "topological_sort",
3196
+ "score": 0.0,
3197
+ "eval_type": "rule",
3198
+ "num_demo": 1,
3199
+ "num_query": 14
3200
+ },
3201
+ {
3202
+ "name": "tqa_textbook_qa",
3203
+ "score": 0.9285714285714286,
3204
+ "eval_type": "rule",
3205
+ "num_demo": 1,
3206
+ "num_query": 14
3207
+ },
3208
+ {
3209
+ "name": "traffic_future_prediction_from_line_plot",
3210
+ "score": 0.7814210526315792,
3211
+ "eval_type": "rule",
3212
+ "num_demo": 1,
3213
+ "num_query": 19
3214
+ },
3215
+ {
3216
+ "name": "transit_map_intersection_points",
3217
+ "score": 0.555952380952381,
3218
+ "eval_type": "rule",
3219
+ "num_demo": 1,
3220
+ "num_query": 14
3221
+ },
3222
+ {
3223
+ "name": "tv_show_retrieval_by_character",
3224
+ "score": 0.7857142857142857,
3225
+ "eval_type": "rule",
3226
+ "num_demo": 0,
3227
+ "num_query": 14
3228
+ },
3229
+ {
3230
+ "name": "veracity",
3231
+ "score": 0.7857142857142857,
3232
+ "eval_type": "rule",
3233
+ "num_demo": 1,
3234
+ "num_query": 14
3235
+ },
3236
+ {
3237
+ "name": "vibe_eval_open",
3238
+ "score": 0.35714285714285715,
3239
+ "eval_type": "rule",
3240
+ "num_demo": 1,
3241
+ "num_query": 14
3242
+ },
3243
+ {
3244
+ "name": "video_action_recognition",
3245
+ "score": 0.8214285714285714,
3246
+ "eval_type": "rule",
3247
+ "num_demo": 1,
3248
+ "num_query": 14
3249
+ },
3250
+ {
3251
+ "name": "video_camera_motion_description",
3252
+ "score": 0.21428571428571427,
3253
+ "eval_type": "rule",
3254
+ "num_demo": 1,
3255
+ "num_query": 14
3256
+ },
3257
+ {
3258
+ "name": "video_content_reasoning",
3259
+ "score": 0.3333333333333333,
3260
+ "eval_type": "rule",
3261
+ "num_demo": 1,
3262
+ "num_query": 9
3263
+ },
3264
+ {
3265
+ "name": "video_eval_dynamic_pref",
3266
+ "score": 0.0,
3267
+ "eval_type": "rule",
3268
+ "num_demo": 1,
3269
+ "num_query": 16
3270
+ },
3271
+ {
3272
+ "name": "video_eval_factual_pref",
3273
+ "score": 0.0,
3274
+ "eval_type": "rule",
3275
+ "num_demo": 1,
3276
+ "num_query": 14
3277
+ },
3278
+ {
3279
+ "name": "video_eval_visual_pref",
3280
+ "score": 0.0,
3281
+ "eval_type": "rule",
3282
+ "num_demo": 1,
3283
+ "num_query": 16
3284
+ },
3285
+ {
3286
+ "name": "video_grounding_spatial",
3287
+ "score": 0.35714285714285715,
3288
+ "eval_type": "rule",
3289
+ "num_demo": 1,
3290
+ "num_query": 14
3291
+ },
3292
+ {
3293
+ "name": "video_grounding_temporal",
3294
+ "score": 0.2,
3295
+ "eval_type": "rule",
3296
+ "num_demo": 1,
3297
+ "num_query": 15
3298
+ },
3299
+ {
3300
+ "name": "video_intent_recognition",
3301
+ "score": 0.6428571428571429,
3302
+ "eval_type": "rule",
3303
+ "num_demo": 1,
3304
+ "num_query": 14
3305
+ },
3306
+ {
3307
+ "name": "video_motion_matching_3D_real",
3308
+ "score": 0.26666666666666666,
3309
+ "eval_type": "rule",
3310
+ "num_demo": 1,
3311
+ "num_query": 15
3312
+ },
3313
+ {
3314
+ "name": "video_motion_matching_real_3D",
3315
+ "score": 0.21428571428571427,
3316
+ "eval_type": "rule",
3317
+ "num_demo": 1,
3318
+ "num_query": 14
3319
+ },
3320
+ {
3321
+ "name": "video_segments_reordering",
3322
+ "score": 0.14285714285714285,
3323
+ "eval_type": "rule",
3324
+ "num_demo": 1,
3325
+ "num_query": 14
3326
+ },
3327
+ {
3328
+ "name": "video_to_camera_trajectory_retrieval",
3329
+ "score": 0.0,
3330
+ "eval_type": "rule",
3331
+ "num_demo": 1,
3332
+ "num_query": 14
3333
+ },
3334
+ {
3335
+ "name": "visual_correspondance_in_two_images",
3336
+ "score": 0.7857142857142857,
3337
+ "eval_type": "rule",
3338
+ "num_demo": 1,
3339
+ "num_query": 14
3340
+ },
3341
+ {
3342
+ "name": "visual_dialog_image_guessing",
3343
+ "score": 0.9333333333333333,
3344
+ "eval_type": "rule",
3345
+ "num_demo": 0,
3346
+ "num_query": 15
3347
+ },
3348
+ {
3349
+ "name": "visual_prediction_rater_3d_assembled_quality_understanding",
3350
+ "score": 0.35714285714285715,
3351
+ "eval_type": "rule",
3352
+ "num_demo": 1,
3353
+ "num_query": 14
3354
+ },
3355
+ {
3356
+ "name": "visual_prediction_rater_depth_estimation",
3357
+ "score": 0.21428571428571427,
3358
+ "eval_type": "rule",
3359
+ "num_demo": 1,
3360
+ "num_query": 14
3361
+ },
3362
+ {
3363
+ "name": "visual_prediction_rater_novel_view_synthesis",
3364
+ "score": 0.21428571428571427,
3365
+ "eval_type": "rule",
3366
+ "num_demo": 1,
3367
+ "num_query": 14
3368
+ },
3369
+ {
3370
+ "name": "visual_prediction_rater_openable_part_segmentation",
3371
+ "score": 0.0,
3372
+ "eval_type": "rule",
3373
+ "num_demo": 1,
3374
+ "num_query": 14
3375
+ },
3376
+ {
3377
+ "name": "visual_prediction_rater_panoptic_segmentation",
3378
+ "score": 0.07142857142857142,
3379
+ "eval_type": "rule",
3380
+ "num_demo": 1,
3381
+ "num_query": 14
3382
+ },
3383
+ {
3384
+ "name": "visual_prediction_rater_plane_segmentation",
3385
+ "score": 0.4222222222222222,
3386
+ "eval_type": "rule",
3387
+ "num_demo": 1,
3388
+ "num_query": 15
3389
+ },
3390
+ {
3391
+ "name": "visual_prediction_rater_semantic_segmentation",
3392
+ "score": 0.5208333333333333,
3393
+ "eval_type": "rule",
3394
+ "num_demo": 1,
3395
+ "num_query": 16
3396
+ },
3397
+ {
3398
+ "name": "visual_prediction_rater_surface_normal_estimation",
3399
+ "score": 0.5952380952380952,
3400
+ "eval_type": "rule",
3401
+ "num_demo": 1,
3402
+ "num_query": 14
3403
+ },
3404
+ {
3405
+ "name": "vizwiz_quality_accessment_for_blind",
3406
+ "score": 0.21428571428571427,
3407
+ "eval_type": "rule",
3408
+ "num_demo": 1,
3409
+ "num_query": 14
3410
+ },
3411
+ {
3412
+ "name": "vln_english_next_step",
3413
+ "score": 0.3333333333333333,
3414
+ "eval_type": "rule",
3415
+ "num_demo": 1,
3416
+ "num_query": 15
3417
+ },
3418
+ {
3419
+ "name": "vln_hindi_next_step",
3420
+ "score": 0.0,
3421
+ "eval_type": "rule",
3422
+ "num_demo": 1,
3423
+ "num_query": 15
3424
+ },
3425
+ {
3426
+ "name": "vln_identify_location",
3427
+ "score": 0.14545454545454545,
3428
+ "eval_type": "rule",
3429
+ "num_demo": 1,
3430
+ "num_query": 15
3431
+ },
3432
+ {
3433
+ "name": "vln_identify_robot",
3434
+ "score": 0.4666666666666667,
3435
+ "eval_type": "rule",
3436
+ "num_demo": 0,
3437
+ "num_query": 15
3438
+ },
3439
+ {
3440
+ "name": "vln_tegulu_next_step",
3441
+ "score": 0.26666666666666666,
3442
+ "eval_type": "rule",
3443
+ "num_demo": 1,
3444
+ "num_query": 15
3445
+ },
3446
+ {
3447
+ "name": "vlnqa_egocentric_navigation_video",
3448
+ "score": 0.3125,
3449
+ "eval_type": "rule",
3450
+ "num_demo": 1,
3451
+ "num_query": 16
3452
+ },
3453
+ {
3454
+ "name": "waldo",
3455
+ "score": 0.0,
3456
+ "eval_type": "rule",
3457
+ "num_demo": 1,
3458
+ "num_query": 18
3459
+ },
3460
+ {
3461
+ "name": "waybill_number_sequence_extraction",
3462
+ "score": 0.5714285714285714,
3463
+ "eval_type": "rule",
3464
+ "num_demo": 1,
3465
+ "num_query": 14
3466
+ },
3467
+ {
3468
+ "name": "weather_info_parsing",
3469
+ "score": 0.8968253968253969,
3470
+ "eval_type": "rule",
3471
+ "num_demo": 1,
3472
+ "num_query": 14
3473
+ },
3474
+ {
3475
+ "name": "weather_info_retrieval",
3476
+ "score": 0.14285714285714285,
3477
+ "eval_type": "rule",
3478
+ "num_demo": 0,
3479
+ "num_query": 14
3480
+ },
3481
+ {
3482
+ "name": "weather_map_climate_type_temperature_parsing",
3483
+ "score": 0.5,
3484
+ "eval_type": "rule",
3485
+ "num_demo": 1,
3486
+ "num_query": 14
3487
+ },
3488
+ {
3489
+ "name": "web_action_grounding",
3490
+ "score": 0.7857142857142857,
3491
+ "eval_type": "rule",
3492
+ "num_demo": 1,
3493
+ "num_query": 14
3494
+ },
3495
+ {
3496
+ "name": "web_action_prediction",
3497
+ "score": 0.7857142857142857,
3498
+ "eval_type": "rule",
3499
+ "num_demo": 1,
3500
+ "num_query": 14
3501
+ },
3502
+ {
3503
+ "name": "webpage_code_understanding",
3504
+ "score": 0.5555555555555556,
3505
+ "eval_type": "rule",
3506
+ "num_demo": 1,
3507
+ "num_query": 9
3508
+ },
3509
+ {
3510
+ "name": "widerface_face_count_and_event_classification",
3511
+ "score": 0.6785714285714286,
3512
+ "eval_type": "rule",
3513
+ "num_demo": 1,
3514
+ "num_query": 14
3515
+ },
3516
+ {
3517
+ "name": "worldle",
3518
+ "score": 0.1642857142857143,
3519
+ "eval_type": "rule",
3520
+ "num_demo": 1,
3521
+ "num_query": 14
3522
+ },
3523
+ {
3524
+ "name": "xor_images",
3525
+ "score": 0.3333333333333333,
3526
+ "eval_type": "rule",
3527
+ "num_demo": 0,
3528
+ "num_query": 15
3529
+ },
3530
+ {
3531
+ "name": "youtube_video_info_parsing",
3532
+ "score": 0.75,
3533
+ "eval_type": "rule",
3534
+ "num_demo": 1,
3535
+ "num_query": 14
3536
+ }
3537
+ ]