Hugging Face's logo

Spaces:
bardsai
/
performance-llm-board
Running

App Files Community
piotr-szleg-bards-ai commited on
Commit
96a59f8
·
1 Parent(s): da3a772

2024-02-06 15:37:17 Publish script update

Browse files
Files changed (3) hide show
  1. app.py +1 -1
  2. data/2024-02-06 15:35:24.490454_general_plot.csv +452 -0
  3. pipeline/models.py +2 -0
app.py CHANGED
@@ -48,7 +48,7 @@ output_types_df = pd.DataFrame({
48
 
49
  summary_df: pd.DataFrame = pd.read_csv("data/2024-02-05 23:33:22.947120_summary.csv")
50
  time_of_day_comparison_df = pd.read_csv("data/2024-02-06 09:49:19.637072_time_of_day_comparison.csv")
51
- general_plots = pd.read_csv("data/2024-02-06 13:51:35.085414_general_plot.csv")
52
  model_costs_df = pd.read_csv("data/2024-02-05 12:03:45.281624_model_costs.csv")
53
 
54
 
 
48
 
49
  summary_df: pd.DataFrame = pd.read_csv("data/2024-02-05 23:33:22.947120_summary.csv")
50
  time_of_day_comparison_df = pd.read_csv("data/2024-02-06 09:49:19.637072_time_of_day_comparison.csv")
51
+ general_plots = pd.read_csv("data/2024-02-06 15:35:24.490454_general_plot.csv")
52
  model_costs_df = pd.read_csv("data/2024-02-05 12:03:45.281624_model_costs.csv")
53
 
54
 
data/2024-02-06 15:35:24.490454_general_plot.csv ADDED
@@ -0,0 +1,452 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ path,plot_object,header,description,plot_json
2
+ ./html/plots/response_times.html,"Figure({
3
+ 'data': [{'alignmentgroup': 'True',
4
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
5
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
6
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
7
+ 'name': 'Mistral-7B-Instruct-v0.2',
8
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
9
+ 'orientation': 'v',
10
+ 'showlegend': True,
11
+ 'textposition': 'auto',
12
+ 'type': 'bar',
13
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
14
+ 'xaxis': 'x',
15
+ 'y': array([3.27102681]),
16
+ 'yaxis': 'y'},
17
+ {'alignmentgroup': 'True',
18
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
19
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
20
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
21
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
22
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
23
+ 'orientation': 'v',
24
+ 'showlegend': True,
25
+ 'textposition': 'auto',
26
+ 'type': 'bar',
27
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
28
+ 'xaxis': 'x',
29
+ 'y': array([4.83500647]),
30
+ 'yaxis': 'y'},
31
+ {'alignmentgroup': 'True',
32
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
33
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
34
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
35
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
36
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
37
+ 'orientation': 'v',
38
+ 'showlegend': True,
39
+ 'textposition': 'auto',
40
+ 'type': 'bar',
41
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
42
+ 'xaxis': 'x',
43
+ 'y': array([1.79273835]),
44
+ 'yaxis': 'y'},
45
+ {'alignmentgroup': 'True',
46
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
47
+ 'legendgroup': 'chat-bison',
48
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
49
+ 'name': 'chat-bison',
50
+ 'offsetgroup': 'chat-bison',
51
+ 'orientation': 'v',
52
+ 'showlegend': True,
53
+ 'textposition': 'auto',
54
+ 'type': 'bar',
55
+ 'x': array(['chat-bison'], dtype=object),
56
+ 'xaxis': 'x',
57
+ 'y': array([3.89024363]),
58
+ 'yaxis': 'y'},
59
+ {'alignmentgroup': 'True',
60
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
61
+ 'legendgroup': 'chat-bison (PaLM 2)',
62
+ 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
63
+ 'name': 'chat-bison (PaLM 2)',
64
+ 'offsetgroup': 'chat-bison (PaLM 2)',
65
+ 'orientation': 'v',
66
+ 'showlegend': True,
67
+ 'textposition': 'auto',
68
+ 'type': 'bar',
69
+ 'x': array(['chat-bison (PaLM 2)'], dtype=object),
70
+ 'xaxis': 'x',
71
+ 'y': array([2.66830386]),
72
+ 'yaxis': 'y'},
73
+ {'alignmentgroup': 'True',
74
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
75
+ 'legendgroup': 'chat-bison-32k',
76
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
77
+ 'name': 'chat-bison-32k',
78
+ 'offsetgroup': 'chat-bison-32k',
79
+ 'orientation': 'v',
80
+ 'showlegend': True,
81
+ 'textposition': 'auto',
82
+ 'type': 'bar',
83
+ 'x': array(['chat-bison-32k'], dtype=object),
84
+ 'xaxis': 'x',
85
+ 'y': array([4.12817777]),
86
+ 'yaxis': 'y'},
87
+ {'alignmentgroup': 'True',
88
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
89
+ 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
90
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
91
+ 'name': 'chat-bison-32k (PaLM 2 32K)',
92
+ 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
93
+ 'orientation': 'v',
94
+ 'showlegend': True,
95
+ 'textposition': 'auto',
96
+ 'type': 'bar',
97
+ 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
98
+ 'xaxis': 'x',
99
+ 'y': array([6.73886213]),
100
+ 'yaxis': 'y'},
101
+ {'alignmentgroup': 'True',
102
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
103
+ 'legendgroup': 'gemini-pro',
104
+ 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
105
+ 'name': 'gemini-pro',
106
+ 'offsetgroup': 'gemini-pro',
107
+ 'orientation': 'v',
108
+ 'showlegend': True,
109
+ 'textposition': 'auto',
110
+ 'type': 'bar',
111
+ 'x': array(['gemini-pro'], dtype=object),
112
+ 'xaxis': 'x',
113
+ 'y': array([3.03678889]),
114
+ 'yaxis': 'y'},
115
+ {'alignmentgroup': 'True',
116
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
117
+ 'legendgroup': 'gpt-3.5-turbo',
118
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
119
+ 'name': 'gpt-3.5-turbo',
120
+ 'offsetgroup': 'gpt-3.5-turbo',
121
+ 'orientation': 'v',
122
+ 'showlegend': True,
123
+ 'textposition': 'auto',
124
+ 'type': 'bar',
125
+ 'x': array(['gpt-3.5-turbo'], dtype=object),
126
+ 'xaxis': 'x',
127
+ 'y': array([4.29307391]),
128
+ 'yaxis': 'y'},
129
+ {'alignmentgroup': 'True',
130
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
131
+ 'legendgroup': 'gpt-4',
132
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
133
+ 'name': 'gpt-4',
134
+ 'offsetgroup': 'gpt-4',
135
+ 'orientation': 'v',
136
+ 'showlegend': True,
137
+ 'textposition': 'auto',
138
+ 'type': 'bar',
139
+ 'x': array(['gpt-4'], dtype=object),
140
+ 'xaxis': 'x',
141
+ 'y': array([14.32373439]),
142
+ 'yaxis': 'y'},
143
+ {'alignmentgroup': 'True',
144
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
145
+ 'legendgroup': 'gpt-4-turbo',
146
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
147
+ 'name': 'gpt-4-turbo',
148
+ 'offsetgroup': 'gpt-4-turbo',
149
+ 'orientation': 'v',
150
+ 'showlegend': True,
151
+ 'textposition': 'auto',
152
+ 'type': 'bar',
153
+ 'x': array(['gpt-4-turbo'], dtype=object),
154
+ 'xaxis': 'x',
155
+ 'y': array([15.5354884]),
156
+ 'yaxis': 'y'},
157
+ {'alignmentgroup': 'True',
158
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
159
+ 'legendgroup': 'llama-2-70b-chat',
160
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
161
+ 'name': 'llama-2-70b-chat',
162
+ 'offsetgroup': 'llama-2-70b-chat',
163
+ 'orientation': 'v',
164
+ 'showlegend': True,
165
+ 'textposition': 'auto',
166
+ 'type': 'bar',
167
+ 'x': array(['llama-2-70b-chat'], dtype=object),
168
+ 'xaxis': 'x',
169
+ 'y': array([3.6742315]),
170
+ 'yaxis': 'y'},
171
+ {'alignmentgroup': 'True',
172
+ 'hovertemplate': 'Model=%{x}<br>Execution time (s)=%{y}<extra></extra>',
173
+ 'legendgroup': 'zephyr-7b-beta',
174
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
175
+ 'name': 'zephyr-7b-beta',
176
+ 'offsetgroup': 'zephyr-7b-beta',
177
+ 'orientation': 'v',
178
+ 'showlegend': True,
179
+ 'textposition': 'auto',
180
+ 'type': 'bar',
181
+ 'x': array(['zephyr-7b-beta'], dtype=object),
182
+ 'xaxis': 'x',
183
+ 'y': array([3.6370095]),
184
+ 'yaxis': 'y'}],
185
+ 'layout': {'barmode': 'relative',
186
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
187
+ 'template': '...',
188
+ 'title': {'text': 'Model endpoints response times'},
189
+ 'xaxis': {'anchor': 'y',
190
+ 'categoryarray': [Mistral-7B-Instruct-v0.2,
191
+ Mixtral-8x7B-Instruct-v0.1,
192
+ TinyLlama/TinyLlama-1.1B-Chat-v1.0,
193
+ chat-bison, chat-bison (PaLM 2), chat-
194
+ bison-32k, chat-bison-32k (PaLM 2 32K),
195
+ gemini-pro, gpt-3.5-turbo, gpt-4,
196
+ gpt-4-turbo, llama-2-70b-chat,
197
+ zephyr-7b-beta],
198
+ 'categoryorder': 'array',
199
+ 'domain': [0.0, 1.0],
200
+ 'title': {'text': 'Model'}},
201
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
202
+ })",Model endpoints response times,"This plot shows averaged response time for one query for each model.
203
+
204
+ We used `time.time()` result saved to variable before making the call to API and compared it to `time.time()` result after receiving the results.
205
+
206
+ It is also important to note that we used litellm python library for all of the models which adds some overhead compared to pure curl calls.","{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.271026806036631],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[4.835006467343727],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.7927383464436197],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[2.668303858522159],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[6.738862129823485],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[3.036788886637026],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[4.293073908173212],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[14.323734385105828],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[15.535488395149827],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[3.6742314998615178],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.6370095004383316],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model endpoints response times""},""barmode"":""relative""}}"
207
+ ./html/plots/execution_costs.html,"Figure({
208
+ 'data': [{'alignmentgroup': 'True',
209
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
210
+ 'legendgroup': 'gpt-3.5-turbo',
211
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
212
+ 'name': 'gpt-3.5-turbo',
213
+ 'offsetgroup': 'gpt-3.5-turbo',
214
+ 'orientation': 'v',
215
+ 'showlegend': True,
216
+ 'textposition': 'auto',
217
+ 'type': 'bar',
218
+ 'x': array(['gpt-3.5-turbo'], dtype=object),
219
+ 'xaxis': 'x',
220
+ 'y': array([0.23604]),
221
+ 'yaxis': 'y'},
222
+ {'alignmentgroup': 'True',
223
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
224
+ 'legendgroup': 'gpt-4-turbo',
225
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
226
+ 'name': 'gpt-4-turbo',
227
+ 'offsetgroup': 'gpt-4-turbo',
228
+ 'orientation': 'v',
229
+ 'showlegend': True,
230
+ 'textposition': 'auto',
231
+ 'type': 'bar',
232
+ 'x': array(['gpt-4-turbo'], dtype=object),
233
+ 'xaxis': 'x',
234
+ 'y': array([2.5988]),
235
+ 'yaxis': 'y'},
236
+ {'alignmentgroup': 'True',
237
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
238
+ 'legendgroup': 'gpt-4',
239
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
240
+ 'name': 'gpt-4',
241
+ 'offsetgroup': 'gpt-4',
242
+ 'orientation': 'v',
243
+ 'showlegend': True,
244
+ 'textposition': 'auto',
245
+ 'type': 'bar',
246
+ 'x': array(['gpt-4'], dtype=object),
247
+ 'xaxis': 'x',
248
+ 'y': array([5.8344]),
249
+ 'yaxis': 'y'},
250
+ {'alignmentgroup': 'True',
251
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
252
+ 'legendgroup': 'llama-2-70b-chat',
253
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
254
+ 'name': 'llama-2-70b-chat',
255
+ 'offsetgroup': 'llama-2-70b-chat',
256
+ 'orientation': 'v',
257
+ 'showlegend': True,
258
+ 'textposition': 'auto',
259
+ 'type': 'bar',
260
+ 'x': array(['llama-2-70b-chat'], dtype=object),
261
+ 'xaxis': 'x',
262
+ 'y': array([0.184464]),
263
+ 'yaxis': 'y'},
264
+ {'alignmentgroup': 'True',
265
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
266
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
267
+ 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
268
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
269
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
270
+ 'orientation': 'v',
271
+ 'showlegend': True,
272
+ 'textposition': 'auto',
273
+ 'type': 'bar',
274
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
275
+ 'xaxis': 'x',
276
+ 'y': array([0.143622]),
277
+ 'yaxis': 'y'},
278
+ {'alignmentgroup': 'True',
279
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
280
+ 'legendgroup': 'zephyr-7b-beta',
281
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
282
+ 'name': 'zephyr-7b-beta',
283
+ 'offsetgroup': 'zephyr-7b-beta',
284
+ 'orientation': 'v',
285
+ 'showlegend': True,
286
+ 'textposition': 'auto',
287
+ 'type': 'bar',
288
+ 'x': array(['zephyr-7b-beta'], dtype=object),
289
+ 'xaxis': 'x',
290
+ 'y': array([0.15650749]),
291
+ 'yaxis': 'y'},
292
+ {'alignmentgroup': 'True',
293
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
294
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
295
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
296
+ 'name': 'Mistral-7B-Instruct-v0.2',
297
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
298
+ 'orientation': 'v',
299
+ 'showlegend': True,
300
+ 'textposition': 'auto',
301
+ 'type': 'bar',
302
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
303
+ 'xaxis': 'x',
304
+ 'y': array([0.08633046]),
305
+ 'yaxis': 'y'},
306
+ {'alignmentgroup': 'True',
307
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
308
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
309
+ 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
310
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
311
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
312
+ 'orientation': 'v',
313
+ 'showlegend': True,
314
+ 'textposition': 'auto',
315
+ 'type': 'bar',
316
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
317
+ 'xaxis': 'x',
318
+ 'y': array([0.02132704]),
319
+ 'yaxis': 'y'},
320
+ {'alignmentgroup': 'True',
321
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
322
+ 'legendgroup': 'gemini-pro',
323
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
324
+ 'name': 'gemini-pro',
325
+ 'offsetgroup': 'gemini-pro',
326
+ 'orientation': 'v',
327
+ 'showlegend': True,
328
+ 'textposition': 'auto',
329
+ 'type': 'bar',
330
+ 'x': array(['gemini-pro'], dtype=object),
331
+ 'xaxis': 'x',
332
+ 'y': array([0.05781]),
333
+ 'yaxis': 'y'},
334
+ {'alignmentgroup': 'True',
335
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
336
+ 'legendgroup': 'chat-bison (PaLM 2)',
337
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
338
+ 'name': 'chat-bison (PaLM 2)',
339
+ 'offsetgroup': 'chat-bison (PaLM 2)',
340
+ 'orientation': 'v',
341
+ 'showlegend': True,
342
+ 'textposition': 'auto',
343
+ 'type': 'bar',
344
+ 'x': array(['chat-bison (PaLM 2)'], dtype=object),
345
+ 'xaxis': 'x',
346
+ 'y': array([0.05291]),
347
+ 'yaxis': 'y'},
348
+ {'alignmentgroup': 'True',
349
+ 'hovertemplate': 'Model=%{x}<br>Execution cost ($)=%{y}<extra></extra>',
350
+ 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
351
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
352
+ 'name': 'chat-bison-32k (PaLM 2 32K)',
353
+ 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
354
+ 'orientation': 'v',
355
+ 'showlegend': True,
356
+ 'textposition': 'auto',
357
+ 'type': 'bar',
358
+ 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
359
+ 'xaxis': 'x',
360
+ 'y': array([0.04954]),
361
+ 'yaxis': 'y'}],
362
+ 'layout': {'barmode': 'relative',
363
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
364
+ 'template': '...',
365
+ 'title': {'text': 'Costs of execution of 800 test queries per model'},
366
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
367
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}}}
368
+ })",Costs of execution of 800 test queries per model,"Costs of experiments carried out to create this table, 800 test queries per model.
369
+
370
+ Note that pause and resume time cost was not included in the cost calculation for the models billed hourly.
371
+
372
+ Also although Gemini-Pro model is free for under 60 queries per minute we calculated the full cost for generating the tokens.","{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[0.23604],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[2.5987999999999998],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[5.8344000000000005],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[0.18446400000000002],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[0.14362200000000003],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[0.15650748757521313],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[0.08633045819070603],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[0.02132703948020935],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[0.05781],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[0.05291],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution cost ($)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[0.04954],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 800 test queries per model""},""barmode"":""relative""}}"
373
+ ./html/plots/model_sizes.html,"Figure({
374
+ 'data': [{'alignmentgroup': 'True',
375
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
376
+ 'legendgroup': 'llama-2-70b-chat',
377
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
378
+ 'name': 'llama-2-70b-chat',
379
+ 'offsetgroup': 'llama-2-70b-chat',
380
+ 'orientation': 'v',
381
+ 'showlegend': True,
382
+ 'textposition': 'auto',
383
+ 'type': 'bar',
384
+ 'x': array(['llama-2-70b-chat'], dtype=object),
385
+ 'xaxis': 'x',
386
+ 'y': array([70.]),
387
+ 'yaxis': 'y'},
388
+ {'alignmentgroup': 'True',
389
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
390
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
391
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
392
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
393
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
394
+ 'orientation': 'v',
395
+ 'showlegend': True,
396
+ 'textposition': 'auto',
397
+ 'type': 'bar',
398
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
399
+ 'xaxis': 'x',
400
+ 'y': array([56.]),
401
+ 'yaxis': 'y'},
402
+ {'alignmentgroup': 'True',
403
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
404
+ 'legendgroup': 'zephyr-7b-beta',
405
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
406
+ 'name': 'zephyr-7b-beta',
407
+ 'offsetgroup': 'zephyr-7b-beta',
408
+ 'orientation': 'v',
409
+ 'showlegend': True,
410
+ 'textposition': 'auto',
411
+ 'type': 'bar',
412
+ 'x': array(['zephyr-7b-beta'], dtype=object),
413
+ 'xaxis': 'x',
414
+ 'y': array([7.]),
415
+ 'yaxis': 'y'},
416
+ {'alignmentgroup': 'True',
417
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
418
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
419
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
420
+ 'name': 'Mistral-7B-Instruct-v0.2',
421
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
422
+ 'orientation': 'v',
423
+ 'showlegend': True,
424
+ 'textposition': 'auto',
425
+ 'type': 'bar',
426
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
427
+ 'xaxis': 'x',
428
+ 'y': array([7.]),
429
+ 'yaxis': 'y'},
430
+ {'alignmentgroup': 'True',
431
+ 'hovertemplate': 'Model=%{x}<br>Model size (B params)=%{y}<extra></extra>',
432
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
433
+ 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
434
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
435
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
436
+ 'orientation': 'v',
437
+ 'showlegend': True,
438
+ 'textposition': 'auto',
439
+ 'type': 'bar',
440
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
441
+ 'xaxis': 'x',
442
+ 'y': array([1.1]),
443
+ 'yaxis': 'y'}],
444
+ 'layout': {'barmode': 'relative',
445
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
446
+ 'template': '...',
447
+ 'title': {'text': 'Model sizes in billions of parameters'},
448
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model'}},
449
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}}}
450
+ })",Model sizes in billions of parameters,"Model sizes based on release blogs and documentation of their respective developers.
451
+
452
+ Note that models, which sizes were not disclosed publicly, were omitted from this comparison.","{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[70.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[56.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[7.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eModel size (B params)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.1],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}"
pipeline/models.py CHANGED
@@ -146,6 +146,8 @@ MODELS = [
146
  ),
147
  ]
148
 
 
149
  MODELS = [model for model in MODELS
150
  if model.model_name=="together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1"
151
  or model.model_name=="huggingface/HuggingFaceH4/zephyr-7b-beta"]
 
 
146
  ),
147
  ]
148
 
149
+ """
150
  MODELS = [model for model in MODELS
151
  if model.model_name=="together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1"
152
  or model.model_name=="huggingface/HuggingFaceH4/zephyr-7b-beta"]
153
+ """