diff --git a/app.py b/app.py
index 9131283088f1026cbe1c5a153b6256b711925daa..3bdcce66388b55ad22a9537f95f04da35b4872b2 100644
--- a/app.py
+++ b/app.py
@@ -222,24 +222,6 @@ with gr.Blocks() as demo:
queries_config.base_query_template.replace("```", "'''"), output_types_df.to_markdown(index=False)
)
)
- with gr.Tab("General plots"):
- general_plots.apply(display_plot, axis=1)
- with gr.Tab("Output characteristics"):
- with gr.Row():
- collapse_languages_button.render()
- collapse_output_method_button.render()
- summary_ui = gr.DataFrame(dataframe_style(summary_df), label="Output characteristics")
- gr.Markdown(
- """\
-This table compares output characteristics of different models which include execution time, output size and chunking of the output. Some providers and models don't support output chunking, in this case chunk related fields are left empty.
-
-Execution time refers to averaged time needed to execute one query.
-
-To count words we split the output string by whitespace `\w` regex character.
-
-Chunk sizes are measured in the characters count."""
- )
- output_plots.apply(display_plot, axis=1)
with gr.Tab("Performance by time of the day"):
# display only first plot for all models
time_of_day_plots[0:1].apply(display_plot, axis=1)
@@ -256,11 +238,28 @@ Execution time refers to averaged time needed to execute one query.
Hours and times of day in the table and in the plot are based on Central European Time.
Measurements were made during a normal work week.
+
+To compare providers we checked execution times for the same model `Mistral (7B) Instruct v0.2` on both Hugging Face and Together AI.
"""
)
# display rest of the plots
time_of_day_plots[1:].apply(display_plot, axis=1)
+ with gr.Tab("Output characteristics"):
+ with gr.Row():
+ collapse_languages_button.render()
+ collapse_output_method_button.render()
+ summary_ui = gr.DataFrame(dataframe_style(summary_df), label="Output characteristics")
+ gr.Markdown(
+ """\
+This table compares output characteristics of different models which include execution time, output size and chunking of the output. Some providers and models don't support output chunking, in this case chunk related fields are left empty.
+
+Execution time refers to averaged time needed to execute one query.
+
+To count words we split the output string by whitespace `\w` regex character.
+Chunk sizes are measured in the characters count."""
+ )
+ output_plots.apply(display_plot, axis=1)
with gr.Tab("Costs comparison"):
models_costs_ui = gr.DataFrame(dataframe_style(model_costs_df), label="Costs comparison")
gr.Markdown(
@@ -273,6 +272,14 @@ for models hosted this way we calculated "Cost Per Token" column using data coll
Note that pause and resume time cost was not included in the "Cost Per Token" column calculation.
"""
)
+ general_plots[general_plots.plot_name=="execution_costs"].apply(display_plot, axis=1)
+ with gr.Tab("Model dimensions"):
+ general_plots[general_plots.plot_name!="execution_costs"].apply(display_plot, axis=1)
+ gr.Markdown("""
+Model dimensions based on release blogs and documentation of their respective developers.
+
+Note that models, which dimensions were not disclosed publicly, were omitted from the plots.
+""")
filter_button.click(
fn=filter_dataframes,
inputs=filter_textbox,
diff --git a/data/general_plots.csv b/data/general_plots.csv
index 34670ce4a28c7b7587c27cab658347eab8d8e1d0..0efae2205a6d262f555b806b84d621b2952b8bcc 100644
--- a/data/general_plots.csv
+++ b/data/general_plots.csv
@@ -1,659 +1,617 @@
-path,plot_object,header,description,plot_json,arguments,df
-./html/plots/response_times.html,"Figure({
+plot_name,path,plot_object,header,description,plot_json,arguments,df
+execution_costs,./html/plots/execution_costs.html,"Figure({
'data': [{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': '01-ai Yi Chat (34B)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'gpt-4',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': '01-ai Yi Chat (34B)',
- 'offsetgroup': '01-ai Yi Chat (34B)',
- 'orientation': 'v',
+ 'name': 'gpt-4',
+ 'offsetgroup': 'gpt-4',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['01-ai Yi Chat (34B)'], dtype=object),
+ 'x': array([0.0249]),
'xaxis': 'x',
- 'y': array([nan]),
+ 'y': array(['gpt-4'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Chronos Hermes (13B)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'gpt-4-turbo',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'Chronos Hermes (13B)',
- 'offsetgroup': 'Chronos Hermes (13B)',
- 'orientation': 'v',
+ 'name': 'gpt-4-turbo',
+ 'offsetgroup': 'gpt-4-turbo',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Chronos Hermes (13B)'], dtype=object),
+ 'x': array([0.0229]),
'xaxis': 'x',
- 'y': array([4.16887414]),
+ 'y': array(['gpt-4-turbo'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Falcon Instruct (7B)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'gpt-3.5-turbo',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'Falcon Instruct (7B)',
- 'offsetgroup': 'Falcon Instruct (7B)',
- 'orientation': 'v',
+ 'name': 'gpt-3.5-turbo',
+ 'offsetgroup': 'gpt-3.5-turbo',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Falcon Instruct (7B)'], dtype=object),
+ 'x': array([0.00083]),
'xaxis': 'x',
- 'y': array([1.07813787]),
+ 'y': array(['gpt-3.5-turbo'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'LLaMA-2 Chat (7B)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'llama-2-70b-chat',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'LLaMA-2 Chat (7B)',
- 'offsetgroup': 'LLaMA-2 Chat (7B)',
- 'orientation': 'v',
+ 'name': 'llama-2-70b-chat',
+ 'offsetgroup': 'llama-2-70b-chat',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['LLaMA-2 Chat (7B)'], dtype=object),
+ 'x': array([0.000711]),
'xaxis': 'x',
- 'y': array([1.68668771]),
+ 'y': array(['llama-2-70b-chat'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
- 'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
- 'orientation': 'v',
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
+ 'x': array([0.000711]),
'xaxis': 'x',
- 'y': array([1.77096045]),
+ 'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Mistral-7B-Instruct-v0.2',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'gemini-pro',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
- 'name': 'Mistral-7B-Instruct-v0.2',
- 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
- 'orientation': 'v',
+ 'name': 'gemini-pro',
+ 'offsetgroup': 'gemini-pro',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+ 'x': array([0.0001975]),
'xaxis': 'x',
- 'y': array([3.27102681]),
+ 'y': array(['gemini-pro'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'chat-bison (PaLM 2)',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
- 'name': 'Mixtral-8x7B-Instruct-v0.1',
- 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
- 'orientation': 'v',
+ 'name': 'chat-bison (PaLM 2)',
+ 'offsetgroup': 'chat-bison (PaLM 2)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+ 'x': array([0.0001975]),
'xaxis': 'x',
- 'y': array([4.83500647]),
+ 'y': array(['chat-bison (PaLM 2)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
- 'name': 'OpenHermes-2.5-Mistral (7B)',
- 'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
- 'xaxis': 'x',
- 'y': array([3.33096337]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Qwen 1.5 Chat (7B)',
- 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
- 'name': 'Qwen 1.5 Chat (7B)',
- 'offsetgroup': 'Qwen 1.5 Chat (7B)',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['Qwen 1.5 Chat (7B)'], dtype=object),
- 'xaxis': 'x',
- 'y': array([nan]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'RedPajama-INCITE Chat (7B)',
- 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
- 'name': 'RedPajama-INCITE Chat (7B)',
- 'offsetgroup': 'RedPajama-INCITE Chat (7B)',
- 'orientation': 'v',
+ 'name': 'chat-bison-32k (PaLM 2 32K)',
+ 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
+ 'x': array([0.0001975]),
'xaxis': 'x',
- 'y': array([1.34753919]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
+ 'y': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+ 'yaxis': 'y'}],
+ 'layout': {'barmode': 'relative',
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+ 'template': '...',
+ 'title': {'text': 'Costs of execution of 20 test queries per model'},
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}},
+ 'yaxis': {'anchor': 'x',
+ 'categoryarray': [chat-bison-32k (PaLM 2 32K), chat-bison
+ (PaLM 2), gemini-pro,
+ Mixtral-8x7B-Instruct-v0.1,
+ llama-2-70b-chat, gpt-3.5-turbo,
+ gpt-4-turbo, gpt-4],
+ 'categoryorder': 'array',
+ 'domain': [0.0, 1.0],
+ 'title': {'text': 'Model'}}}
+})",Costs of execution of 20 test queries per model,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0249],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0229],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.00083],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.000711],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.000711],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0001975],""xaxis"":""x"",""y"":[""gemini-pro""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0001975],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0001975],""xaxis"":""x"",""y"":[""chat-bison-32k (PaLM 2 32K)""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""chat-bison-32k (PaLM 2 32K)"",""chat-bison (PaLM 2)"",""gemini-pro"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat"",""gpt-3.5-turbo"",""gpt-4-turbo"",""gpt-4""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 20 test queries per model""},""barmode"":""relative""}}","{""y"": ""model"", ""x"": ""model_query_costs"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Costs of execution of 20 test queries per model"", ""labels"": {""model"": ""Model"", ""model_query_costs"": ""Execution cost ($)""}}",",model_query_costs,model
+2,0.0249,gpt-4
+1,0.0229,gpt-4-turbo
+0,0.00083,gpt-3.5-turbo
+3,0.000711,llama-2-70b-chat
+4,0.000711,Mixtral-8x7B-Instruct-v0.1
+8,0.0001975,gemini-pro
+9,0.0001975,chat-bison (PaLM 2)
+10,0.0001975,chat-bison-32k (PaLM 2 32K)
+"
+model_sizes,./html/plots/model_sizes.html,"Figure({
+ 'data': [{'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'llama-2-70b-chat',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'Snorkel Mistral PairRM DPO (7B)',
- 'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
- 'orientation': 'v',
+ 'name': 'llama-2-70b-chat',
+ 'offsetgroup': 'llama-2-70b-chat',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
+ 'x': array([70.]),
'xaxis': 'x',
- 'y': array([4.39964783]),
+ 'y': array(['llama-2-70b-chat'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'orientation': 'v',
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'x': array([56.]),
'xaxis': 'x',
- 'y': array([1.79273835]),
+ 'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'WizardLM v1.2 (13B)',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'Upstage SOLAR Instruct v1 (11B)',
- 'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
- 'orientation': 'v',
+ 'name': 'WizardLM v1.2 (13B)',
+ 'offsetgroup': 'WizardLM v1.2 (13B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
+ 'x': array([13.]),
'xaxis': 'x',
- 'y': array([nan]),
+ 'y': array(['WizardLM v1.2 (13B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Vicuna v1.5 (7B)',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'Vicuna v1.5 (7B)',
- 'offsetgroup': 'Vicuna v1.5 (7B)',
- 'orientation': 'v',
+ 'name': 'Upstage SOLAR Instruct v1 (11B)',
+ 'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Vicuna v1.5 (7B)'], dtype=object),
+ 'x': array([11.]),
'xaxis': 'x',
- 'y': array([nan]),
+ 'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'WizardLM v1.2 (13B)',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'zephyr-7b-beta',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'WizardLM v1.2 (13B)',
- 'offsetgroup': 'WizardLM v1.2 (13B)',
- 'orientation': 'v',
+ 'name': 'zephyr-7b-beta',
+ 'offsetgroup': 'zephyr-7b-beta',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['WizardLM v1.2 (13B)'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([3.35003853]),
+ 'y': array(['zephyr-7b-beta'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
- 'name': 'chat-bison',
- 'offsetgroup': 'chat-bison',
- 'orientation': 'v',
+ 'name': 'Mistral-7B-Instruct-v0.2',
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([3.89024363]),
+ 'y': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison (PaLM 2)',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Vicuna v1.5 (7B)',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
- 'name': 'chat-bison (PaLM 2)',
- 'offsetgroup': 'chat-bison (PaLM 2)',
- 'orientation': 'v',
+ 'name': 'Vicuna v1.5 (7B)',
+ 'offsetgroup': 'Vicuna v1.5 (7B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison (PaLM 2)'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([2.66830386]),
+ 'y': array(['Vicuna v1.5 (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison-32k',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'LLaMA-2 Chat (7B)',
'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k',
- 'offsetgroup': 'chat-bison-32k',
- 'orientation': 'v',
+ 'name': 'LLaMA-2 Chat (7B)',
+ 'offsetgroup': 'LLaMA-2 Chat (7B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison-32k'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([4.12817777]),
+ 'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k (PaLM 2 32K)',
- 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
- 'orientation': 'v',
+ 'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([6.73886213]),
+ 'y': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gemini-pro',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Qwen 1.5 Chat (7B)',
'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
- 'name': 'gemini-pro',
- 'offsetgroup': 'gemini-pro',
- 'orientation': 'v',
+ 'name': 'Qwen 1.5 Chat (7B)',
+ 'offsetgroup': 'Qwen 1.5 Chat (7B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['gemini-pro'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([3.03678889]),
+ 'y': array(['Qwen 1.5 Chat (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gpt-3.5-turbo',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'RedPajama-INCITE Chat (7B)',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'gpt-3.5-turbo',
- 'offsetgroup': 'gpt-3.5-turbo',
- 'orientation': 'v',
+ 'name': 'RedPajama-INCITE Chat (7B)',
+ 'offsetgroup': 'RedPajama-INCITE Chat (7B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['gpt-3.5-turbo'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([4.29307391]),
+ 'y': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gpt-4',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'gpt-4',
- 'offsetgroup': 'gpt-4',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['gpt-4'], dtype=object),
- 'xaxis': 'x',
- 'y': array([14.32373439]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gpt-4-turbo',
- 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'gpt-4-turbo',
- 'offsetgroup': 'gpt-4-turbo',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['gpt-4-turbo'], dtype=object),
- 'xaxis': 'x',
- 'y': array([15.5354884]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'llama-2-70b-chat',
- 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'llama-2-70b-chat',
- 'offsetgroup': 'llama-2-70b-chat',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['llama-2-70b-chat'], dtype=object),
- 'xaxis': 'x',
- 'y': array([3.6742315]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'zephyr-7b-beta',
- 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'zephyr-7b-beta',
- 'offsetgroup': 'zephyr-7b-beta',
- 'orientation': 'v',
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['zephyr-7b-beta'], dtype=object),
+ 'x': array([1.1]),
'xaxis': 'x',
- 'y': array([3.6370095]),
+ 'y': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
'yaxis': 'y'}],
'layout': {'barmode': 'relative',
'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
'template': '...',
- 'title': {'text': 'Model endpoints response times'},
- 'xaxis': {'anchor': 'y',
- 'categoryarray': [01-ai Yi Chat (34B), Chronos Hermes
- (13B), Falcon Instruct (7B), LLaMA-2
+ 'title': {'text': 'Model sizes in billions of parameters'},
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}},
+ 'yaxis': {'anchor': 'x',
+ 'categoryarray': [TinyLlama/TinyLlama-1.1B-Chat-v1.0,
+ RedPajama-INCITE Chat (7B), Qwen 1.5
Chat (7B), Mistral (7B) Instruct v0.2
- (Together AI), Mistral-7B-Instruct-v0.2,
+ (Together AI), LLaMA-2 Chat (7B), Vicuna
+ v1.5 (7B), Mistral-7B-Instruct-v0.2,
+ zephyr-7b-beta, Upstage SOLAR Instruct
+ v1 (11B), WizardLM v1.2 (13B),
Mixtral-8x7B-Instruct-v0.1,
- OpenHermes-2.5-Mistral (7B), Qwen 1.5
- Chat (7B), RedPajama-INCITE Chat (7B),
- Snorkel Mistral PairRM DPO (7B),
- TinyLlama/TinyLlama-1.1B-Chat-v1.0,
- Upstage SOLAR Instruct v1 (11B), Vicuna
- v1.5 (7B), WizardLM v1.2 (13B), chat-
- bison, chat-bison (PaLM 2), chat-
- bison-32k, chat-bison-32k (PaLM 2 32K),
- gemini-pro, gpt-3.5-turbo, gpt-4,
- gpt-4-turbo, llama-2-70b-chat,
- zephyr-7b-beta],
+ llama-2-70b-chat],
'categoryorder': 'array',
'domain': [0.0, 1.0],
- 'title': {'text': 'Model'}},
- 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
-})",Model endpoints response times,"This plot shows averaged response time for one query for each model.
-
-We used `time.time()` result saved to variable before making the call to API and compared it to `time.time()` result after receiving the results.
-
-It is also important to note that we used litellm python library for all of the models which adds some overhead compared to pure curl calls.","{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""01-ai Yi Chat (34B)""],""xaxis"":""x"",""y"":[null],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Chronos Hermes (13B)""],""xaxis"":""x"",""y"":[4.168874144554138],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Falcon Instruct (7B)""],""xaxis"":""x"",""y"":[1.0781378746032715],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""LLaMA-2 Chat (7B)""],""xaxis"":""x"",""y"":[1.686687707901001],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""xaxis"":""x"",""y"":[1.7709604501724243],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.271026806036631],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[4.835006467343727],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""OpenHermes-2.5-Mistral (7B)""],""xaxis"":""x"",""y"":[3.330963373184204],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Qwen 1.5 Chat (7B)""],""xaxis"":""x"",""y"":[null],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""RedPajama-INCITE Chat (7B)""],""xaxis"":""x"",""y"":[1.3475391864776611],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Snorkel Mistral PairRM DPO (7B)""],""xaxis"":""x"",""y"":[4.399647831916809],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.7927383464436197],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Upstage SOLAR Instruct v1 (11B)""],""xaxis"":""x"",""y"":[null],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Vicuna v1.5 (7B)""],""xaxis"":""x"",""y"":[null],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""WizardLM v1.2 (13B)""],""xaxis"":""x"",""y"":[3.350038528442383],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[2.668303858522159],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[6.738862129823485],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[3.036788886637026],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[4.293073908173212],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[14.323734385105828],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[15.535488395149827],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[3.6742314998615178],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.6370095004383316],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""01-ai Yi Chat (34B)"",""Chronos Hermes (13B)"",""Falcon Instruct (7B)"",""LLaMA-2 Chat (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""OpenHermes-2.5-Mistral (7B)"",""Qwen 1.5 Chat (7B)"",""RedPajama-INCITE Chat (7B)"",""Snorkel Mistral PairRM DPO (7B)"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""Upstage SOLAR Instruct v1 (11B)"",""Vicuna v1.5 (7B)"",""WizardLM v1.2 (13B)"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model endpoints response times""},""barmode"":""relative""}}","{""x"": ""model"", ""y"": ""execution_time"", ""color"": ""model"", ""title"": ""Model endpoints response times"", ""labels"": {""model"": ""Model"", ""execution_time"": ""Execution time (s)""}}",",model,execution_time
-0,01-ai Yi Chat (34B),
-1,Chronos Hermes (13B),4.168874144554138
-2,Falcon Instruct (7B),1.0781378746032715
-3,LLaMA-2 Chat (7B),1.686687707901001
-4,Mistral (7B) Instruct v0.2 (Together AI),1.7709604501724243
-5,Mistral-7B-Instruct-v0.2,3.271026806036631
-6,Mixtral-8x7B-Instruct-v0.1,4.835006467343727
-7,OpenHermes-2.5-Mistral (7B),3.330963373184204
-8,Qwen 1.5 Chat (7B),
-9,RedPajama-INCITE Chat (7B),1.3475391864776611
-10,Snorkel Mistral PairRM DPO (7B),4.399647831916809
-11,TinyLlama/TinyLlama-1.1B-Chat-v1.0,1.7927383464436197
-12,Upstage SOLAR Instruct v1 (11B),
-13,Vicuna v1.5 (7B),
-14,WizardLM v1.2 (13B),3.350038528442383
-15,chat-bison,3.890243631601334
-16,chat-bison (PaLM 2),2.668303858522159
-17,chat-bison-32k,4.128177767992019
-18,chat-bison-32k (PaLM 2 32K),6.738862129823485
-19,gemini-pro,3.036788886637026
-20,gpt-3.5-turbo,4.293073908173212
-21,gpt-4,14.323734385105828
-22,gpt-4-turbo,15.535488395149827
-23,llama-2-70b-chat,3.6742314998615178
-24,zephyr-7b-beta,3.6370095004383316
+ 'title': {'text': 'Model'}}}
+})",Model sizes in billions of parameters,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[70.0],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[56.0],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[13.0],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[11.0],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""zephyr-7b-beta""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Mistral-7B-Instruct-v0.2""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[1.1],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""RedPajama-INCITE Chat (7B)"",""Qwen 1.5 Chat (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""LLaMA-2 Chat (7B)"",""Vicuna v1.5 (7B)"",""Mistral-7B-Instruct-v0.2"",""zephyr-7b-beta"",""Upstage SOLAR Instruct v1 (11B)"",""WizardLM v1.2 (13B)"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}","{""x"": ""model_sizes"", ""y"": ""model"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Model sizes in billions of parameters"", ""labels"": {""model"": ""Model"", ""model_sizes"": ""Model size (B params)""}}",",model_sizes,model
+0,70.0,llama-2-70b-chat
+1,56.0,Mixtral-8x7B-Instruct-v0.1
+10,13.0,WizardLM v1.2 (13B)
+11,11.0,Upstage SOLAR Instruct v1 (11B)
+2,7.0,zephyr-7b-beta
+3,7.0,Mistral-7B-Instruct-v0.2
+5,7.0,Vicuna v1.5 (7B)
+6,7.0,LLaMA-2 Chat (7B)
+7,7.0,Mistral (7B) Instruct v0.2 (Together AI)
+8,7.0,Qwen 1.5 Chat (7B)
+9,7.0,RedPajama-INCITE Chat (7B)
+4,1.1,TinyLlama/TinyLlama-1.1B-Chat-v1.0
"
-./html/plots/execution_costs.html,"Figure({
+model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
'data': [{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'gpt-3.5-turbo',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'gpt-4-turbo',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'gpt-3.5-turbo',
- 'offsetgroup': 'gpt-3.5-turbo',
+ 'name': 'gpt-4-turbo',
+ 'offsetgroup': 'gpt-4-turbo',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.00105]),
+ 'x': array([128000]),
'xaxis': 'x',
- 'y': array(['gpt-3.5-turbo'], dtype=object),
+ 'y': array(['gpt-4-turbo'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'gpt-4-turbo',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'gpt-4-turbo',
- 'offsetgroup': 'gpt-4-turbo',
+ 'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.0125]),
+ 'x': array([32768]),
'xaxis': 'x',
- 'y': array(['gpt-4-turbo'], dtype=object),
+ 'y': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'gpt-4',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'gpt-4',
- 'offsetgroup': 'gpt-4',
+ 'name': 'Snorkel Mistral PairRM DPO (7B)',
+ 'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.0315]),
+ 'x': array([32768]),
'xaxis': 'x',
- 'y': array(['gpt-4'], dtype=object),
+ 'y': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'llama-2-70b-chat',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Qwen 1.5 Chat (7B)',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'llama-2-70b-chat',
- 'offsetgroup': 'llama-2-70b-chat',
+ 'name': 'Qwen 1.5 Chat (7B)',
+ 'offsetgroup': 'Qwen 1.5 Chat (7B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.002529]),
+ 'x': array([32768]),
'xaxis': 'x',
- 'y': array(['llama-2-70b-chat'], dtype=object),
+ 'y': array(['Qwen 1.5 Chat (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'gpt-4',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'Mixtral-8x7B-Instruct-v0.1',
- 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'name': 'gpt-4',
+ 'offsetgroup': 'gpt-4',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.000855]),
+ 'x': array([32000]),
'xaxis': 'x',
- 'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+ 'y': array(['gpt-4'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'chat-bison (PaLM 2)',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
- 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'name': 'chat-bison (PaLM 2)',
+ 'offsetgroup': 'chat-bison (PaLM 2)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.00031001]),
+ 'x': array([8196]),
'xaxis': 'x',
- 'y': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'y': array(['chat-bison (PaLM 2)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'gemini-pro',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
- 'name': 'gemini-pro',
- 'offsetgroup': 'gemini-pro',
+ 'name': 'OpenHermes-2.5-Mistral (7B)',
+ 'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.0002825]),
+ 'x': array([8192]),
'xaxis': 'x',
- 'y': array(['gemini-pro'], dtype=object),
+ 'y': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'chat-bison (PaLM 2)',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'gpt-3.5-turbo',
'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
- 'name': 'chat-bison (PaLM 2)',
- 'offsetgroup': 'chat-bison (PaLM 2)',
+ 'name': 'gpt-3.5-turbo',
+ 'offsetgroup': 'gpt-3.5-turbo',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.0002825]),
+ 'x': array([4096]),
'xaxis': 'x',
- 'y': array(['chat-bison (PaLM 2)'], dtype=object),
+ 'y': array(['gpt-3.5-turbo'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'WizardLM v1.2 (13B)',
'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k (PaLM 2 32K)',
- 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'name': 'WizardLM v1.2 (13B)',
+ 'offsetgroup': 'WizardLM v1.2 (13B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.0002825]),
+ 'x': array([4096]),
'xaxis': 'x',
- 'y': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
- 'yaxis': 'y'}],
- 'layout': {'barmode': 'relative',
- 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
- 'template': '...',
- 'title': {'text': 'Costs of execution of 20 test queries per model'},
- 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}},
- 'yaxis': {'anchor': 'x',
- 'categoryarray': [chat-bison-32k (PaLM 2 32K), chat-bison
- (PaLM 2), gemini-pro,
- TinyLlama/TinyLlama-1.1B-Chat-v1.0,
- Mixtral-8x7B-Instruct-v0.1,
- llama-2-70b-chat, gpt-4, gpt-4-turbo,
- gpt-3.5-turbo],
- 'categoryorder': 'array',
- 'domain': [0.0, 1.0],
- 'title': {'text': 'Model'}}}
-})",Costs of execution of 20 test queries per model,"Costs of experiments carried out to create this table, 20 test queries per model.
-
-Note that pause and resume time cost was not included in the cost calculation for the models billed hourly.
-
-Also although Gemini-Pro model is free for under 60 queries per minute we calculated the full cost for generating the tokens.","{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.00105],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0125],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0315],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.002529],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.000855],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0003100131352742513],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0002825],""xaxis"":""x"",""y"":[""gemini-pro""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0002825],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0002825],""xaxis"":""x"",""y"":[""chat-bison-32k (PaLM 2 32K)""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""chat-bison-32k (PaLM 2 32K)"",""chat-bison (PaLM 2)"",""gemini-pro"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat"",""gpt-4"",""gpt-4-turbo"",""gpt-3.5-turbo""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 20 test queries per model""},""barmode"":""relative""}}","{""y"": ""model"", ""x"": ""model_query_costs"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Costs of execution of 20 test queries per model"", ""labels"": {""model"": ""Model"", ""model_query_costs"": ""Execution cost ($)""}}",",model_query_costs,model
-0,0.00105,gpt-3.5-turbo
-1,0.0125,gpt-4-turbo
-2,0.0315,gpt-4
-3,0.002529,llama-2-70b-chat
-4,0.000855,Mixtral-8x7B-Instruct-v0.1
-7,0.0003100131352742513,TinyLlama/TinyLlama-1.1B-Chat-v1.0
-8,0.0002825,gemini-pro
-9,0.0002825,chat-bison (PaLM 2)
-10,0.0002825,chat-bison-32k (PaLM 2 32K)
-"
-./html/plots/model_sizes.html,"Figure({
- 'data': [{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
- 'legendgroup': 'llama-2-70b-chat',
+ 'y': array(['WizardLM v1.2 (13B)'], dtype=object),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'LLaMA-2 Chat (7B)',
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+ 'name': 'LLaMA-2 Chat (7B)',
+ 'offsetgroup': 'LLaMA-2 Chat (7B)',
+ 'orientation': 'h',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array([4096]),
+ 'xaxis': 'x',
+ 'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Vicuna v1.5 (7B)',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'llama-2-70b-chat',
- 'offsetgroup': 'llama-2-70b-chat',
+ 'name': 'Vicuna v1.5 (7B)',
+ 'offsetgroup': 'Vicuna v1.5 (7B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([70.]),
+ 'x': array([4096]),
'xaxis': 'x',
- 'y': array(['llama-2-70b-chat'], dtype=object),
+ 'y': array(['Vicuna v1.5 (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
- 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': '01-ai Yi Chat (34B)',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'Mixtral-8x7B-Instruct-v0.1',
- 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'name': '01-ai Yi Chat (34B)',
+ 'offsetgroup': '01-ai Yi Chat (34B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([56.]),
+ 'x': array([4096]),
'xaxis': 'x',
- 'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+ 'y': array(['01-ai Yi Chat (34B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
- 'legendgroup': 'zephyr-7b-beta',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'zephyr-7b-beta',
- 'offsetgroup': 'zephyr-7b-beta',
+ 'name': 'Upstage SOLAR Instruct v1 (11B)',
+ 'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([7.]),
+ 'x': array([4096]),
'xaxis': 'x',
- 'y': array(['zephyr-7b-beta'], dtype=object),
+ 'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
- 'legendgroup': 'Mistral-7B-Instruct-v0.2',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Chronos Hermes (13B)',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'Mistral-7B-Instruct-v0.2',
- 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+ 'name': 'Chronos Hermes (13B)',
+ 'offsetgroup': 'Chronos Hermes (13B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([7.]),
+ 'x': array([2048]),
'xaxis': 'x',
- 'y': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+ 'y': array(['Chronos Hermes (13B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
- 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Falcon Instruct (7B)',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'name': 'Falcon Instruct (7B)',
+ 'offsetgroup': 'Falcon Instruct (7B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([1.1]),
+ 'x': array([2048]),
'xaxis': 'x',
- 'y': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'y': array(['Falcon Instruct (7B)'], dtype=object),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'RedPajama-INCITE Chat (7B)',
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+ 'name': 'RedPajama-INCITE Chat (7B)',
+ 'offsetgroup': 'RedPajama-INCITE Chat (7B)',
+ 'orientation': 'h',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array([2048]),
+ 'xaxis': 'x',
+ 'y': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'llama-2-70b-chat',
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+ 'name': 'llama-2-70b-chat',
+ 'offsetgroup': 'llama-2-70b-chat',
+ 'orientation': 'h',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array([2048]),
+ 'xaxis': 'x',
+ 'y': array(['llama-2-70b-chat'], dtype=object),
'yaxis': 'y'}],
'layout': {'barmode': 'relative',
'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
'template': '...',
- 'title': {'text': 'Model sizes in billions of parameters'},
- 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}},
+ 'title': {'text': 'Model input sizes in tokens'},
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model input size (tokens)'}},
'yaxis': {'anchor': 'x',
- 'categoryarray': [TinyLlama/TinyLlama-1.1B-Chat-v1.0,
- Mistral-7B-Instruct-v0.2,
- zephyr-7b-beta,
- Mixtral-8x7B-Instruct-v0.1,
- llama-2-70b-chat],
+ 'categoryarray': [llama-2-70b-chat, RedPajama-INCITE Chat
+ (7B), Falcon Instruct (7B), Chronos
+ Hermes (13B), Upstage SOLAR Instruct v1
+ (11B), 01-ai Yi Chat (34B), Vicuna v1.5
+ (7B), LLaMA-2 Chat (7B), WizardLM v1.2
+ (13B), gpt-3.5-turbo,
+ OpenHermes-2.5-Mistral (7B), chat-bison
+ (PaLM 2), gpt-4, Qwen 1.5 Chat (7B),
+ Snorkel Mistral PairRM DPO (7B), Mistral
+ (7B) Instruct v0.2 (Together AI),
+ gpt-4-turbo],
'categoryorder': 'array',
'domain': [0.0, 1.0],
'title': {'text': 'Model'}}}
-})",Model sizes in billions of parameters,"Model sizes based on release blogs and documentation of their respective developers.
-
-Note that models, which sizes were not disclosed publicly, were omitted from this comparison.","{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[70.0],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[56.0],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""zephyr-7b-beta""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Mistral-7B-Instruct-v0.2""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[1.1],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""Mistral-7B-Instruct-v0.2"",""zephyr-7b-beta"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}","{""x"": ""model_sizes"", ""y"": ""model"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Model sizes in billions of parameters"", ""labels"": {""model"": ""Model"", ""model_sizes"": ""Model size (B params)""}}",",model_sizes,model
-0,70.0,llama-2-70b-chat
-1,56.0,Mixtral-8x7B-Instruct-v0.1
-2,7.0,zephyr-7b-beta
-3,7.0,Mistral-7B-Instruct-v0.2
-4,1.1,TinyLlama/TinyLlama-1.1B-Chat-v1.0
+})",Model input sizes in tokens,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[128000],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Snorkel Mistral PairRM DPO (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32000],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[8196],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[8192],""xaxis"":""x"",""y"":[""OpenHermes-2.5-Mistral (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""01-ai Yi Chat (34B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""Chronos Hermes (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""Falcon Instruct (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model input size (tokens)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""llama-2-70b-chat"",""RedPajama-INCITE Chat (7B)"",""Falcon Instruct (7B)"",""Chronos Hermes (13B)"",""Upstage SOLAR Instruct v1 (11B)"",""01-ai Yi Chat (34B)"",""Vicuna v1.5 (7B)"",""LLaMA-2 Chat (7B)"",""WizardLM v1.2 (13B)"",""gpt-3.5-turbo"",""OpenHermes-2.5-Mistral (7B)"",""chat-bison (PaLM 2)"",""gpt-4"",""Qwen 1.5 Chat (7B)"",""Snorkel Mistral PairRM DPO (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""gpt-4-turbo""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model input sizes in tokens""},""barmode"":""relative""}}","{""x"": ""model_input_sizes"", ""y"": ""model"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Model input sizes in tokens"", ""labels"": {""model"": ""Model"", ""model_input_sizes"": ""Model input size (tokens)""}}",",model_input_sizes,model
+1,128000,gpt-4-turbo
+9,32768,Mistral (7B) Instruct v0.2 (Together AI)
+11,32768,Snorkel Mistral PairRM DPO (7B)
+10,32768,Qwen 1.5 Chat (7B)
+2,32000,gpt-4
+4,8196,chat-bison (PaLM 2)
+12,8192,OpenHermes-2.5-Mistral (7B)
+0,4096,gpt-3.5-turbo
+15,4096,WizardLM v1.2 (13B)
+8,4096,LLaMA-2 Chat (7B)
+7,4096,Vicuna v1.5 (7B)
+5,4096,01-ai Yi Chat (34B)
+16,4096,Upstage SOLAR Instruct v1 (11B)
+6,2048,Chronos Hermes (13B)
+13,2048,Falcon Instruct (7B)
+14,2048,RedPajama-INCITE Chat (7B)
+3,2048,llama-2-70b-chat
"
diff --git a/data/output_plots.csv b/data/output_plots.csv
index 69974f7b537906b0afd3e7ff83328d88c0e1138a..43b16a8f0fec56be3c0866834a72074667d91970 100644
--- a/data/output_plots.csv
+++ b/data/output_plots.csv
@@ -1,4 +1,4 @@
-plot_object,header,plot_json,df,arguments
+plot_object,header,plot_json,df,arguments,description
"Figure({
'data': [{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
@@ -213,10 +213,14 @@ plot_object,header,plot_json,df,arguments
10,gpt-4-turbo,15.852052640759334
11,llama-2-70b-chat,6.065365233652719
12,zephyr-7b-beta,3.389894526934474
-","{""x"": ""model"", ""y"": ""execution_time"", ""color"": ""model"", ""labels"": {""model"": ""Model"", ""execution_time"": ""Execution time (s)""}, ""title"": ""Plot of execution time (s)""}"
+","{""x"": ""model"", ""y"": ""execution_time"", ""color"": ""model"", ""labels"": {""model"": ""Model"", ""execution_time"": ""Execution time (s)""}, ""title"": ""Plot of execution time (s)""}","This plot shows averaged response time for one query for each model.
+
+We used `time.time()` result saved to variable before making the call to API and compared it to `time.time()` result after receiving the results.
+
+It is also important to note that we used litellm python library for all of the models which adds some overhead compared to pure curl calls."
"Figure({
'data': [{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'Mistral-7B-Instruct-v0.2',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
'name': 'Mistral-7B-Instruct-v0.2',
@@ -230,7 +234,7 @@ plot_object,header,plot_json,df,arguments
'y': array([29.32127976]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
'name': 'Mixtral-8x7B-Instruct-v0.1',
@@ -244,7 +248,7 @@ plot_object,header,plot_json,df,arguments
'y': array([54.6203696]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
@@ -258,7 +262,7 @@ plot_object,header,plot_json,df,arguments
'y': array([32.55470546]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'chat-bison',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
'name': 'chat-bison',
@@ -272,7 +276,7 @@ plot_object,header,plot_json,df,arguments
'y': array([49.]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'chat-bison (PaLM 2)',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
'name': 'chat-bison (PaLM 2)',
@@ -286,7 +290,7 @@ plot_object,header,plot_json,df,arguments
'y': array([40.9371875]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'chat-bison-32k',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
'name': 'chat-bison-32k',
@@ -300,7 +304,7 @@ plot_object,header,plot_json,df,arguments
'y': array([48.025]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
'name': 'chat-bison-32k (PaLM 2 32K)',
@@ -314,7 +318,7 @@ plot_object,header,plot_json,df,arguments
'y': array([38.72477679]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'gemini-pro',
'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
'name': 'gemini-pro',
@@ -328,7 +332,7 @@ plot_object,header,plot_json,df,arguments
'y': array([39.69346182]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'gpt-3.5-turbo',
'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
'name': 'gpt-3.5-turbo',
@@ -342,7 +346,7 @@ plot_object,header,plot_json,df,arguments
'y': array([39.25851284]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'gpt-4',
'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
'name': 'gpt-4',
@@ -356,7 +360,7 @@ plot_object,header,plot_json,df,arguments
'y': array([32.90737496]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'gpt-4-turbo',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
'name': 'gpt-4-turbo',
@@ -370,7 +374,7 @@ plot_object,header,plot_json,df,arguments
'y': array([38.36012473]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'llama-2-70b-chat',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
'name': 'llama-2-70b-chat',
@@ -384,7 +388,7 @@ plot_object,header,plot_json,df,arguments
'y': array([95.35066814]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Words count=%{y}',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'zephyr-7b-beta',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
'name': 'zephyr-7b-beta',
@@ -400,7 +404,7 @@ plot_object,header,plot_json,df,arguments
'layout': {'barmode': 'relative',
'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
'template': '...',
- 'title': {'text': 'Plot of words count'},
+ 'title': {'text': 'Plot of output words count'},
'xaxis': {'anchor': 'y',
'categoryarray': [Mistral-7B-Instruct-v0.2,
Mixtral-8x7B-Instruct-v0.1,
@@ -413,8 +417,8 @@ plot_object,header,plot_json,df,arguments
'categoryorder': 'array',
'domain': [0.0, 1.0],
'title': {'text': 'Model'}},
- 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Words count'}}}
-})",Plot of words count,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[29.321279761904762],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[54.620369597277694],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[32.55470545977011],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[49.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[40.9371875],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[48.025],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[38.72477678571428],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[39.69346182236394],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[39.25851283971182],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[32.907374959094994],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[38.36012473477448],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[95.35066814260509],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eWords count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[31.159257075471697],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Words count""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Plot of words count""},""barmode"":""relative""}}",",model,words_count
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Output words count'}}}
+})",Plot of output words count,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[29.321279761904762],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[54.620369597277694],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[32.55470545977011],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[49.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[40.9371875],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[48.025],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[38.72477678571428],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[39.69346182236394],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[39.25851283971182],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[32.907374959094994],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[38.36012473477448],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[95.35066814260509],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[31.159257075471697],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Output words count""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Plot of output words count""},""barmode"":""relative""}}",",model,words_count
0,Mistral-7B-Instruct-v0.2,29.321279761904762
1,Mixtral-8x7B-Instruct-v0.1,54.620369597277694
2,TinyLlama/TinyLlama-1.1B-Chat-v1.0,32.55470545977011
@@ -428,10 +432,10 @@ plot_object,header,plot_json,df,arguments
10,gpt-4-turbo,38.36012473477448
11,llama-2-70b-chat,95.35066814260509
12,zephyr-7b-beta,31.159257075471697
-","{""x"": ""model"", ""y"": ""words_count"", ""color"": ""model"", ""labels"": {""model"": ""Model"", ""words_count"": ""Words count""}, ""title"": ""Plot of words count""}"
+","{""x"": ""model"", ""y"": ""words_count"", ""color"": ""model"", ""labels"": {""model"": ""Model"", ""words_count"": ""Output words count""}, ""title"": ""Plot of output words count""}",
"Figure({
'data': [{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'Mistral-7B-Instruct-v0.2',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
'name': 'Mistral-7B-Instruct-v0.2',
@@ -445,7 +449,7 @@ plot_object,header,plot_json,df,arguments
'y': array([2.69413923]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
'name': 'Mixtral-8x7B-Instruct-v0.1',
@@ -459,7 +463,7 @@ plot_object,header,plot_json,df,arguments
'y': array([2.95690172]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
@@ -473,7 +477,7 @@ plot_object,header,plot_json,df,arguments
'y': array([3.14663951]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'chat-bison',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
'name': 'chat-bison',
@@ -487,7 +491,7 @@ plot_object,header,plot_json,df,arguments
'y': array([61.46100879]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'chat-bison (PaLM 2)',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
'name': 'chat-bison (PaLM 2)',
@@ -501,7 +505,7 @@ plot_object,header,plot_json,df,arguments
'y': array([57.12081731]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'chat-bison-32k',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
'name': 'chat-bison-32k',
@@ -515,7 +519,7 @@ plot_object,header,plot_json,df,arguments
'y': array([62.01633273]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
'name': 'chat-bison-32k (PaLM 2 32K)',
@@ -529,7 +533,7 @@ plot_object,header,plot_json,df,arguments
'y': array([56.57639874]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'gemini-pro',
'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
'name': 'gemini-pro',
@@ -543,7 +547,7 @@ plot_object,header,plot_json,df,arguments
'y': array([142.50301526]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'gpt-3.5-turbo',
'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
'name': 'gpt-3.5-turbo',
@@ -557,7 +561,7 @@ plot_object,header,plot_json,df,arguments
'y': array([3.04399416]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'gpt-4',
'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
'name': 'gpt-4',
@@ -571,7 +575,7 @@ plot_object,header,plot_json,df,arguments
'y': array([3.01875517]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'gpt-4-turbo',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
'name': 'gpt-4-turbo',
@@ -585,7 +589,7 @@ plot_object,header,plot_json,df,arguments
'y': array([3.0126559]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'llama-2-70b-chat',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
'name': 'llama-2-70b-chat',
@@ -599,7 +603,7 @@ plot_object,header,plot_json,df,arguments
'y': array([3.20032418]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (in characters)=%{y}',
'legendgroup': 'zephyr-7b-beta',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
'name': 'zephyr-7b-beta',
@@ -615,7 +619,7 @@ plot_object,header,plot_json,df,arguments
'layout': {'barmode': 'relative',
'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
'template': '...',
- 'title': {'text': 'Plot of chunks mean size (characters)'},
+ 'title': {'text': 'Plot of chunks mean size (in characters)'},
'xaxis': {'anchor': 'y',
'categoryarray': [Mistral-7B-Instruct-v0.2,
Mixtral-8x7B-Instruct-v0.1,
@@ -628,8 +632,8 @@ plot_object,header,plot_json,df,arguments
'categoryorder': 'array',
'domain': [0.0, 1.0],
'title': {'text': 'Model'}},
- 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Chunks mean size (characters)'}}}
-})",Plot of chunks mean size (characters),"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[2.6941392346739055],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[2.9569017240855695],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[3.146639511119026],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[61.46100879376141],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[57.12081730998353],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[62.01633272616879],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[56.57639873825073],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[142.5030152620613],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[3.0439941593593898],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[3.018755174356351],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[3.0126559039321035],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[3.2003241836714382],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[2.711003271075019],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Chunks mean size (characters)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Plot of chunks mean size (characters)""},""barmode"":""relative""}}",",model,chunk_sizes_mean
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Chunks mean size (in characters)'}}}
+})",Plot of chunks mean size (in characters),"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[2.6941392346739055],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[2.9569017240855695],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[3.146639511119026],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[61.46100879376141],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[57.12081730998353],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[62.01633272616879],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[56.57639873825073],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[142.5030152620613],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[3.0439941593593898],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[3.018755174356351],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[3.0126559039321035],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[3.2003241836714382],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (in characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[2.711003271075019],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Chunks mean size (in characters)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Plot of chunks mean size (in characters)""},""barmode"":""relative""}}",",model,chunk_sizes_mean
0,Mistral-7B-Instruct-v0.2,2.6941392346739055
1,Mixtral-8x7B-Instruct-v0.1,2.9569017240855695
2,TinyLlama/TinyLlama-1.1B-Chat-v1.0,3.146639511119026
@@ -643,4 +647,4 @@ plot_object,header,plot_json,df,arguments
10,gpt-4-turbo,3.0126559039321035
11,llama-2-70b-chat,3.2003241836714382
12,zephyr-7b-beta,2.711003271075019
-","{""x"": ""model"", ""y"": ""chunk_sizes_mean"", ""color"": ""model"", ""labels"": {""model"": ""Model"", ""chunk_sizes_mean"": ""Chunks mean size (characters)""}, ""title"": ""Plot of chunks mean size (characters)""}"
+","{""x"": ""model"", ""y"": ""chunk_sizes_mean"", ""color"": ""model"", ""labels"": {""model"": ""Model"", ""chunk_sizes_mean"": ""Chunks mean size (in characters)""}, ""title"": ""Plot of chunks mean size (in characters)""}",
diff --git a/data/time_of_day_plots.csv b/data/time_of_day_plots.csv
index 78ea26378a6308eb82ac334bb3e7e40cfda05641..f1c095fd35601f0a9a867bb63c855a60e28ac14d 100644
--- a/data/time_of_day_plots.csv
+++ b/data/time_of_day_plots.csv
@@ -430,6 +430,45 @@ plot_object,header,plot_json,df,arguments
184,zephyr-7b-beta,midnight,3.7253047794103624,217.82083333333333,33.67916666666667,Hugging Face Inference Endpoint
185,zephyr-7b-beta,night,3.6954557319482166,216.55833333333334,33.46666666666667,Hugging Face Inference Endpoint
","{""x"": ""time_of_day"", ""y"": ""execution_time"", ""color"": ""model"", ""barmode"": ""group"", ""category_orders"": {""time_of_day"": [""early morning"", ""morning"", ""afternoon"", ""late afternoon"", ""evening"", ""late evening"", ""midnight"", ""night""]}, ""labels"": {""time_of_day"": ""Time of day"", ""execution_time"": ""Execution Time (s)"", ""model"": ""Model""}, ""title"": ""Execution time for different times of the day""}"
+"Figure({
+ 'data': [{'hovertemplate': ('Provider=Hugging Face Inferenc' ... 'n Time (s)=%{y}'),
+ 'legendgroup': 'Hugging Face Inference Endpoint',
+ 'line': {'color': '#636efa', 'dash': 'solid'},
+ 'marker': {'symbol': 'circle'},
+ 'mode': 'lines+markers',
+ 'name': 'Hugging Face Inference Endpoint',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'type': 'scatter',
+ 'x': array(['morning', 'afternoon', 'late afternoon', 'evening'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([3.734026 , 3.22897372, 3.20486431, 3.39764007]),
+ 'yaxis': 'y'},
+ {'hovertemplate': ('Provider=Together AI
Time o' ... 'n Time (s)=%{y}'),
+ 'legendgroup': 'Together AI',
+ 'line': {'color': '#EF553B', 'dash': 'solid'},
+ 'marker': {'symbol': 'circle'},
+ 'mode': 'lines+markers',
+ 'name': 'Together AI',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'type': 'scatter',
+ 'x': array(['morning', 'afternoon', 'late afternoon', 'evening'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([ 1.81441077, 3.43125006, 3.40168245, 11.98982314]),
+ 'yaxis': 'y'}],
+ 'layout': {'legend': {'title': {'text': 'Provider'}, 'tracegroupgap': 0},
+ 'template': '...',
+ 'title': {'text': 'Execution time for different times of the day for different providers'},
+ 'xaxis': {'anchor': 'y',
+ 'categoryarray': [early morning, morning, afternoon, late
+ afternoon, evening, late evening,
+ midnight, night],
+ 'categoryorder': 'array',
+ 'domain': [0.0, 1.0],
+ 'title': {'text': 'Time of day'}},
+ 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution Time (s)'}}}
+})",Time of the day plot for different providers,"{""data"":[{""hovertemplate"":""Provider=Hugging Face Inference Endpoint\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Hugging Face Inference Endpoint"",""line"":{""color"":""#636efa"",""dash"":""solid""},""marker"":{""symbol"":""circle""},""mode"":""lines+markers"",""name"":""Hugging Face Inference Endpoint"",""orientation"":""v"",""showlegend"":true,""x"":[""morning"",""afternoon"",""late afternoon"",""evening""],""xaxis"":""x"",""y"":[3.734026002883911,3.228973722457886,3.2048643112182615,3.397640073299408],""yaxis"":""y"",""type"":""scatter""},{""hovertemplate"":""Provider=Together AI\u003cbr\u003eTime of day=%{x}\u003cbr\u003eExecution Time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Together AI"",""line"":{""color"":""#EF553B"",""dash"":""solid""},""marker"":{""symbol"":""circle""},""mode"":""lines+markers"",""name"":""Together AI"",""orientation"":""v"",""showlegend"":true,""x"":[""morning"",""afternoon"",""late afternoon"",""evening""],""xaxis"":""x"",""y"":[1.8144107659657795,3.4312500613076344,3.4016824527220293,11.989823137010847],""yaxis"":""y"",""type"":""scatter""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Time of day""},""categoryorder"":""array"",""categoryarray"":[""early morning"",""morning"",""afternoon"",""late afternoon"",""evening"",""late evening"",""midnight"",""night""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution Time (s)""}},""legend"":{""title"":{""text"":""Provider""},""tracegroupgap"":0},""title"":{""text"":""Execution time for different times of the day for different providers""}}}",,
"Figure({
'data': [{'hovertemplate': 'Time of day=%{x}
Execution Time (s)=%{y}',
'legendgroup': '',
diff --git a/pipeline/models.py b/pipeline/models.py
index 38c660a5b301af0ece80d52d22118cd5397bc686..3b494a7d9a41bc47c0598b0dae30ba823af1bfae 100644
--- a/pipeline/models.py
+++ b/pipeline/models.py
@@ -190,7 +190,7 @@ MODELS = [
"Together AI",
input_size=4096,
),
- Model("Vicuna v1.5 (7B)", "together_ai/lmsys/vicuna-7b-v1.5", None, "Together AI", input_size=4096, selected=True),
+ Model("Vicuna v1.5 (7B)", "together_ai/lmsys/vicuna-7b-v1.5", None, "Together AI", input_size=4096, size_billion_parameters=7, selected=True),
Model(
"Code Llama Instruct (13B)",
"together_ai/codellama/CodeLlama-13b-Instruct-hf",
@@ -217,6 +217,7 @@ MODELS = [
"together_ai/codellama/CodeLlama-7b-Instruct-hf",
None,
"Together AI",
+ size_billion_parameters=7,
input_size=16384,
),
Model(
@@ -224,6 +225,7 @@ MODELS = [
"together_ai/togethercomputer/llama-2-13b-chat",
None,
"Together AI",
+ size_billion_parameters=13,
input_size=4096,
),
Model(
@@ -231,6 +233,7 @@ MODELS = [
"together_ai/togethercomputer/llama-2-70b-chat",
None,
"Together AI",
+ size_billion_parameters=70,
input_size=4096,
),
Model(
@@ -238,6 +241,7 @@ MODELS = [
"together_ai/togethercomputer/llama-2-7b-chat",
None,
"Together AI",
+ size_billion_parameters=7,
input_size=4096,
# selected=True,
),
@@ -246,6 +250,7 @@ MODELS = [
"together_ai/mistralai/Mistral-7B-Instruct-v0.1",
None,
"Together AI",
+ size_billion_parameters=7,
input_size=4096,
),
Model(
@@ -253,6 +258,7 @@ MODELS = [
"together_ai/mistralai/Mistral-7B-Instruct-v0.2",
None,
"Together AI",
+ size_billion_parameters=7,
input_size=32768,
selected=True,
),
@@ -261,6 +267,7 @@ MODELS = [
"together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1",
None,
"Together AI",
+ size_billion_parameters=8*7,
input_size=32768,
),
Model(
@@ -268,6 +275,7 @@ MODELS = [
"together_ai/NousResearch/Nous-Capybara-7B-V1p9",
None,
"Together AI",
+ size_billion_parameters=7,
input_size=8192,
),
Model(
@@ -275,6 +283,7 @@ MODELS = [
"together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
None,
"Together AI",
+ size_billion_parameters=8*7,
input_size=32768,
),
Model(
@@ -282,6 +291,7 @@ MODELS = [
"together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
None,
"Together AI",
+ size_billion_parameters=8*7,
input_size=32768,
),
Model(
@@ -289,6 +299,7 @@ MODELS = [
"together_ai/NousResearch/Nous-Hermes-llama-2-7b",
None,
"Together AI",
+ size_billion_parameters=7,
input_size=4096,
),
Model(
@@ -296,6 +307,7 @@ MODELS = [
"together_ai/NousResearch/Nous-Hermes-Llama2-13b",
None,
"Together AI",
+ size_billion_parameters=13,
input_size=4096,
),
Model(
@@ -303,6 +315,7 @@ MODELS = [
"together_ai/NousResearch/Nous-Hermes-2-Yi-34B",
None,
"Together AI",
+ size_billion_parameters=34,
input_size=4096,
),
Model(
@@ -310,6 +323,7 @@ MODELS = [
"together_ai/openchat/openchat-3.5-1210",
None,
"Together AI",
+ size_billion_parameters=7,
input_size=8192,
),
Model(
@@ -317,6 +331,7 @@ MODELS = [
"together_ai/Open-Orca/Mistral-7B-OpenOrca",
None,
"Together AI",
+ size_billion_parameters=7,
input_size=8192,
),
Model(
@@ -324,6 +339,7 @@ MODELS = [
"together_ai/togethercomputer/Qwen-7B-Chat",
None,
"Together AI",
+ size_billion_parameters=7,
input_size=8192,
),
Model(
@@ -331,6 +347,7 @@ MODELS = [
"together_ai/Qwen/Qwen1.5-0.5B-Chat",
None,
"Together AI",
+ size_billion_parameters=0.5,
input_size=32768,
),
Model(
@@ -339,16 +356,18 @@ MODELS = [
None,
"Together AI",
input_size=32768,
+ size_billion_parameters=1.8,
),
Model(
"Qwen 1.5 Chat (4B)",
"together_ai/Qwen/Qwen1.5-4B-Chat",
None,
"Together AI",
+ size_billion_parameters=4,
input_size=32768,
),
Model(
- "Qwen 1.5 Chat (7B)", "together_ai/Qwen/Qwen1.5-7B-Chat", None, "Together AI", input_size=32768,
+ "Qwen 1.5 Chat (7B)", "together_ai/Qwen/Qwen1.5-7B-Chat", None, "Together AI", input_size=32768, size_billion_parameters=7
# selected=True
),
Model(
@@ -356,6 +375,7 @@ MODELS = [
"together_ai/Qwen/Qwen1.5-14B-Chat",
None,
"Together AI",
+ size_billion_parameters=14,
input_size=32768,
),
Model(
@@ -422,6 +442,7 @@ MODELS = [
"together_ai/togethercomputer/RedPajama-INCITE-Chat-3B-v1",
None,
"Together AI",
+ size_billion_parameters=3,
input_size=2048,
),
Model(
@@ -430,6 +451,7 @@ MODELS = [
None,
"Together AI",
input_size=2048,
+ size_billion_parameters=7,
# selected=True,
),
Model(
@@ -438,6 +460,7 @@ MODELS = [
None,
"Together AI",
input_size=32768,
+ size_billion_parameters=7,
),
Model(
"ReMM SLERP L2 (13B)",
@@ -445,6 +468,7 @@ MODELS = [
None,
"Together AI",
input_size=4096,
+ size_billion_parameters=13,
),
Model(
"Toppy M (7B)",
@@ -452,6 +476,7 @@ MODELS = [
None,
"Together AI",
input_size=4096,
+ size_billion_parameters=7
),
Model(
"WizardLM v1.2 (13B)",
@@ -459,6 +484,7 @@ MODELS = [
None,
"Together AI",
input_size=4096,
+ size_billion_parameters=13,
# selected=True,
),
Model(
@@ -467,6 +493,7 @@ MODELS = [
None,
"Together AI",
input_size=4096,
+ size_billion_parameters=11,
# selected=True,
),
]