interactive backend and benchmark

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+runs

app.py

@@ -1,136 +1,28 @@
 import random
-import subprocess
 import gradio as gr
-from ansi2html import Ansi2HTMLConverter
 from optimum_benchmark.task_utils import (
     TASKS_TO_AUTOMODELS,
     infer_task_from_model_name_or_path,
 )
+from run import run_benchmark
+from configs import (
+    get_training_config,
+    get_inference_config,
+    get_neural_compressor_config,
+    get_onnxruntime_config,
+    get_openvino_config,
+    get_pytorch_config,
+)
 
-
-def get_backend_config():
-    return [
-        # seed
-        gr.Textbox(label="backend.seed", value=42),
-        # inter_op_num_threads
-        gr.Textbox(
-            label="backend.inter_op_num_threads",
-            value=None,
-            placeholder=None,
-        ),
-        # intra_op_num_threads
-        gr.Textbox(
-            label="backend.intra_op_num_threads",
-            value=None,
-            placeholder=None,
-        ),
-        # initial_isolation_check
-        gr.Checkbox(label="backend.initial_isolation_check", value=True),
-        # continous_isolation_check
-        gr.Checkbox(label="backend.continous_isolation_check", value=True),
-        # delete_cache
-        gr.Checkbox(label="backend.delete_cache", value=False),
-    ]
-
-
-def get_inference_config():
-    return [
-        # duration
-        gr.Textbox(label="benchmark.duration", value=10),
-        # warmup runs
-        gr.Textbox(label="benchmark.warmup_runs", value=1),
-    ]
-
-
-def get_pytorch_config():
-    return [
-        # no_weights
-        gr.Checkbox(label="backend.no_weights"),
-        # device_map
-        gr.Dropdown(["auto", "sequential"], label="backend.device_map"),
-        # torch_dtype
-        gr.Dropdown(
-            ["bfloat16", "float16", "float32", "auto"],
-            label="backend.torch_dtype",
-        ),
-        # disable_grad
-        gr.Checkbox(label="backend.disable_grad"),
-        # eval_mode
-        gr.Checkbox(label="backend.eval_mode"),
-        # amp_autocast
-        gr.Checkbox(label="backend.amp_autocast"),
-        # amp_dtype
-        gr.Dropdown(["bfloat16", "float16"], label="backend.amp_dtype"),
-        # torch_compile
-        gr.Checkbox(label="backend.torch_compile"),
-        # bettertransformer
-        gr.Checkbox(label="backend.bettertransformer"),
-        # quantization_scheme
-        gr.Dropdown(["gptq", "bnb"], label="backend.quantization_scheme"),
-        # use_ddp
-        gr.Checkbox(label="backend.use_ddp"),
-        # peft_strategy
-        gr.Textbox(label="backend.peft_strategy"),
-    ]
-
-
-conv = Ansi2HTMLConverter()
-
-
-def run_experiment(kwargs):
-    arguments = [
-        "optimum-benchmark",
-        "--config-dir",
-        "./",
-        "--config-name",
-        "base_config",
-    ]
-    for key, value in kwargs.items():
-        arguments.append(f"{key.label}={value if value != '' else 'null'}")
-
-    # stream subprocess output
-    process = subprocess.Popen(
-        arguments,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        universal_newlines=True,
-    )
-
-    ansi_text = ""
-    for ansi_line in iter(process.stdout.readline, ""):
-        # stream process output
-        print(ansi_line, end="")
-        # append line to ansi text
-        ansi_text += ansi_line
-        # convert ansi to html
-        html_text = conv.convert(ansi_text)
-        # extract style from html
-        style = html_text.split('<style type="text/css">')[1].split("</style>")[0]
-        # parse style into dict
-        style_dict = {}
-        for line in style.split("\n"):
-            if line:
-                key, value = line.split("{")
-                key = key.replace(".", "").strip()
-                value = value.split("}")[0].strip()
-                style_dict[key] = value
-
-        # replace style in html
-        for key, value in style_dict.items():
-            html_text = html_text.replace(f'class="{key}"', f'style="{value}"')
-
-        yield html_text
-
-    return html_text
+BACKENDS = ["pytorch", "onnxruntime", "openvino", "neural-compressor"]
+BENCHMARKS = ["inference", "training"]
 
 
 with gr.Blocks() as demo:
     # title text
-    gr.HTML("<h1 style='text-align: center'>🤗 Optimum Benchmark 🏋️</h1>")
+    gr.HTML("<h1 style='text-align: center'>🤗 Optimum Benchmark UI 🏋️</h1>")
     # explanation text
-    gr.Markdown(
-        "This is a demo space of [Optimum-Benchmark](https://github.com/huggingface/optimum-benchmark.git)."
-    )
+    gr.Markdown("This is a demo space of [Optimum-Benchmark](https://github.com/huggingface/optimum-benchmark.git).")
 
     model = gr.Textbox(
         label="model",
@@ -143,64 +35,90 @@ with gr.Blocks() as demo:
     )
     device = gr.Dropdown(
         value="cpu",
-        choices=["cpu", "cuda"],
         label="device",
+        choices=["cpu", "cuda"],
     )
-    expetiment_name = gr.Textbox(
+    experiment = gr.Textbox(
         label="experiment_name",
         value=f"experiment_{random.getrandbits(16)}",
     )
-
-    model.submit(fn=infer_task_from_model_name_or_path, inputs=[model], outputs=[task])
+    model.submit(fn=infer_task_from_model_name_or_path, inputs=model, outputs=task)
 
     with gr.Row():
-        with gr.Column(variant="panel"):
-            backend = gr.Dropdown(
-                ["pytorch", "onnxruntime", "openvino", "neural-compressor"],
-                label="backend",
-                value="pytorch",
-                container=True,
-            )
-
-        with gr.Column(variant="panel"):
-            with gr.Accordion(label="Backend Config", open=False):
-                backend_config = get_backend_config() + get_pytorch_config()
-
-    with gr.Row():
-        with gr.Column(variant="panel"):
-            benchmark = gr.Dropdown(
-                choices=["inference", "training"],
-                label="benchmark",
-                value="inference",
-                container=True,
-            )
-
-        with gr.Column(variant="panel"):
-            with gr.Accordion(label="Benchmark Config", open=False):
-                benchmark_config = get_inference_config()
-
-    # run benchmark button
-    run_benchmark = gr.Button(value="Run Benchmark", variant="primary")
-    # accordion with output logs
-    with gr.Accordion(label="Logs:", open=True):
-        logs = gr.HTML()
-
-    run_benchmark.click(
-        fn=run_experiment,
+        with gr.Column():
+            with gr.Row():
+                backend = gr.Dropdown(
+                    label="backend",
+                    choices=BACKENDS,
+                    value=BACKENDS[0],
+                )
+
+            with gr.Row() as backend_configs:
+                with gr.Accordion(label="Pytorch Config", open=False, visible=True):
+                    pytorch_config = get_pytorch_config()
+                with gr.Accordion(label="OnnxRunTime Config", open=False, visible=False):
+                    onnxruntime_config = get_onnxruntime_config()
+                with gr.Accordion(label="OpenVINO Config", open=False, visible=False):
+                    openvino_config = get_openvino_config()
+                with gr.Accordion(label="Neural Compressor Config", open=False, visible=False):
+                    neural_compressor_config = get_neural_compressor_config()
+
+        # hide backend configs based on backend
+        backend.change(
+            inputs=backend,
+            outputs=backend_configs.children,
+            fn=lambda value: [gr.update(visible=value == key) for key in BACKENDS],
+        )
+
+        with gr.Column():
+            with gr.Row():
+                benchmark = gr.Dropdown(
+                    label="benchmark",
+                    choices=BENCHMARKS,
+                    value=BENCHMARKS[0],
+                )
+
+            with gr.Row() as benchmark_configs:
+                with gr.Accordion(label="Inference Config", open=False, visible=True):
+                    inference_config = get_inference_config()
+                with gr.Accordion(label="Training Config", open=False, visible=False):
+                    training_config = get_training_config()
+
+        # hide benchmark configs based on benchmark
+        benchmark.change(
+            inputs=benchmark,
+            outputs=benchmark_configs.children,
+            fn=lambda value: [gr.update(visible=value == key) for key in BENCHMARKS],
+        )
+
+    button = gr.Button(value="Run Benchmark", variant="primary")
+    with gr.Accordion(label="LOGS", open=True, visible=False):
+        output = gr.HTML()
+
+    button.click(
+        fn=run_benchmark,
         inputs={
-            expetiment_name,
+            experiment,
             model,
             task,
             device,
             backend,
             benchmark,
-            *backend_config,
-            *benchmark_config,
+            *pytorch_config,
+            *openvino_config,
+            *onnxruntime_config,
+            *neural_compressor_config,
+            *inference_config,
+            *training_config,
         },
-        outputs=[logs],
+        outputs=output,
         queue=True,
     )
+    button.click(
+        inputs=[],
+        outputs=output.parent,
+        fn=lambda: gr.update(visible=True),
+    )
 
 
-if __name__ == "__main__":
-    demo.queue().launch()
+demo.queue().launch()

base_config.yaml

@@ -1,6 +1,6 @@
 defaults:
-  - backend: pytorch # default backend
-  - benchmark: inference # default benchmark
+  - backend: null # default backend
+  - benchmark: null # default benchmark
   - experiment # inheriting experiment schema
   - _self_ # for hydra 1.1 compatibility
   - override hydra/job_logging: colorlog # colorful logging

configs.py

@@ -0,0 +1,254 @@
+import gradio as gr
+
+
+def get_base_backend_config(backend_name="pytorch"):
+    return [
+        # seed
+        gr.Textbox(
+            value=42,
+            label=f"{backend_name}.seed",
+            info="Sets seed for reproducibility",
+        ),
+        # inter_op_num_threads
+        gr.Textbox(
+            value="null",
+            label=f"{backend_name}.inter_op_num_threads",
+            info="Use null for default and -1 for cpu_count()",
+        ),
+        # intra_op_num_threads
+        gr.Textbox(
+            value="null",
+            label=f"{backend_name}.intra_op_num_threads",
+            info="Use null for default and -1 for cpu_count()",
+        ),
+        # initial_isolation_check
+        gr.Checkbox(
+            value=True,
+            label=f"{backend_name}.initial_isolation_check",
+            info="Makes sure that initially, no other process is running on the target device",
+        ),
+        # continous_isolation_check
+        gr.Checkbox(
+            value=True,
+            label=f"{backend_name}.continous_isolation_check",
+            info="Makes sure that throughout the benchmark, no other process is running on the target device",
+        ),
+        # delete_cache
+        gr.Checkbox(
+            value=False,
+            label=f"{backend_name}.delete_cache",
+            info="Deletes model cache (weights & configs) after benchmark is done",
+        ),
+    ]
+
+
+def get_pytorch_config():
+    return get_base_backend_config(backend_name="pytorch") + [
+        # no_weights
+        gr.Checkbox(
+            value=False,
+            label="pytorch.no_weights",
+            info="Generates random weights instead of downloading pretrained ones",
+        ),
+        # # device_map
+        # gr.Dropdown(
+        #     value="null",
+        #
+        #     label="pytorch.device_map",
+        #     choices=["null", "auto", "sequential"],
+        #     info="Use null for default and `auto` or `sequential` the same way as in `from_pretrained`",
+        # ),
+        # torch_dtype
+        gr.Dropdown(
+            value="null",
+            label="pytorch.torch_dtype",
+            choices=["null", "bfloat16", "float16", "float32", "auto"],
+            info="Use null for default and `auto` for automatic dtype selection",
+        ),
+        # amp_autocast
+        gr.Checkbox(
+            value=False,
+            label="pytorch.amp_autocast",
+            info="Enables Pytorch's native Automatic Mixed Precision",
+        ),
+        # amp_dtype
+        gr.Dropdown(
+            value="null",
+            label="pytorch.amp_dtype",
+            info="Use null for default",
+            choices=["null", "bfloat16", "float16"],
+        ),
+        # torch_compile
+        gr.Checkbox(
+            value=False,
+            label="pytorch.torch_compile",
+            info="Compiles the model with torch.compile",
+        ),
+        # bettertransformer
+        gr.Checkbox(
+            value=False,
+            label="pytorch.bettertransformer",
+            info="Applies optimum.BetterTransformer for fastpath anf optimized attention",
+        ),
+        # quantization_scheme
+        gr.Dropdown(
+            value="null",
+            choices=["null", "gptq", "bnb"],
+            label="pytorch.quantization_scheme",
+            info="Use null for no quantization",
+        ),
+        # # use_ddp
+        # gr.Checkbox(
+        #     value=False,
+        #
+        #     label="pytorch.use_ddp",
+        #     info="Uses DistributedDataParallel for multi-gpu training",
+        # ),
+        # peft_strategy
+        gr.Textbox(
+            value="null",
+            label="pytorch.peft_strategy",
+        ),
+    ]
+
+
+def get_onnxruntime_config():
+    return get_base_backend_config(backend_name="onnxruntime")
+        # no_weights
+
+
+
+    # no_weights: bool = False
+
+    # # export options
+    # export: bool = True
+    # use_cache: bool = True
+    # use_merged: bool = False
+    # torch_dtype: Optional[str] = None
+
+    # # provider options
+    # provider: str = "${infer_provider:${device}}"
+    # device_id: Optional[int] = "${oc.deprecated:backend.provider_options.device_id}"
+    # provider_options: Dict[str, Any] = field(default_factory=lambda: {"device_id": "${infer_device_id:${device}}"})
+
+    # # inference options
+    # use_io_binding: bool = "${is_gpu:${device}}"
+    # enable_profiling: bool = "${oc.deprecated:backend.session_options.enable_profiling}"
+    # session_options: Dict[str, Any] = field(
+    #     default_factory=lambda: {"enable_profiling": "${is_profiling:${benchmark.name}}"}
+    # )
+
+    # # optimization options
+    # optimization: bool = False
+    # optimization_config: Dict[str, Any] = field(default_factory=dict)
+
+    # # quantization options
+    # quantization: bool = False
+    # quantization_config: Dict[str, Any] = field(default_factory=dict)
+
+    # # calibration options
+    # calibration: bool = False
+    # calibration_config: Dict[str, Any] = field(default_factory=dict)
+
+    # # null, O1, O2, O3, O4
+    # auto_optimization: Optional[str] = None
+    # auto_optimization_config: Dict[str, Any] = field(default_factory=dict)
+
+    # # null, arm64, avx2, avx512, avx512_vnni, tensorrt
+    # auto_quantization: Optional[str] = None
+    # auto_quantization_config: Dict[str, Any] = field(default_factory=dict)
+
+    # # ort-training is basically a different package so we might need to seperate these two backends in the future
+    # use_inference_session: bool = "${is_inference:${benchmark.name}}"
+
+    # # training options
+    # use_ddp: bool = False
+    # ddp_config: Dict[str, Any] = field(default_factory=dict)
+
+    # # peft options
+    # peft_strategy: Optional[str] = None
+    # peft_config: Dict[str, Any] = field(default_factory=dict)
+
+def get_openvino_config():
+    return get_base_backend_config(backend_name="openvino")
+
+
+def get_neural_compressor_config():
+    return get_base_backend_config(backend_name="neural_compressor")
+
+
+def get_inference_config():
+    return [
+        # duration
+        gr.Textbox(
+            value=10,
+            label="inference.duration",
+            info="Minimum duration of benchmark in seconds",
+        ),
+        # warmup runs
+        gr.Textbox(
+            value=10,
+            label="inference.warmup_runs",
+            info="Number of warmup runs before measurements",
+        ),
+        # memory
+        gr.Checkbox(
+            value=False,
+            label="inference.memory",
+            info="Measures the peak memory footprint",
+        ),
+        # energy
+        gr.Checkbox(
+            value=False,
+            label="inference.energy",
+            info="Measures energy consumption and carbon emissions",
+        ),
+        # input_shapes
+        gr.Dataframe(
+            type="array",
+            value=[[2, 16]],
+            row_count=(1, "static"),
+            col_count=(2, "dynamic"),
+            label="inference.input_shapes",
+            headers=["batch_size", "sequence_length"],
+            info="Controllable input shapes, add more columns for more inputs",
+        ),
+        # forward kwargs
+        gr.Dataframe(
+            type="array",
+            value=[[False]],
+            headers=["return_dict"],
+            row_count=(1, "static"),
+            col_count=(1, "dynamic"),
+            label="inference.forward_kwargs",
+            info="Keyword arguments for the forward pass, add more columns for more arguments",
+        ),
+    ]
+
+
+def get_training_config():
+    return [
+        # warmup steps
+        gr.Textbox(
+            value=40,
+            label="training.warmup_steps",
+        ),
+        # dataset_shapes
+        gr.Dataframe(
+            type="array",
+            value=[[500, 16]],
+            headers=["dataset_size", "sequence_length"],
+            row_count=(1, "static"),
+            col_count=(2, "dynamic"),
+            label="training.dataset_shapes",
+        ),
+        # training_arguments
+        gr.Dataframe(
+            value=[[2]],
+            type="array",
+            row_count=(1, "static"),
+            col_count=(1, "dynamic"),
+            label="training.training_arguments",
+            headers=["per_device_train_batch_size"],
+        ),
+    ]

pyproject.toml

@@ -0,0 +1,3 @@
+[tool.black]
+line-length = 119
+target-version = ['py37']

requirements.txt

@@ -1,3 +1,3 @@
 gradio
 ansi2html
-optimum-benchmark[onnxruntime,openvino,neural-compressor]@git+https://github.com/huggingface/optimum-benchmark.git
+optimum-benchmark[onnxruntime,openvino,neural-compressor,diffusers,peft]@git+https://github.com/huggingface/optimum-benchmark.git

run.py

@@ -0,0 +1,79 @@
+import pprint
+import subprocess
+import gradio as gr
+from ansi2html import Ansi2HTMLConverter
+
+ansi2html_converter = Ansi2HTMLConverter(inline=True)
+
+
+def run_benchmark(kwargs):
+    for key, value in kwargs.copy().items():
+        if key.label == "experiment_name":
+            experiment_name = value
+            kwargs.pop(key)
+        elif key.label == "model":
+            model = value
+            kwargs.pop(key)
+        elif key.label == "task":
+            task = value
+            kwargs.pop(key)
+        elif key.label == "device":
+            device = value
+            kwargs.pop(key)
+        elif key.label == "backend":
+            backend = value
+            kwargs.pop(key)
+        elif key.label == "benchmark":
+            benchmark = value
+            kwargs.pop(key)
+        else:
+            continue
+
+    arguments = [
+        "optimum-benchmark",
+        "--config-dir",
+        "./",
+        "--config-name",
+        "base_config",
+        f"task={task}",
+        f"model={model}",
+        f"device={device}",
+        f"backend={backend}",
+        f"benchmark={benchmark}",
+        f"experiment_name={experiment_name}",
+    ]
+
+    for component, value in kwargs.items():
+        if f"{backend}." in component.label or f"{benchmark}." in component.label:
+            label = component.label.replace(f"{backend}.", "backend.").replace(f"{benchmark}.", "benchmark.")
+
+            if isinstance(component, gr.Dataframe):
+                for sub_key, sub_value in zip(component.headers, value[0]):
+                    arguments.append(f"++{label}.{sub_key}={sub_value}")
+            else:
+                arguments.append(f"{label}={value}")
+
+    pprint.pprint(arguments)
+
+    # stream subprocess output
+    process = subprocess.Popen(
+        arguments,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        universal_newlines=True,
+    )
+
+    ansi_text = ""
+    for ansi_line in iter(process.stdout.readline, ""):
+        if "torch.distributed.nn.jit.instantiator" in ansi_line:
+            continue
+        # stream process output
+        print(ansi_line, end="")
+        # append line to ansi text
+        ansi_text += ansi_line
+        # convert ansi to html
+        html_text = ansi2html_converter.convert(ansi_text)
+        # stream html output
+        yield html_text
+
+    return html_text