Add try/catch in bash entrypoint to manage failed experiments

entrypoint.sh

@@ -8,12 +8,25 @@ echo "Attempting to run."
 python /parse_requests.py | while read line; do
     IFS="," read backend_model experiment_name <<< $(echo ${line})
     echo "Benchmarking Model: ${backend_model}, Task: ${experiment_name}"
-    optimum-benchmark --config-name ${experiment_name}  --config-dir /optimum-benchmark/examples/energy_star/ backend.model=${backend_model} backend.processor=${backend_model} hydra.run.dir="./runs/${experiment_name}/${backend_model}/${now:%Y-%m-%d-%H-%M-%S}"
+    export run_dir= "./runs/${experiment_name}/${backend_model}/${now:%Y-%m-%d-%H-%M-%S}"
+
+    optimum-benchmark --config-name ${experiment_name}  --config-dir /optimum-benchmark/examples/energy_star/ backend.model=${backend_model} backend.processor=${backend_model} hydra.run.dir=${run_dir} 2> $run_dir/error.log
+
+    if [ -s $run_dir/error.log ]; then
+        # error.log is not-empty, an error was raised
+        echo "An error was raised while benchmarking the model..."
+        python /failed_run.py --run_dir $run_dir --model_name $backend_model
+
+        # Delete the current run directory so that it is not pushed by create_results.py later
+        rm -rf $run_dir
+    fi
 done
+
+# The file is empty, so no error
 echo "Finished; uploading dataset results"
 python /create_results.py ./runs
+
 # Pausing space
 echo "Pausing space."
 python /pause_space.py
 echo "Done."
-#fi

failed_run.py

@@ -0,0 +1,46 @@
+import argparse
+import os
+
+from datasets import load_dataset, Dataset
+from huggingface_hub import HfApi
+
+
+TOKEN = os.environ.get("DEBUG")
+api = HfApi(token=TOKEN)
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--run_dir",
+    default=None,
+    type=str,
+    required=True,
+    help="Path to the run directory.",
+)
+parser.add_argument(
+    "--model_name",
+    default=None,
+    type=str,
+    required=True,
+    help="Model to benchmark.",
+)
+args = parser.parse_args()
+
+# Updating request
+dataset = load_dataset("EnergyStarAI/requests_debug", split="test", token=TOKEN).to_pandas()
+
+# Set benchmark to failed
+dataset.loc[dataset["model"].isin(args.model_name), ['status']] = "FAILED"
+
+# Read error message
+with open(f"{args.run_dir}/error.log", 'r') as file:
+    error_message = file.read()
+
+# Add a new column for the error message if necessary
+if "error_message" not in dataset.columns:
+    dataset["error_message"] = ""
+dataset.loc[dataset["model"].isin(args.model_name), ['error_message']] = error_message
+
+updated_dataset = Dataset.from_pandas(dataset)
+updated_dataset.push_to_hub("EnergyStarAI/requests_debug", split="test", token=TOKEN)
+
+print("Status set to FAILED")