Update process_runs.py

process_runs.py

@@ -75,18 +75,21 @@ def update_requests(requests, all_attempts, failed_attempts):
     # Each line is a run directory, where
     # run_dir="/runs/${experiment_name}/${backend_model}/${now}"
     for line in all_attempts:
+        print(f"Checking {line}")
         split_run_dir = line.strip().split("/")
         task = split_run_dir[1]
         model = split_run_dir[2]
-        if line not in failed_attempts:
-            traceback_error = check_for_traceback(line)
-            if traceback_error != "":
-                print("Found a traceback error!")
-                print(traceback_error)
-                requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['status']] = FAILED
-                requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['error_message']] = traceback_error
-            else:
-                requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['status']] = COMPLETED
+        traceback_error = check_for_traceback(line)
+        if traceback_error != "":
+            print("Found a traceback error!")
+            print(traceback_error)
+            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['status']] = FAILED
+            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['error_message']] = traceback_error
+        elif line in failed_attempts:
+            print(f"Job failed, but not sure why -- didn't find a traceback in {line}")
+            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['status']] = FAILED
+        else:
+            requests_df.loc[(requests_df["status"] == PENDING) & (requests_df["model"] == model) & (requests_df["task"] == task), ['status']] = COMPLETED
     updated_dset = Dataset.from_pandas(requests_df)
     return updated_dset