Replicating 404 error with a test so I can troubleshoot

details_data_processor.py

@@ -48,7 +48,7 @@ class DetailsDataProcessor:
         # Format the timestamp as a string, suitable for use in a filename
         filename_timestamp = timestamp.strftime("%Y-%m-%dT%H-%M-%S")
 
-        # Example usage in a filename
+        # Construct the full save file path
         save_file_path = save_file_path + filename_timestamp + ".json"
 
         print(save_file_path)  # Output will be something like "results_2023-08-20T12-34-56.txt"
@@ -56,25 +56,30 @@ class DetailsDataProcessor:
         try:
             # Sending a GET request
             r = requests.get(url, allow_redirects=True)
-            r.raise_for_status() # Raises an HTTPError if the HTTP request returned an unsuccessful status code
+            r.raise_for_status()  # Raises an HTTPError if the HTTP request returned an unsuccessful status code
             
             # Writing the content to the specified file
             with open(save_file_path, 'wb') as file:
                 file.write(r.content)
-            
+
             print(f"Successfully downloaded file: {save_file_path}")
-        except requests.ConnectionError:
+        except requests.ConnectionError as e:
             print(f"Failed to connect to the URL: {url}")
+            raise e
         except requests.HTTPError as e:
             print(f"HTTP error occurred: {e}")
-        except FileNotFoundError:
+            raise e
+        except FileNotFoundError as e:
             print(f"File not found at path: {save_file_path}")
+            raise e
         except Exception as e:
             print(f"An unexpected error occurred: {e}")
+            raise e
 
         return None
 
 
+
     @staticmethod
     def single_file_pipeline(url, filename):
         DetailsDataProcessor.download_file(url, filename)

test_details_data_processing.py

@@ -20,6 +20,15 @@ class TestDetailsDataProcessor(unittest.TestCase):
         self.assertTrue(os.path.exists('test.html'))
         os.remove('test.html')
 
+
+    # there is a file path that results in a 404 error.
+    # create a test replicate that error
+    def test_download_file_404(self):
+        file_path_with_error = 'results/shaohang/Sparse0.5_OPT-1.3/results_2023-07-19T19:10:31.005235.json'
+        url = self.processor.build_url(file_path_with_error)
+        DetailsDataProcessor.download_file(url, 'test_file_please_remove')
+
+
     def test_build_url(self):
         test_cases = [
             ('results/64bits/LexPodLM-13B/results_2023-07-25T13:41:51.227672.json',