added mostly hardcoded generate url method and test

details_data_processor.py

@@ -8,7 +8,7 @@ import requests
 
 class DetailsDataProcessor:
     # Download 
-    #url example
+    #url example https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/64bits/LexPodLM-13B/details_harness%7ChendrycksTest-moral_scenarios%7C5_2023-07-25T13%3A41%3A51.227672.json
     
     def __init__(self, directory='results', pattern='results*.json'):
         self.directory = directory
@@ -18,10 +18,38 @@ class DetailsDataProcessor:
 
     # download a file from a single url and save it to a local directory
     @staticmethod
-    def _download_file(url, filename):
+    def download_file(url, filename):
         r = requests.get(url, allow_redirects=True)
         open(filename, 'wb').write(r.content)
 
+    @staticmethod
+    def single_file_pipeline(url, filename):
+        DetailsDataProcessor.download_file(url, filename)
+        # read file
+        with open(filename) as f:
+            data = json.load(f)
+        # convert to dataframe
+        df = pd.DataFrame(data)
+        return df
+
+    @staticmethod
+    def generate_url(file_path):
+        base_url = 'https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/'
+
+
+        organization = '64bits'
+        model = 'LexPodLM-13B'
+        filename = '_2023-07-25T13%3A41%3A51.227672.json'
+        # extract organization, model, and filename from file_path instead of hardcoding
+        # filename = file_path.split('/')[-1]
+
+
+
+        other_chunk = 'details_harness%7ChendrycksTest-moral_scenarios%7C5'
+        constructed_url = base_url + organization + '/' + model + '/' + other_chunk + filename
+        return constructed_url
+
+
     # @staticmethod
     # def _find_files(directory, pattern):
     #     for root, dirs, files in os.walk(directory):

test_details_data_processing.py

@@ -16,10 +16,18 @@ class TestDetailsDataProcessor(unittest.TestCase):
         # self.assertIsInstance(data, pd.DataFrame)
 
     def test_download_file(self):
-        DetailsDataProcessor._download_file('https://www.google.com', 'test.html')
+        DetailsDataProcessor.download_file('https://www.google.com', 'test.html')
         self.assertTrue(os.path.exists('test.html'))
         os.remove('test.html')
 
+    def test_generate_url(self):
+        results_file_path = "64bits/LexPodLM-13B/results_2023-07-25T13:41:51.227672.json"
+        expected_url = 'https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/64bits/LexPodLM-13B/details_harness%7ChendrycksTest-moral_scenarios%7C5_2023-07-25T13%3A41%3A51.227672.json'
+
+
+        constructed_url = self.processor.generate_url(results_file_path)
+        self.assertEqual(expected_url, constructed_url)
+
         
 if __name__ == '__main__':
     unittest.main()