Update parse_requests.py

parse_requests.py

@@ -1,22 +1,16 @@
 import os
-from huggingface_hub import snapshot_download
-import json
+from datasets import load_dataset, Dataset
 
 TOKEN = os.environ.get("DEBUG")
-requests_dataset = snapshot_download('EnergyStarAI/requests_debug', token=TOKEN, repo_type="dataset", allow_patterns="*.json")
+requests_dataset = load_dataset("EnergyStarAI/requests_debug", split="test")
 
 def normalize_task(task):
     # Makes assumption about how the task names are being written, and called.
     return '_'.join(task.split()).lower()
 
 
-for dir, path, files in os.walk(requests_dataset):
-    for fid in files:
-        file_path = os.path.join(dir, fid)
-        with open(file_path) as fp:
-            request = json.load(fp)
-            status = request['status']
-            if status == 'PENDING':
-                model = request['model']
-                task = normalize_task(request['task'])
-                print("%s,%s" % (model, task))
+requests_dset = requests_dataset.to_pandas()
+
+for model, task in requests_dset[['model','task']].loc[requests_dset['status'] == 'PENDING'].to_dict(orient= 'split', index=False)['data']:
+    print("%s,%s" % (model, normalize_task(task)))
+