parse_requests.py

import os
from huggingface_hub import snapshot_download
import json

TOKEN = os.environ.get("DEBUG")
requests_dataset = snapshot_download('EnergyStarAI/requests_debug', token=TOKEN, repo_type="dataset", allow_patterns="*.json")

def normalize_task(task):
    # Makes assumption about how the task names are being written, and called.
    return '_'.join(task.split()).lower()


for dir, path, files in os.walk(requests_dataset):
    for fid in files:
        file_path = os.path.join(dir, fid)
        with open(file_path) as fp:
            request = json.load(fp)
            status = request['status']
            if status == 'PENDING':
                model = request['model']
                task = normalize_task(request['task'])
                print("%s,%s" % (model, task))