src/dataloading.py

from huggingface_hub import HfApi
from functools import lru_cache


def get_leaderboard_models():
    api = HfApi()
    
    # List all datasets in the open-llm-leaderboard organization
    datasets = api.list_datasets(author="open-llm-leaderboard")
    
    models = []
    #for dataset in datasets:
    #    if dataset.id.endswith("-details"):
    #        # Format: "open-llm-leaderboard/<provider>__<model_name>-details"
    #        model_part = dataset.id.split("/")[-1].replace("-details", "")
    #        provider, model = model_part.split("__", 1)
    #        models.append(f"{provider}/{model}")
    
    # Example models
    models = [
        "meta_llama/Llama-3.2-1B-Instruct",
        "meta_llama/Llama-3.2-3B-Instruct",
        "meta_llama/Llama-3.1-8B-Instruct",
        "meta_llama/Llama-3.1-70B-Instruct",
        "meta_llama/Llama-3.3-70B-Instruct",
        ]

    return sorted(models)



@lru_cache(maxsize=1)
def get_leaderboard_models_cached():
    return get_leaderboard_models()


def get_leaderboard_datasets():
    return [
        "ai2_arc",
        "hellaswag",
        "mmlu",
        "truthful_qa",
        "winogrande",
        "gsm8k"
    ]