Merge branch 'pr/15' into pr/18

app.py

@@ -2,10 +2,11 @@
 import os
 import datetime
 import socket
+from threading import Thread
 
 import gradio as gr
 import pandas as pd
-
+import time
 from apscheduler.schedulers.background import BackgroundScheduler
 
 from huggingface_hub import snapshot_download
@@ -37,11 +38,24 @@ from src.display.utils import (
     Precision,
 )
 
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, \
+    QUEUE_REPO, REPO_ID, RESULTS_REPO, DEBUG_QUEUE_REPO, DEBUG_RESULTS_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 from src.utils import get_dataset_summary_table
 
+def get_args():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Run the LLM Leaderboard")
+    parser.add_argument("--debug", action="store_true", help="Run in debug mode")
+    return parser.parse_args()
+
+args = get_args()
+if args.debug:
+    print("Running in debug mode")
+    QUEUE_REPO = DEBUG_QUEUE_REPO
+    RESULTS_REPO = DEBUG_RESULTS_REPO
 
 def ui_snapshot_download(repo_id, local_dir, repo_type, tqdm_class, etag_timeout):
     try:
@@ -75,11 +89,6 @@ def init_space():
     )
     return dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
 
-
-dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
-leaderboard_df = original_df.copy()
-
-
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame, columns: list, type_query: list, precision_query: list, size_query: list, query: str
@@ -142,6 +151,51 @@ def filter_models(df: pd.DataFrame, type_query: list, size_query: list, precisio
 
     return filtered_df
 
+shown_columns = None
+dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
+leaderboard_df = original_df.copy()
+
+def update_leaderboard_table():
+    global leaderboard_df, shown_columns
+    print("Updating leaderboard table")
+    return leaderboard_df[
+                [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+                + shown_columns.value
+                + [AutoEvalColumn.dummy.name]
+            ] if not leaderboard_df.empty else leaderboard_df
+        
+
+def update_hidden_leaderboard_table():
+    global original_df
+    return original_df[COLS] if original_df.empty is False else original_df
+
+def update_dataset_table():
+    global dataset_df
+    return dataset_df
+
+def update_finish_table():
+    global finished_eval_queue_df
+    return finished_eval_queue_df
+
+def update_running_table():
+    global running_eval_queue_df
+    return running_eval_queue_df
+
+def update_pending_table():
+    global pending_eval_queue_df
+    return pending_eval_queue_df
+
+def update_finish_num():
+    global finished_eval_queue_df
+    return len(finished_eval_queue_df)
+
+def update_running_num():
+    global running_eval_queue_df
+    return len(running_eval_queue_df)
+
+def update_pending_num():
+    global pending_eval_queue_df
+    return len(pending_eval_queue_df)
 
 # triggered only once at startup => read query parameter if it exists
 def load_query(request: gr.Request):
@@ -385,8 +439,7 @@ with demo:
 
 scheduler = BackgroundScheduler()
 
-scheduler.add_job(restart_space, "interval", seconds=6 * 60 * 60)
-
+scheduler.add_job(restart_space, "interval", hours=6)
 
 def launch_backend():
     import subprocess
@@ -395,8 +448,9 @@ def launch_backend():
     if DEVICE not in {"cpu"}:
         _ = subprocess.run(["python", "backend-cli.py"])
 
-
+Thread(target=periodic_init, daemon=True).start()
 # scheduler.add_job(launch_backend, "interval", seconds=120)
-
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+if __name__ == "__main__":
+    scheduler.start()
+    block_launch()
+    

src/backend/envs.py

@@ -63,4 +63,4 @@ EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_REQUESTS_PATH_BACKEND_SYNC = os.path.join(CACHE_PATH, "eval-queue-bk-sync")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
-DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu"

src/display/utils.py

@@ -140,6 +140,7 @@ class EvalQueueColumn:  # Queue column
     private = ColumnContent("private", "bool", True)
     precision = ColumnContent("precision", "str", True)
     weight_type = ColumnContent("weight_type", "str", "Original")
+    model_framework = ColumnContent("inference_framework", "str", True)
     status = ColumnContent("status", "str", True)
 
 

src/envs.py

@@ -12,8 +12,8 @@ QUEUE_REPO = "sparse-generative-ai/requests"
 QUEUE_REPO_OPEN_LLM = "open-llm-leaderboard/requests"
 RESULTS_REPO = "sparse-generative-ai/results"
 
-PRIVATE_QUEUE_REPO = "sparse-generative-ai/private-requests"
-PRIVATE_RESULTS_REPO = "sparse-generative-ai/private-results"
+DEBUG_QUEUE_REPO = "sparse-generative-ai/debug_requests"
+DEBUG_RESULTS_REPO = "sparse-generative-ai/debug_results"
 
 IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
 

src/populate.py

@@ -95,6 +95,7 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> tuple[pd.DataFrame, p
 
             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            data[EvalQueueColumn.model_framework.name] = data.get("inference_framework", "-")
 
             all_evals.append(data)
         elif ".md" not in entry:
@@ -107,6 +108,7 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> tuple[pd.DataFrame, p
 
                 data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                data[EvalQueueColumn.model_framework.name] = data.get("inference_framework", "-")
                 all_evals.append(data)
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]

src/submission/submit.py

@@ -3,7 +3,7 @@ import os
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA, DEBUG_QUEUE_REPO
 from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
 from src.submission.check_validity import (
     already_submitted_models,
@@ -26,12 +26,16 @@ def add_new_eval(
     weight_type: str,
     model_type: str,
     inference_framework: str,
+    debug: bool = False
 ):
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
+    if debug:
+        QUEUE_REPO = DEBUG_QUEUE_REPO
+    
     user_name = ""
     model_path = model
     if "/" in model: