chore: Add dataset storage and CommitScheduler for dataset uploads

app.py

@@ -3,12 +3,13 @@ import json
 from datetime import datetime
 from theme import TufteInspired
 import uuid
-from huggingface_hub import InferenceClient
+from huggingface_hub import InferenceClient, CommitScheduler, hf_hub_download
 from openai import OpenAI
 from huggingface_hub import get_token, login
 from prompts import detailed_genre_description_prompt, basic_prompt
 import random
 import os
+from pathlib import Path
 
 # Ensure you're logged in to Hugging Face
 login(get_token())
@@ -21,6 +22,55 @@ MODELS = [
 
 CHOSEN_MODEL = None
 
+# Set up dataset storage
+dataset_folder = Path("dataset")
+dataset_folder.mkdir(exist_ok=True)
+
+
+# Function to get the latest dataset file
+def get_latest_dataset_file():
+    files = list(dataset_folder.glob("data_*.jsonl"))
+    return max(files, key=os.path.getctime) if files else None
+
+
+# Check for existing dataset and create or append to it
+latest_file = get_latest_dataset_file()
+if latest_file:
+    dataset_file = latest_file
+    print(f"Appending to existing dataset file: {dataset_file}")
+else:
+    dataset_file = dataset_folder / f"data_{uuid.uuid4()}.jsonl"
+    print(f"Creating new dataset file: {dataset_file}")
+
+# Set up CommitScheduler for dataset uploads
+repo_id = "your-username/your-dataset-repo"  # Replace with your desired dataset repo
+scheduler = CommitScheduler(
+    repo_id=repo_id,
+    repo_type="dataset",
+    folder_path=dataset_folder,
+    path_in_repo="data",
+    every=5,  # Upload every 5 minutes
+)
+
+
+# Function to download existing dataset files
+def download_existing_dataset():
+    try:
+        files = hf_hub_download(
+            repo_id=repo_id, filename="data", repo_type="dataset", recursive=True
+        )
+        for file in Path(files).glob("*.jsonl"):
+            dest_file = dataset_folder / file.name
+            if not dest_file.exists():
+                dest_file.write_bytes(file.read_bytes())
+                print(f"Downloaded existing dataset file: {dest_file}")
+    except Exception as e:
+        print(f"Error downloading existing dataset: {e}")
+
+
+# Download existing dataset files at startup
+download_existing_dataset()
+
 
 def get_random_model():
     global CHOSEN_MODEL
@@ -84,8 +134,9 @@ def log_blurb_and_vote(prompt, blurb, vote, user_info: gr.OAuthProfile | None, *
         "user_id": user_id,
         "model": CHOSEN_MODEL,
     }
-    with open("blurb_log.jsonl", "a") as f:
-        f.write(json.dumps(log_entry) + "\n")
+    with scheduler.lock:
+        with dataset_file.open("a") as f:
+            f.write(json.dumps(log_entry) + "\n")
     gr.Info("Thank you for voting!")
     return f"Logged: {vote} by user {user_id}"
 
@@ -98,7 +149,7 @@ with gr.Blocks(theme=tufte_theme) as demo:
     gr.Markdown("<h1 style='text-align: center;'>Would you read this book?</h1>")
     gr.Markdown(
         """<p style='text-align: center;'>Looking for your next summer read? 
-    Would you read a book based on this LLM generated blurb? <br> Your vote will be added to <a href="https://example.com">this</a> Hugging Face dataset</p>"""
+    Would you read a book based on this LLM generated blurb? <br> Your vote will be added to <a href="https://huggingface.co/datasets/your-username/your-dataset-repo">this</a> Hugging Face dataset</p>"""
     )
 
     # Add the login button