debug

app.py

@@ -4,40 +4,36 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
 import uuid
 import os
 from datetime import datetime
-import spaces  # required for ZeroGPU
+import spaces
 
-# ----- Constants -----
 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
 with open("system_prompt.txt", "r") as f:
     SYSTEM_PROMPT = f.read()
 LOG_DIR = "chat_logs"
 os.makedirs(LOG_DIR, exist_ok=True)
 
-# Global vars to hold model and tokenizer
+# Globals
 model = None
 tokenizer = None
 session_id = str(uuid.uuid4())
 
-# ----- Log Chat -----
 def log_chat(session_id, user_msg, bot_msg):
     timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     with open(os.path.join(LOG_DIR, f"{session_id}.txt"), "a") as f:
         f.write(f"[{timestamp}] User: {user_msg}\n")
         f.write(f"[{timestamp}] Bot: {bot_msg}\n\n")
 
-# ----- Required by ZeroGPU -----
+# This function will be run by ZeroGPU
 @spaces.GPU
 def load_model():
-    global model, tokenizer
     tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
     model = AutoModelForCausalLM.from_pretrained(
         MODEL_NAME,
         torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
         device_map="auto"
     )
-    model.eval()
+    return tokenizer, model
 
-# ----- Inference Function -----
 def format_chat_prompt(history, new_input):
     messages = [{"role": "system", "content": SYSTEM_PROMPT}]
     for user_msg, bot_msg in history:
@@ -48,6 +44,12 @@ def format_chat_prompt(history, new_input):
 
 @torch.no_grad()
 def respond(message, history):
+    global tokenizer, model
+
+    # Lazy-load model only when needed
+    if tokenizer is None or model is None:
+        tokenizer, model = load_model()
+
     prompt = format_chat_prompt(history, message)
     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
     output = model.generate(
@@ -63,9 +65,6 @@ def respond(message, history):
     log_chat(session_id, message, response)
     return response
 
-load_model()
-
-# ----- Gradio App -----
 gr.ChatInterface(
     fn=respond,
     title="BoundrAI",

requirements.txt

@@ -2,4 +2,5 @@ huggingface_hub==0.25.2
 gradio
 transformers
 torch
-spaces
+spaces
+accelerate>=0.26.0