Update app.py

app.py

@@ -19,6 +19,8 @@ model, tokenizer = FastLanguageModel.from_pretrained(
     load_in_4bit = load_in_4bit,
 )
 FastLanguageModel.for_inference(model) # Enable native 2x faster inference
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+model = model.to(device)
 
 def respond(message, history, system_message, max_tokens, temperature, top_p):
     # Combine system message and chat history
@@ -33,7 +35,7 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
         return_tensors="pt",
         truncation=True,
         max_length=max_seq_length,
-    ).to("cuda")
+    ).to(device)
     
     # Generate the response
     with torch.no_grad():