Chat interface

app.py

@@ -1,7 +1,21 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
 import gradio as gr
 
-def greet(name):
-    return "Hello " + name + "!!"
+checkpoint = "WillHeld/soft-raccoon"
+device = "cuda"  # "cuda" or "cpu"
+tokenizer = AutoTokenizer.from_pretrained(checkpoint)
+model = AutoModelForCausalLM.from_pretrained(checkpoint).to(device)
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+@spaces.GPU(duration=120)
+def predict(message, history):
+    history.append({"role": "user", "content": message})
+    input_text = tokenizer.apply_chat_template(history, tokenize=False)
+    inputs = tokenizer.encode(input_text, return_tensors="pt").to(device)  
+    outputs = model.generate(inputs, max_new_tokens=100, temperature=0.2, top_p=0.9, do_sample=True)
+    decoded = tokenizer.decode(outputs[0])
+    response = decoded.split("<|im_start|>assistant\n")[-1].split("<|im_end|>")[0]
+    return response
+
+demo = gr.ChatInterface(predict, type="messages")
+
+demo.launch()