Update app.py

app.py

@@ -6,14 +6,13 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStream
 import gradio as gr
 from threading import Thread
 
+device = "auto"
 model_id = "ibm-granite/granite-3.0-8b-instruct"
 tokenizer = AutoTokenizer.from_pretrained(model_id)
-model = AutoModelForCausalLM.from_pretrained(
-  model_id,
-  torch_dtype=torch.bfloat16,
-  device_map="auto",
-  #use_flash_attention_2=True,
-)
+# drop device_map if running on CPU
+model = AutoModelForCausalLM.from_pretrained(model_path, device_map=device)
+model.eval()
+# change input text as desired
 
 TITLE = "<h1><center>ibm-granite/granite-3.0-8b-instruct Chat webui</center></h1>"