load model bf16

app.py

@@ -62,9 +62,8 @@ def predict(message, history, system_prompt, temperature, max_tokens):
 if __name__ == "__main__":
     args = parse_args()
     tokenizer = AutoTokenizer.from_pretrained("stabilityai/stable-code-instruct-3b")
-    model = AutoModelForCausalLM.from_pretrained("stabilityai/stable-code-instruct-3b")
+    model = AutoModelForCausalLM.from_pretrained("stabilityai/stable-code-instruct-3b", torch_dtype=torch.bfloat16)
     device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-    model = model.half()
     model = model.to(device)
     gr.ChatInterface(
         predict,