Removed 4bit

app.py

@@ -4,7 +4,6 @@ from transformers import (
     AutoModelForCausalLM,
     AutoTokenizer,
     TextIteratorStreamer,
-    BitsAndBytesConfig,
 )
 import os
 from threading import Thread
@@ -13,12 +12,9 @@ import time
 
 token = os.environ["HF_TOKEN"]
 
-quantization_config = BitsAndBytesConfig(
-    load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16
-)
 
 model = AutoModelForCausalLM.from_pretrained(
-    "microsoft/Phi-3-mini-128k-instruct", quantization_config=quantization_config, token=token,trust_remote_code=True
+    "microsoft/Phi-3-mini-128k-instruct", token=token,trust_remote_code=True
 )
 tok = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-128k-instruct", token=token)
 terminators = [
@@ -32,13 +28,12 @@ else:
     device = torch.device("cpu")
     print("Using CPU")
 
-# model = model.to(device)
+model = model.to(device)
 # Dispatch Errors
 
 
 @spaces.GPU(duration=60)
 def chat(message, history, temperature,do_sample, max_tokens):
-    start_time = time.time()
     chat = []
     for item in history:
         chat.append({"role": "user", "content": item[0]})
@@ -66,19 +61,11 @@ def chat(message, history, temperature,do_sample, max_tokens):
     t.start()
 
     partial_text = ""
-    first_token_time = None
     for new_text in streamer:
-        if not first_token_time:
-            first_token_time = time.time() - start_time
         partial_text += new_text
         yield partial_text
 
-    total_time = time.time() - start_time
-    tokens = len(tok.tokenize(partial_text))
-    tokens_per_second = tokens / total_time if total_time > 0 else 0
-
-    timing_info = f"\n\nTime taken to first token: {first_token_time:.2f} seconds\nTokens per second: {tokens_per_second:.2f}"
-    yield partial_text +  timing_info
+    yield partial_text 
 
 
 demo = gr.ChatInterface(
@@ -104,6 +91,6 @@ demo = gr.ChatInterface(
     ],
     stop_btn="Stop Generation",
     title="Chat With LLMs",
-    description="Now Running [microsoft/Phi-3-mini-128k-instruct](https://huggingface.com/microsoft/Phi-3-mini-128k-instruct) in 4bit"
+    description="Now Running [microsoft/Phi-3-mini-128k-instruct](https://huggingface.com/microsoft/Phi-3-mini-128k-instruct)"
 )
 demo.launch()