Update app.py

app.py

@@ -1,48 +1,41 @@
-import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import torch
-from ctransformers import AutoConfig
-import os 
+from langchain_community.llms.ctransformers import CTransformers
 
 hf_token = os.environ.get('HF_TOKEN')
 
 from huggingface_hub import login
 login(token=hf_token)
 
-config = AutoConfig.from_pretrained( "mistralai/Mistral-7B-Instruct-v0.1")
-config.config.max_new_tokens = 2000
-config.config.context_length = 4000                                                    
 
-model = AutoModelForCausalLM.from_pretrained(
-    "mistralai/Mistral-7B-Instruct-v0.1",
-    token = hf_token,
-    torch_dtype=torch.bfloat16,
-    trust_remote_code=True,
-    device_map="auto",
-    config=config)
-tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1", token = hf_token)
+# config = AutoConfig.from_pretrained("Mistral-7B-v0.1-GGUF")
+# config.config.max_new_tokens = 2000
+# config.config.context_length = 6000
+
+# llm = AutoModelForCausalLM.from_pretrained("TheBloke/Mistral-7B-v0.1-GGUF", model_file="mistral-7b-v0.1.Q5_K_M.gguf", model_type="mistral",gpu_layers=0, config=config)
+
+MODEL_TYPE = 'mistral'
+MODEL_BIN_PATH =  "mistral-7b-instruct-v0.1.Q3_K_S.gguf"
+MAX_NEW_TOKEN =  600
+TEMPRATURE =  0.01
+CONTEXT_LENGTH =   6000
+
+
+llm = CTransformers(
+            model=MODEL_BIN_PATH,
+            config={
+                'max_new_tokens': MAX_NEW_TOKEN,
+                'temperature': TEMPRATURE,
+                'context_length': CONTEXT_LENGTH
+            },
+            model_type=MODEL_TYPE
+        )
 
 def generate_text(input_text):
-    input_ids = tokenizer.encode(input_text, return_tensors="pt")
-    attention_mask = torch.ones(input_ids.shape)
-
-    output = model.generate(
-        input_ids,
-        attention_mask=attention_mask,
-        max_length=200,
-        do_sample=True,
-        top_k=10,
-        num_return_sequences=1,
-        eos_token_id=tokenizer.eos_token_id,
-        
-    )
-
-    output_text = tokenizer.decode(output[0], skip_special_tokens=True)
-    print(output_text)
-
-    # Remove Prompt Echo from Generated Text
-    cleaned_output_text = output_text.replace(input_text, "")
-    return cleaned_output_text
+
+
+    output = llm(input_text)
+    print(output)
+
+    return output
 
 
 text_generation_interface = gr.Interface(