Update app.py

app.py

@@ -1,33 +1,19 @@
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer
 import torch
-from peft import PeftModel
-
-# Model and tokenizer names
-model_name = "google/gemma-2-2b-it"
-lora_model_name = "Anlam-Lab/gemma-2-2b-it-anlamlab-SA-Chatgpt4mini"
-
-# Configure 4-bit quantization
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_quant_type="nf4",
-    bnb_4bit_compute_dtype=torch.float16,
-    bnb_4bit_use_double_quant=True,
+from peft import (
+    LoraConfig,
+    PeftModel,
+    prepare_model_for_kbit_training,
+    get_peft_model,
 )
+model_name = "google/gemma-2-2b-it"
+lora_model_name="Anlam-Lab/gemma-2-2b-it-anlamlab-SA-Chatgpt4mini"
 
-# Initialize tokenizer
 tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", torch_dtype=torch.float16)
 
-# Load the base model with 4-bit quantization
-model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-    device_map="auto",
-    quantization_config=bnb_config
-)
-
-# Load the LoRA adapter
 model = PeftModel.from_pretrained(model, lora_model_name)
-
 def generate_response(input_text):
     inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
     
@@ -40,12 +26,14 @@ def generate_response(input_text):
     }
     
     with torch.no_grad():
-        outputs = model.generate(**inputs, **generation_config)
-        
+        outputs = model.generate(
+            **inputs,
+            **generation_config
+        )
+    
     response = tokenizer.decode(outputs[0])
     return response.split("<start_of_turn>model\n")[1].split("<end_of_turn>")[0]
 
-# Create Gradio interface
 iface = gr.Interface(
     fn=generate_response,
     inputs=gr.Textbox(lines=5, placeholder="Metninizi buraya girin..."),