Update app.py

app.py

@@ -1,62 +1,11 @@
-from transformers import AutoTokenizer
-from exllamav2 import (
-    ExLlamaV2,
-    ExLlamaV2Config,
-    ExLlamaV2Cache_CPU,
-    ExLlamaV2Tokenizer
-)
-from exllamav2.generator import (
-    ExLlamaV2StreamingGenerator,
-    ExLlamaV2Sampler
-)
-import torch
-import os
+from gptqmodel import GPTQModel
+# load Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4 from modelscope
+model = GPTQModel.load("TheBloke/Wizard-Vicuna-13B-Uncensored-SuperHOT-8K-GPTQ")
 
-# disable CUDA
-os.environ["CUDA_VISIBLE_DEVICES"] = "-1"  # Disables GPU detection
-os.environ["EXLLAMA_NO_CUDA"] = "1"  # Forces CPU mode in ExLlamaV2
-
-# Configure model
-model_dir = "TheBloke_Wizard-Vicuna-13B-GPTQ"  # Path to downloaded model
-config = ExLlamaV2Config()
-config.model_dir = model_dir
-config.device_map = "cpu"
-config.no_flash_attn = True  # Disable flash attention
-config.prepare()
-
-# Load model
-model = ExLlamaV2(config)
-cache = ExLlamaV2Cache_CPU(model)
-model.load_autosplit(cache)
-
-# Load tokenizer (HF-compatible)
-tokenizer = AutoTokenizer.from_pretrained(model_dir)
-
-def generate_response(prompt, max_tokens=200, temperature=0.7):
-    # Initialize generator
-    generator = ExLlamaV2StreamingGenerator(model, cache, tokenizer)
-    generator.set_stop_conditions([tokenizer.eos_token_id])
-    
-    # Configure sampler
-    settings = ExLlamaV2Sampler.Settings()
-    settings.temperature = temperature
-    settings.top_k = 50
-    settings.top_p = 0.8
+async def generate_response(input: str):
+    result = model.generate(input)[0] 
+    return model.tokenizer.decode(result)
     
-    # Encode prompt
-    input_ids = tokenizer(prompt, return_tensors="pt").input_ids.cuda()
-    
-    # Generate
-    output = generator.generate_simple(
-        input_ids,
-        settings,
-        max_tokens,
-        seed=42
-    )
-    
-    return tokenizer.decode(output[0], skip_special_tokens=True)
-
-
 
 ##############################################