app.py

from gptqmodel import GPTQModel
# load Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4 from modelscope
model = GPTQModel.load("TheBloke/Wizard-Vicuna-13B-Uncensored-SuperHOT-8K-GPTQ")

async def generate_response(input: str):
    result = model.generate(input)[0] 
    return model.tokenizer.decode(result)
    

##############################################

from fastapi import FastAPI
import uvicorn

app = FastAPI()

@app.get("/")
def greet_json():
    return {"Hello": "World!"}

@app.get("/message")
async def message(input: str):
    return generate_response(input)

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=7860)