updare app & requirements

app.py

@@ -1,10 +1,11 @@
 from fastapi import FastAPI
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
 import torch
 import os
 import uvicorn
+import threading
 
 app = FastAPI()
 
@@ -39,28 +40,45 @@ class ChatRequest(BaseModel):
 # Generator untuk streaming token
 def generate_stream(prompt, max_new_tokens=128):
     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    streamer = tokenizer.as_target_tokenizer()
+    # streamer = tokenizer.as_target_tokenizer()
+
+    # # pakai generate incremental
+    # with torch.no_grad():
+    #     output_ids = model.generate(
+    #         **inputs,
+    #         max_new_tokens=max_new_tokens,
+    #         do_sample=True,
+    #         top_p=0.9,
+    #         temperature=0.7
+    #     )[0]
+
+    # # Ambil hasil tanpa input
+    # generated_tokens = output_ids[inputs["input_ids"].shape[1]:]
+
+    # for tok in generated_tokens:
+    #     text = tokenizer.decode(tok, skip_special_tokens=True)
+    #     if text.strip():
+    #         yield text
+            
+    streamer = TextIteratorStreamer(tokenizer, skip_special_tokens=True)
+    generation_kwargs = dict(
+        **inputs,
+        max_new_tokens=max_new_tokens,
+        temperature=0.7,
+        streamer=streamer
+    )
 
-    # pakai generate incremental
-    with torch.no_grad():
-        output_ids = model.generate(
-            **inputs,
-            max_new_tokens=max_new_tokens,
-            do_sample=True,
-            top_p=0.9,
-            temperature=0.7
-        )[0]
+    thread = threading.Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
 
-    # Ambil hasil tanpa input
-    generated_tokens = output_ids[inputs["input_ids"].shape[1]:]
+    def token_stream():
+        for token in streamer:
+            yield token
 
-    for tok in generated_tokens:
-        text = tokenizer.decode(tok, skip_special_tokens=True)
-        if text.strip():
-            yield text
+    return StreamingResponse(token_stream(), media_type="text/plain")
 
 
-@app.post("/strean")
+@app.post("/stream")
 async def chat(req: ChatRequest):
     # Format prompt sesuai chat template
     text = tokenizer.apply_chat_template(

app_quantized.py

@@ -0,0 +1,45 @@
+from fastapi import FastAPI
+from pydantic import BaseModel
+from llama_cpp import Llama
+import os
+import uvicorn
+
+app = FastAPI()
+
+# --- Konfigurasi Model ---
+# Pastikan sudah download model GGUF dari Hugging Face, contoh:
+# https://huggingface.co/Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF
+MODEL_PATH = "./Qwen2.5-Coder-0.5B-Instruct-Q4_K_M.gguf"  # ganti sesuai file lokal
+
+llm = Llama(
+    model_path=MODEL_PATH,
+    n_ctx=2048,          # konteks token
+    n_threads=4,         # sesuaikan dengan jumlah CPU core
+    n_batch=512          # batch size
+)
+
+# --- Schema Request ---
+class ChatRequest(BaseModel):
+    prompt: str
+    max_new_tokens: int = 256
+
+# --- Endpoint Chat ---
+@app.post("/chat")
+def chat(req: ChatRequest):
+    output = llm(
+        req.prompt,
+        max_tokens=req.max_new_tokens,
+        stop=["</s>", "User:", "Assistant:"],
+        echo=False
+    )
+    response = output["choices"][0]["text"].strip()
+    return {"response": response}
+
+# --- Root Endpoint ---
+@app.get("/")
+def root():
+    return {"message": "Qwen GGUF FastAPI running 🚀"}
+
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 7860))
+    uvicorn.run("app:app", host="0.0.0.0", port=port)

requirements.txt

@@ -8,4 +8,5 @@ einops
 transformers_stream_generator
 scipy
 sentencepiece
-optimum
+optimum
+threading