test

libs/transformer/get_chat_transformer.py

@@ -0,0 +1,17 @@
+from sklearn import pipeline
+from transformers import AutoProcessor, AutoModelForImageTextToText
+
+
+def get_chat_transformers(messages: list):
+    model = AutoModelForImageTextToText.from_pretrained("Xkev/Llama-3.2V-11B-cot")
+
+    pipe = pipeline("text-generation",
+    model=model,
+
+    device_map="auto",)
+
+    pipe = pipeline("text-generation", model="meta-llama/Llama-3.1-8B-Instruct")
+
+    outputs = pipe(messages)
+
+    return outputs[0]["generated_text"][-1]

routers/get_chatrespone.py

@@ -12,6 +12,7 @@ from fastapi.responses import StreamingResponse
 from langchain_ollama import ChatOllama, OllamaLLM
 
 from libs.transformer.get_chat_gradio import get_chat_gradio
+from libs.transformer.get_chat_transformer import get_chat_transformers
 
 load_dotenv()
 HUGGINGFACEHUB_API_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN", )
@@ -42,9 +43,18 @@ async def get_chat_respone(body: ChatInputForm, api_key: str = Depends(get_api_k
         #     ("human", body.textInput)
         # ]
 
+        messages = [
+            {"role": "system", "content": prompt},
+            {"role": "user", "content": body.textInput},
+        ]
+
         # response = llm.stream(messages)
 
-        response = get_chat_gradio(body.textInput)
+        # response = get_chat_gradio(body.textInput)
+
+        response = get_chat_transformers(messages)
+
+        print(response)
 
         return StreamingResponse(get_response(response), media_type='text/event-stream')
     except Exception:

routers/get_transcript_transformer.py

@@ -30,7 +30,7 @@ def get_transcript(audio_path: str, model_size: str = "distil-whisper/distil-sma
         convert_to_audio(audio_path.strip(), output_file)
 
     try:
-        text, chunks = get_transcribe_transformers(output_file, model_size)
+        text, chunks = get_transcribe_transformers(output_file, "Xkev/Llama-3.2V-11B-cot")
     except Exception as error:
         raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"error>>>: {error}")
     finally: