Create app.py

app.py

@@ -0,0 +1,132 @@
+import gradio as gr
+import edge_tts
+import asyncio
+import tempfile
+import os
+from huggingface_hub import InferenceClient
+import re
+from streaming_stt_nemo import Model
+import torch
+import random
+from openai import OpenAI
+
+default_lang = "en"
+
+engines = { default_lang: Model(default_lang) }
+
+def transcribe(audio):
+    lang = "en"
+    model = engines[lang]
+    text = model.stt_file(audio)[0]
+    return text
+
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
+
+def client_fn(model):
+    if "Mixtral" in model:
+        return InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
+    elif "Llama" in model:
+        return InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")
+    elif "Mistral" in model:
+        return InferenceClient("mistralai/Mistral-7B-Instruct-v0.2")
+    elif "Phi" in model:
+        return InferenceClient("microsoft/Phi-3-mini-4k-instruct")
+    elif "Llama 3B" in model:
+        return OpenAI(
+            base_url="http://52.76.81.56:60002/v1",
+            api_key="token-abc123"
+        )
+    else: 
+        return InferenceClient("microsoft/Phi-3-mini-4k-instruct")
+
+def randomize_seed_fn(seed: int) -> int:
+    seed = random.randint(0, 999999)
+    return seed
+
+system_instructions1 = """
+[SYSTEM] Answer as Real Optimus OPTIMUS, Made by 'Jaward.' 
+Keep conversation friendly, short, clear, and concise. 
+Avoid unnecessary introductions and answer the user's questions directly. 
+Respond in a normal, conversational manner while being friendly and helpful.
+[USER]
+"""
+
+def models(text, model="Mixtral 8x7B", seed=42):
+    seed = int(randomize_seed_fn(seed))
+    generator = torch.Generator().manual_seed(seed)  
+    
+    client = client_fn(model)
+    
+    if "Llama 3B" in model:
+        messages = [
+            {"role": "system", "content": system_instructions1},
+            {"role": "user", "content": text}
+        ]
+        completion = client.chat.completions.create(
+            model="/data/shared/huggingface/hub/models--meta-llama--Meta-Llama-3-8B-Instruct/snapshots/c4a54320a52ed5f88b7a2f84496903ea4ff07b45/",
+            messages=messages
+        )
+        return completion.choices[0].message.content
+    else:
+        generate_kwargs = dict(
+            max_new_tokens=300,
+            seed=seed
+        )    
+        formatted_prompt = system_instructions1 + text + "[OPTIMUS]"
+        stream = client.text_generation(
+            formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+        output = ""
+        for response in stream:
+            if not response.token.text == "</s>":
+                output += response.token.text
+        return output
+
+async def respond(audio, model, seed):
+    user = transcribe(audio)
+    reply = models(user, model, seed)
+    communicate = edge_tts.Communicate(reply)
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
+        tmp_path = tmp_file.name
+        await communicate.save(tmp_path)
+    yield tmp_path
+
+DESCRIPTION = """ # <center><b>OPTIMUS⚡</b></center>
+        ### <center>A personal Assistant of Jaward for YOU
+        ### <center>Voice Chat with your personal Assistant</center>
+        """
+
+with gr.Blocks(css="style.css") as demo:    
+    gr.Markdown(DESCRIPTION)
+    with gr.Row():
+        select = gr.Dropdown([
+            'Mixtral 8x7B',
+            'Llama 3 8B',
+            'Mistral 7B v0.3',
+            'Phi 3 mini',
+            'Llama 3B'
+        ],
+        value="Mistral 7B v0.3",
+        label="Model"
+        )
+        seed = gr.Slider(
+        label="Seed",
+        minimum=0,
+        maximum=999999,
+        step=1,
+        value=0,
+        visible=False
+        )
+        input = gr.Audio(label="User", sources="microphone", type="filepath", waveform_options=False)
+        output = gr.Audio(label="AI", type="filepath",
+                        interactive=False,
+                        autoplay=True,
+                        elem_classes="audio")
+        gr.Interface(
+            batch=True,
+            max_batch_size=10, 
+            fn=respond, 
+            inputs=[input, select, seed],
+            outputs=[output], live=True)  
+
+if __name__ == "__main__":
+    demo.queue(max_size=200).launch()