Update app.py

app.py

@@ -10,6 +10,12 @@ import torch
 import random
 from openai import OpenAI
 import subprocess
+import threading
+import queue
+import sounddevice as sd
+import numpy as np
+import wave
+import sys
 
 default_lang = "en"
 
@@ -112,18 +118,90 @@ def models(text, model="Llama 3 8B Service", seed=42):
         
         return output
 
-async def respond(audio, model, seed):
-    if audio is None:
-        return None
-    user = transcribe(audio)
-    if not user:
-        return None
-    reply = models(user, model, seed)
-    communicate = edge_tts.Communicate(reply, voice="en-US-ChristopherNeural")
+# New global variables for audio processing
+RATE = 16000
+CHUNK = int(RATE / 10)  # 100ms
+audio_queue = queue.Queue()
+is_listening = False
+
+def audio_callback(indata, frames, time, status):
+    if status:
+        print(status, file=sys.stderr)
+    audio_queue.put(indata.copy())
+
+def process_audio_stream(model, seed):
+    global is_listening
+    audio_buffer = []
+    silence_threshold = 0.01
+    silence_duration = 0
+    max_silence = 2  # seconds
+
+    while True:
+        if not is_listening:
+            audio_buffer.clear()
+            silence_duration = 0
+            audio_queue.queue.clear()
+            continue
+
+        try:
+            chunk = audio_queue.get(timeout=1)
+            audio_buffer.append(chunk)
+
+            # Check for silence
+            if np.abs(chunk).mean() < silence_threshold:
+                silence_duration += CHUNK / RATE
+            else:
+                silence_duration = 0
+
+            if silence_duration > max_silence:
+                # Process the buffered audio
+                audio_data = np.concatenate(audio_buffer)
+                with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
+                    tmp_path = tmp_file.name
+                    with wave.open(tmp_path, 'wb') as wf:
+                        wf.setnchannels(1)
+                        wf.setsampwidth(2)
+                        wf.setframerate(RATE)
+                        wf.writeframes((audio_data * 32767).astype(np.int16).tobytes())
+
+                # Transcribe and process
+                user_input = transcribe(tmp_path)
+                if user_input:
+                    is_listening = False
+                    reply = models(user_input, model, seed)
+                    asyncio.run(respond_and_play(reply))
+                    is_listening = True
+
+                # Clear the buffer
+                audio_buffer.clear()
+                silence_duration = 0
+
+        except queue.Empty:
+            pass
+
+async def respond_and_play(text):
+    communicate = edge_tts.Communicate(text, voice="en-US-ChristopherNeural")
     with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
         tmp_path = tmp_file.name
         await communicate.save(tmp_path)
-    return tmp_path
+    
+    # Play the audio
+    with wave.open(tmp_path, 'rb') as wf:
+        data = wf.readframes(wf.getnframes())
+        sd.play(np.frombuffer(data, dtype=np.int16), wf.getframerate())
+        sd.wait()
+
+def start_listening(model, seed):
+    global is_listening
+    is_listening = True
+    threading.Thread(target=process_audio_stream, args=(model, seed), daemon=True).start()
+    with sd.InputStream(callback=audio_callback, channels=1, samplerate=RATE, blocksize=CHUNK):
+        while is_listening:
+            sd.sleep(100)
+
+def stop_listening():
+    global is_listening
+    is_listening = False
 
 # Supported languages for seamless-expressive
 LANGUAGE_CODES = {
@@ -198,17 +276,21 @@ with gr.Blocks(css="style.css") as demo:
             value=0,
             visible=False
             )
-            input = gr.Audio(label="User", sources=["microphone"], type="filepath")
-            output = gr.Audio(label="AI", type="filepath",
-                            interactive=False,
-                            autoplay=True,
-                            elem_classes="audio")
+            start_button = gr.Button("Start Listening")
+            stop_button = gr.Button("Stop Listening")
+            status = gr.Markdown("Status: Not listening")
 
-            gr.Interface(
-                fn=respond, 
-                inputs=[input, select, seed],
-                outputs=[output],
-                live=True
+            start_button.click(
+                fn=lambda model, seed: start_listening(model, seed),
+                inputs=[select, seed],
+                outputs=[status],
+                _js="() => {document.getElementById('status').textContent = 'Status: Listening'}"
+            )
+            stop_button.click(
+                fn=stop_listening,
+                inputs=[],
+                outputs=[status],
+                _js="() => {document.getElementById('status').textContent = 'Status: Not listening'}"
             )
         
         with gr.TabItem("Speech Translation") as speech_translation:
@@ -230,14 +312,6 @@ with gr.Blocks(css="style.css") as demo:
                 live=True
             )
 
-    # clear_button = gr.Button("Clear")
-    # clear_button.click(
-    #     fn=clear_history,
-    #     inputs=[],
-    #     outputs=[input, output, input_audio, output_audio],
-    #     api_name="clear"
-    # )
-
     voice_assistant.select(fn=voice_assistant_tab, inputs=None, outputs=description)
     speech_translation.select(fn=speech_translation_tab, inputs=None, outputs=description)