Update app.py and requirements.txt

app.py

@@ -1,6 +1,8 @@
+import asyncio
 import logging
 import logging.handlers
 import queue
+import threading
 import urllib.request
 from pathlib import Path
 from typing import List, NamedTuple
@@ -12,13 +14,16 @@ except ImportError:
 
 import av
 import cv2
+import matplotlib.pyplot as plt
 import numpy as np
+import pydub
 import streamlit as st
 from aiortc.contrib.media import MediaPlayer
 
 from streamlit_webrtc import (
+    AudioProcessorBase,
     ClientSettings,
-    VideoTransformerBase,
+    VideoProcessorBase,
     WebRtcMode,
     webrtc_streamer,
 )
@@ -87,18 +92,28 @@ def main():
     video_filters_page = (
         "Real time video transform with simple OpenCV filters (sendrecv)"
     )
+    audio_filter_page = "Real time audio filter (sendrecv)"
+    delayed_echo_page = "Delayed echo (sendrecv)"
     streaming_page = (
         "Consuming media files on server-side and streaming it to browser (recvonly)"
     )
-    sendonly_page = "WebRTC is sendonly and images are shown via st.image() (sendonly)"
-    loopback_page = "Simple video loopback (sendrecv)"
+    video_sendonly_page = (
+        "WebRTC is sendonly and images are shown via st.image() (sendonly)"
+    )
+    audio_sendonly_page = (
+        "WebRTC is sendonly and audio frames are visualized with matplotlib (sendonly)"
+    )
+    loopback_page = "Simple video and audio loopback (sendrecv)"
     app_mode = st.sidebar.selectbox(
         "Choose the app mode",
         [
             object_detection_page,
             video_filters_page,
+            audio_filter_page,
+            delayed_echo_page,
             streaming_page,
-            sendonly_page,
+            video_sendonly_page,
+            audio_sendonly_page,
             loopback_page,
         ],
     )
@@ -108,13 +123,24 @@ def main():
         app_video_filters()
     elif app_mode == object_detection_page:
         app_object_detection()
+    elif app_mode == audio_filter_page:
+        app_audio_filter()
+    elif app_mode == delayed_echo_page:
+        app_delayed_echo()
     elif app_mode == streaming_page:
         app_streaming()
-    elif app_mode == sendonly_page:
-        app_sendonly()
+    elif app_mode == video_sendonly_page:
+        app_sendonly_video()
+    elif app_mode == audio_sendonly_page:
+        app_sendonly_audio()
     elif app_mode == loopback_page:
         app_loopback()
 
+    logger.debug("=== Alive threads ===")
+    for thread in threading.enumerate():
+        if thread.is_alive():
+            logger.debug(f"  {thread.name} ({thread.ident})")
+
 
 def app_loopback():
     """ Simple video loopback """
@@ -122,20 +148,20 @@ def app_loopback():
         key="loopback",
         mode=WebRtcMode.SENDRECV,
         client_settings=WEBRTC_CLIENT_SETTINGS,
-        video_transformer_factory=None,  # NoOp
+        video_processor_factory=None,  # NoOp
     )
 
 
 def app_video_filters():
     """ Video transforms with OpenCV """
 
-    class OpenCVVideoTransformer(VideoTransformerBase):
+    class OpenCVVideoProcessor(VideoProcessorBase):
         type: Literal["noop", "cartoon", "edges", "rotate"]
 
         def __init__(self) -> None:
             self.type = "noop"
 
-        def transform(self, frame: av.VideoFrame) -> av.VideoFrame:
+        def recv(self, frame: av.VideoFrame) -> av.VideoFrame:
             img = frame.to_ndarray(format="bgr24")
 
             if self.type == "noop":
@@ -170,18 +196,18 @@ def app_video_filters():
                 M = cv2.getRotationMatrix2D((cols / 2, rows / 2), frame.time * 45, 1)
                 img = cv2.warpAffine(img, M, (cols, rows))
 
-            return img
+            return av.VideoFrame.from_ndarray(img, format="bgr24")
 
     webrtc_ctx = webrtc_streamer(
         key="opencv-filter",
         mode=WebRtcMode.SENDRECV,
         client_settings=WEBRTC_CLIENT_SETTINGS,
-        video_transformer_factory=OpenCVVideoTransformer,
-        async_transform=True,
+        video_processor_factory=OpenCVVideoProcessor,
+        async_processing=True,
     )
 
-    if webrtc_ctx.video_transformer:
-        webrtc_ctx.video_transformer.type = st.radio(
+    if webrtc_ctx.video_processor:
+        webrtc_ctx.video_processor.type = st.radio(
             "Select transform type", ("noop", "cartoon", "edges", "rotate")
         )
 
@@ -192,6 +218,82 @@ def app_video_filters():
     )
 
 
+def app_audio_filter():
+    DEFAULT_GAIN = 1.0
+
+    class AudioProcessor(AudioProcessorBase):
+        gain = DEFAULT_GAIN
+
+        def recv(self, frame: av.AudioFrame) -> av.AudioFrame:
+            raw_samples = frame.to_ndarray()
+            sound = pydub.AudioSegment(
+                data=raw_samples.tobytes(),
+                sample_width=frame.format.bytes,
+                frame_rate=frame.sample_rate,
+                channels=len(frame.layout.channels),
+            )
+
+            sound = sound.apply_gain(self.gain)
+
+            # Ref: https://github.com/jiaaro/pydub/blob/master/API.markdown#audiosegmentget_array_of_samples  # noqa
+            channel_sounds = sound.split_to_mono()
+            channel_samples = [s.get_array_of_samples() for s in channel_sounds]
+            new_samples: np.ndarray = np.array(channel_samples).T
+            new_samples = new_samples.reshape(raw_samples.shape)
+
+            new_frame = av.AudioFrame.from_ndarray(
+                new_samples, layout=frame.layout.name
+            )
+            new_frame.sample_rate = frame.sample_rate
+            return new_frame
+
+    webrtc_ctx = webrtc_streamer(
+        key="audio-filter",
+        mode=WebRtcMode.SENDRECV,
+        client_settings=WEBRTC_CLIENT_SETTINGS,
+        audio_processor_factory=AudioProcessor,
+        async_processing=True,
+    )
+
+    if webrtc_ctx.audio_processor:
+        webrtc_ctx.audio_processor.gain = st.slider(
+            "Gain", -10.0, +20.0, DEFAULT_GAIN, 0.05
+        )
+
+
+def app_delayed_echo():
+    DEFAULT_DELAY = 1.0
+
+    class VideoProcessor(VideoProcessorBase):
+        delay = DEFAULT_DELAY
+
+        async def recv_queued(self, frames: List[av.VideoFrame]) -> List[av.VideoFrame]:
+            logger.debug("Delay:", self.delay)
+            await asyncio.sleep(self.delay)
+            return frames
+
+    class AudioProcessor(AudioProcessorBase):
+        delay = DEFAULT_DELAY
+
+        async def recv_queued(self, frames: List[av.AudioFrame]) -> List[av.AudioFrame]:
+            await asyncio.sleep(self.delay)
+            return frames
+
+    webrtc_ctx = webrtc_streamer(
+        key="delay",
+        mode=WebRtcMode.SENDRECV,
+        client_settings=WEBRTC_CLIENT_SETTINGS,
+        video_processor_factory=VideoProcessor,
+        audio_processor_factory=AudioProcessor,
+        async_processing=True,
+    )
+
+    if webrtc_ctx.video_processor and webrtc_ctx.audio_processor:
+        delay = st.slider("Delay", 0.0, 5.0, DEFAULT_DELAY, 0.05)
+        webrtc_ctx.video_processor.delay = delay
+        webrtc_ctx.audio_processor.delay = delay
+
+
 def app_object_detection():
     """Object detection demo with MobileNet SSD.
     This model and code are based on
@@ -236,7 +338,7 @@ def app_object_detection():
         name: str
         prob: float
 
-    class MobileNetSSDVideoTransformer(VideoTransformerBase):
+    class MobileNetSSDVideoProcessor(VideoProcessorBase):
         confidence_threshold: float
         result_queue: "queue.Queue[List[Detection]]"
 
@@ -280,7 +382,7 @@ def app_object_detection():
                     )
             return image, result
 
-        def transform(self, frame: av.VideoFrame) -> np.ndarray:
+        def recv(self, frame: av.VideoFrame) -> av.VideoFrame:
             image = frame.to_ndarray(format="bgr24")
             blob = cv2.dnn.blobFromImage(
                 cv2.resize(image, (300, 300)), 0.007843, (300, 300), 127.5
@@ -289,25 +391,25 @@ def app_object_detection():
             detections = self._net.forward()
             annotated_image, result = self._annotate_image(image, detections)
 
-            # NOTE: This `transform` method is called in another thread,
+            # NOTE: This `recv` method is called in another thread,
             # so it must be thread-safe.
             self.result_queue.put(result)
 
-            return annotated_image
+            return av.VideoFrame.from_ndarray(annotated_image, format="bgr24")
 
     webrtc_ctx = webrtc_streamer(
         key="object-detection",
         mode=WebRtcMode.SENDRECV,
         client_settings=WEBRTC_CLIENT_SETTINGS,
-        video_transformer_factory=MobileNetSSDVideoTransformer,
-        async_transform=True,
+        video_processor_factory=MobileNetSSDVideoProcessor,
+        async_processing=True,
     )
 
     confidence_threshold = st.slider(
         "Confidence threshold", 0.0, 1.0, DEFAULT_CONFIDENCE_THRESHOLD, 0.05
     )
-    if webrtc_ctx.video_transformer:
-        webrtc_ctx.video_transformer.confidence_threshold = confidence_threshold
+    if webrtc_ctx.video_processor:
+        webrtc_ctx.video_processor.confidence_threshold = confidence_threshold
 
     if st.checkbox("Show the detected labels", value=True):
         if webrtc_ctx.state.playing:
@@ -318,9 +420,9 @@ def app_object_detection():
             # Then the rendered video frames and the labels displayed here
             # are not strictly synchronized.
             while True:
-                if webrtc_ctx.video_transformer:
+                if webrtc_ctx.video_processor:
                     try:
-                        result = webrtc_ctx.video_transformer.result_queue.get(
+                        result = webrtc_ctx.video_processor.result_queue.get(
                             timeout=1.0
                         )
                     except queue.Empty:
@@ -393,7 +495,7 @@ def app_streaming():
     )
 
 
-def app_sendonly():
+def app_sendonly_video():
     """A sample to use WebRTC in sendonly mode to transfer frames
     from the browser to the server and to render frames via `st.image`."""
     webrtc_ctx = webrtc_streamer(
@@ -402,28 +504,110 @@ def app_sendonly():
         client_settings=WEBRTC_CLIENT_SETTINGS,
     )
 
+    image_place = st.empty()
+
     if webrtc_ctx.video_receiver:
-        image_loc = st.empty()
         while True:
             try:
-                frame = webrtc_ctx.video_receiver.get_frame(timeout=1)
+                video_frame = webrtc_ctx.video_receiver.get_frame(timeout=1)
             except queue.Empty:
-                print("Queue is empty. Stop the loop.")
-                webrtc_ctx.video_receiver.stop()
+                logger.warning("Queue is empty. Abort.")
                 break
 
-            img_rgb = frame.to_ndarray(format="rgb24")
-            image_loc.image(img_rgb)
+            img_rgb = video_frame.to_ndarray(format="rgb24")
+            image_place.image(img_rgb)
+
+
+def app_sendonly_audio():
+    """A sample to use WebRTC in sendonly mode to transfer audio frames
+    from the browser to the server and visualize them with matplotlib
+    and `st.pyplog`."""
+    webrtc_ctx = webrtc_streamer(
+        key="loopback",
+        mode=WebRtcMode.SENDONLY,
+        audio_receiver_size=64,
+        client_settings=WEBRTC_CLIENT_SETTINGS,
+    )
+
+    fig_place = st.empty()
+
+    fig, [ax_time, ax_freq] = plt.subplots(
+        2, 1, gridspec_kw={"top": 1.5, "bottom": 0.2}
+    )
+
+    sound_window_len = 5000  # 5s
+    sound_window_buffer = None
+    while True:
+        if webrtc_ctx.audio_receiver:
+            try:
+                audio_frames = webrtc_ctx.audio_receiver.get_frames(timeout=1)
+            except queue.Empty:
+                logger.warning("Queue is empty. Abort.")
+                break
+
+            sound_chunk = pydub.AudioSegment.empty()
+            for audio_frame in audio_frames:
+                sound = pydub.AudioSegment(
+                    data=audio_frame.to_ndarray().tobytes(),
+                    sample_width=audio_frame.format.bytes,
+                    frame_rate=audio_frame.sample_rate,
+                    channels=len(audio_frame.layout.channels),
+                )
+                sound_chunk += sound
+
+            if len(sound_chunk) > 0:
+                if sound_window_buffer is None:
+                    sound_window_buffer = pydub.AudioSegment.silent(
+                        duration=sound_window_len
+                    )
+
+                sound_window_buffer += sound_chunk
+                if len(sound_window_buffer) > sound_window_len:
+                    sound_window_buffer = sound_window_buffer[-sound_window_len:]
+
+            if sound_window_buffer:
+                # Ref: https://own-search-and-study.xyz/2017/10/27/python%E3%82%92%E4%BD%BF%E3%81%A3%E3%81%A6%E9%9F%B3%E5%A3%B0%E3%83%87%E3%83%BC%E3%82%BF%E3%81%8B%E3%82%89%E3%82%B9%E3%83%9A%E3%82%AF%E3%83%88%E3%83%AD%E3%82%B0%E3%83%A9%E3%83%A0%E3%82%92%E4%BD%9C/  # noqa
+                sound_window_buffer = sound_window_buffer.set_channels(
+                    1
+                )  # Stereo to mono
+                sample = np.array(sound_window_buffer.get_array_of_samples())
+
+                ax_time.cla()
+                times = (np.arange(-len(sample), 0)) / sound_window_buffer.frame_rate
+                ax_time.plot(times, sample)
+                ax_time.set_xlabel("Time")
+                ax_time.set_ylabel("Magnitude")
+
+                spec = np.fft.fft(sample)
+                freq = np.fft.fftfreq(sample.shape[0], 1.0 / sound_chunk.frame_rate)
+                freq = freq[: int(freq.shape[0] / 2)]
+                spec = spec[: int(spec.shape[0] / 2)]
+                spec[0] = spec[0] / 2
+
+                ax_freq.cla()
+                ax_freq.plot(freq, np.abs(spec))
+                ax_freq.set_xlabel("Frequency")
+                ax_freq.set_yscale("log")
+                ax_freq.set_ylabel("Magnitude")
+
+                fig_place.pyplot(fig)
+        else:
+            logger.warning("AudioReciver is not set. Abort.")
+            break
 
 
 if __name__ == "__main__":
+    import os
+
+    DEBUG = os.environ.get("DEBUG", "false").lower() not in ["false", "no", "0"]
+
     logging.basicConfig(
         format="[%(asctime)s] %(levelname)7s from %(name)s in %(pathname)s:%(lineno)d: "
         "%(message)s",
         force=True,
     )
 
-    logger.setLevel(level=logging.DEBUG)
+    logger.setLevel(level=logging.DEBUG if DEBUG else logging.INFO)
 
     st_webrtc_logger = logging.getLogger("streamlit_webrtc")
     st_webrtc_logger.setLevel(logging.DEBUG)

requirements.txt

@@ -1,7 +1,9 @@
-aiortc==1.2.0
+aiortc==1.1.2
 av==8.0.3
-numpy==1.20.2
+matplotlib==3.4.2
+numpy==1.19.5
 opencv_python==4.5.1.48
-streamlit==0.80.0
-streamlit_webrtc==0.11.0
+pydub==0.25.1
+streamlit==0.75.0
+streamlit_webrtc==0.20.0
 typing_extensions==3.7.4.3