From e7f3e63c79f609c070968f13b956cf89222cd3c6 Mon Sep 17 00:00:00 2001 From: freddyaboulton Date: Tue, 22 Oct 2024 16:24:21 -0700 Subject: [PATCH] make code --- backend/gradio_webrtc/__init__.py | 2 +- backend/gradio_webrtc/utils.py | 49 ++-- backend/gradio_webrtc/webrtc.py | 132 +++++++---- demo/app.py | 282 +++++++++++++++++++++--- demo/app_orig.py | 11 +- demo/audio_out.py | 8 +- demo/audio_out_2.py | 6 +- demo/inference.py | 5 +- demo/space.py | 3 +- demo/utils.py | 2 +- demo/video_out.py | 6 +- demo/video_out_stream.py | 6 +- frontend/Index.svelte | 2 + frontend/shared/AudioWave.svelte | 13 +- frontend/shared/InteractiveAudio.svelte | 10 +- frontend/shared/InteractiveVideo.svelte | 2 + frontend/shared/StaticAudio.svelte | 32 ++- frontend/shared/Webcam.svelte | 3 +- frontend/shared/stream_utils.ts | 7 +- frontend/shared/webrtc_utils.ts | 2 - 20 files changed, 427 insertions(+), 156 deletions(-) diff --git a/backend/gradio_webrtc/__init__.py b/backend/gradio_webrtc/__init__.py index 2903fed..af3f6cd 100644 --- a/backend/gradio_webrtc/__init__.py +++ b/backend/gradio_webrtc/__init__.py @@ -1,3 +1,3 @@ -from .webrtc import WebRTC, StreamHandler +from .webrtc import StreamHandler, WebRTC __all__ = ["StreamHandler", "WebRTC"] diff --git a/backend/gradio_webrtc/utils.py b/backend/gradio_webrtc/utils.py index d7f6d4f..1f9cb55 100644 --- a/backend/gradio_webrtc/utils.py +++ b/backend/gradio_webrtc/utils.py @@ -2,7 +2,6 @@ import asyncio import fractions import logging import threading -import time from typing import Callable import av @@ -15,35 +14,44 @@ AUDIO_PTIME = 0.020 def player_worker_decode( loop, - next: Callable, + next_frame: Callable, queue: asyncio.Queue, - throttle_playback: bool, thread_quit: threading.Event, + quit_on_none: bool = False, + sample_rate: int = 48000, + frame_size: int = int(48000 * AUDIO_PTIME), ): - audio_sample_rate = 48000 audio_samples = 0 - audio_time_base = fractions.Fraction(1, audio_sample_rate) - audio_resampler = av.AudioResampler( + audio_time_base = fractions.Fraction(1, sample_rate) + audio_resampler = av.AudioResampler( # type: ignore format="s16", layout="stereo", - rate=audio_sample_rate, - frame_size=int(audio_sample_rate * AUDIO_PTIME), + rate=sample_rate, + frame_size=frame_size, ) - frame_time = None - start_time = time.time() - while not thread_quit.is_set(): - frame = next() - logger.debug("emitted %s", frame) - # read up to 1 second ahead - if throttle_playback: - elapsed_time = time.time() - start_time - if frame_time and frame_time > elapsed_time + 1: - time.sleep(0.1) - sample_rate, audio_array = frame + frame = next_frame() + if frame is None: + if quit_on_none: + asyncio.run_coroutine_threadsafe(queue.put(None), loop) + continue + + if len(frame) == 2: + sample_rate, audio_array = frame + layout = "mono" + elif len(frame) == 3: + sample_rate, audio_array, layout = frame + + logger.debug( + "received array with shape %s sample rate %s layout %s", + audio_array.shape, + sample_rate, + layout, + ) format = "s16" if audio_array.dtype == "int16" else "fltp" - frame = av.AudioFrame.from_ndarray(audio_array, format=format, layout="stereo") + + frame = av.AudioFrame.from_ndarray(audio_array, format=format, layout=layout) # type: ignore frame.sample_rate = sample_rate for frame in audio_resampler.resample(frame): # fix timestamps @@ -51,5 +59,4 @@ def player_worker_decode( frame.time_base = audio_time_base audio_samples += frame.samples - frame_time = frame.time asyncio.run_coroutine_threadsafe(queue.put(frame), loop) diff --git a/backend/gradio_webrtc/webrtc.py b/backend/gradio_webrtc/webrtc.py index 955a541..92faa92 100644 --- a/backend/gradio_webrtc/webrtc.py +++ b/backend/gradio_webrtc/webrtc.py @@ -3,24 +3,25 @@ from __future__ import annotations import asyncio -from abc import ABC, abstractmethod import logging import threading import time import traceback +from abc import ABC, abstractmethod from collections.abc import Callable from typing import TYPE_CHECKING, Any, Generator, Literal, Sequence, cast import anyio.to_thread +import av import numpy as np from aiortc import ( AudioStreamTrack, + MediaStreamTrack, RTCPeerConnection, RTCSessionDescription, VideoStreamTrack, - MediaStreamTrack, ) -from aiortc.contrib.media import MediaRelay, AudioFrame, VideoFrame # type: ignore +from aiortc.contrib.media import AudioFrame, MediaRelay, VideoFrame # type: ignore from aiortc.mediastreams import MediaStreamError from gradio import wasm_utils from gradio.components.base import Component, server @@ -104,6 +105,27 @@ class VideoCallback(VideoStreamTrack): class StreamHandler(ABC): + def __init__( + self, + expected_layout: Literal["mono", "stereo"] = "mono", + output_sample_rate: int = 24000, + output_frame_size: int = 960, + ) -> None: + self.expected_layout = expected_layout + self.output_sample_rate = output_sample_rate + self.output_frame_size = output_frame_size + self._resampler = None + + def resample(self, frame: AudioFrame) -> Generator[AudioFrame, None, None]: + if self._resampler is None: + self._resampler = av.AudioResampler( # type: ignore + format="s16", + layout=self.expected_layout, + rate=frame.sample_rate, + frame_size=frame.samples, + ) + yield from self._resampler.resample(frame) + @abstractmethod def receive(self, frame: tuple[int, np.ndarray] | np.ndarray) -> None: pass @@ -124,24 +146,27 @@ class AudioCallback(AudioStreamTrack): self.track = track self.event_handler = event_handler self.current_timestamp = 0 - self.latest_args = "not_set" + self.latest_args: str | list[Any] = "not_set" self.queue = asyncio.Queue() self.thread_quit = threading.Event() self.__thread = None self._start: float | None = None self.has_started = False + self.last_timestamp = 0 super().__init__() async def process_input_frames(self) -> None: while not self.thread_quit.is_set(): try: frame = cast(AudioFrame, await self.track.recv()) - numpy_array = frame.to_ndarray() - logger.debug("numpy array shape %s", numpy_array.shape) - await anyio.to_thread.run_sync( - self.event_handler.receive, (frame.sample_rate, numpy_array) - ) - except MediaStreamError: + for frame in self.event_handler.resample(frame): + numpy_array = frame.to_ndarray() + logger.debug("numpy array shape %s", numpy_array.shape) + await anyio.to_thread.run_sync( + self.event_handler.receive, (frame.sample_rate, numpy_array) + ) + except MediaStreamError as e: + print("MediaStreamError", e) break def start(self): @@ -154,8 +179,10 @@ class AudioCallback(AudioStreamTrack): asyncio.get_event_loop(), self.event_handler.emit, self.queue, - True, self.thread_quit, + False, + self.event_handler.output_sample_rate, + self.event_handler.output_frame_size, ), ) self.__thread.start() @@ -167,23 +194,25 @@ class AudioCallback(AudioStreamTrack): raise MediaStreamError self.start() - data = await self.queue.get() - logger.debug("data %s", data) - if data is None: - self.stop() - return + frame = await self.queue.get() + logger.debug("frame %s", frame) - data_time = data.time + data_time = frame.time + + if time.time() - self.last_timestamp > 10 * ( + self.event_handler.output_frame_size + / self.event_handler.output_sample_rate + ): + self._start = None # control playback rate - if data_time is not None: - if self._start is None: - self._start = time.time() - data_time - else: - wait = self._start + data_time - time.time() - await asyncio.sleep(wait) - - return data + if self._start is None: + self._start = time.time() - data_time + else: + wait = self._start + data_time - time.time() + await asyncio.sleep(wait) + self.last_timestamp = time.time() + return frame except Exception as e: logger.debug(e) exec = traceback.format_exc() @@ -210,6 +239,7 @@ class ServerToClientVideo(VideoStreamTrack): ) -> None: super().__init__() # don't forget this! self.event_handler = event_handler + self.args_set = asyncio.Event() self.latest_args: str | list[Any] = "not_set" self.generator: Generator[Any, None, Any] | None = None @@ -219,12 +249,8 @@ class ServerToClientVideo(VideoStreamTrack): async def recv(self): try: pts, time_base = await self.next_timestamp() - if self.latest_args == "not_set": - frame = self.array_to_frame(np.zeros((480, 640, 3), dtype=np.uint8)) - frame.pts = pts - frame.time_base = time_base - return frame - elif self.generator is None: + await self.args_set.wait() + if self.generator is None: self.generator = cast( Generator[Any, None, Any], self.event_handler(*self.latest_args) ) @@ -255,7 +281,8 @@ class ServerToClientAudio(AudioStreamTrack): self.generator: Generator[Any, None, Any] | None = None self.event_handler = event_handler self.current_timestamp = 0 - self.latest_args = "not_set" + self.latest_args: str | list[Any] = "not_set" + self.args_set = threading.Event() self.queue = asyncio.Queue() self.thread_quit = threading.Event() self.__thread = None @@ -263,23 +290,15 @@ class ServerToClientAudio(AudioStreamTrack): super().__init__() def next(self) -> tuple[int, np.ndarray] | None: - import anyio - - if self.latest_args == "not_set": - return + self.args_set.wait() if self.generator is None: self.generator = self.event_handler(*self.latest_args) if self.generator is not None: try: frame = next(self.generator) return frame - except Exception as exc: - if isinstance(exc, StopIteration): - logger.debug("Stopping audio stream") - asyncio.run_coroutine_threadsafe( - self.queue.put(None), asyncio.get_event_loop() - ) - self.thread_quit.set() + except StopIteration: + pass def start(self): if self.__thread is None: @@ -290,8 +309,8 @@ class ServerToClientAudio(AudioStreamTrack): asyncio.get_event_loop(), self.next, self.queue, - False, self.thread_quit, + True, ), ) self.__thread.start() @@ -370,6 +389,7 @@ class WebRTC(Component): key: int | str | None = None, mirror_webcam: bool = True, rtc_configuration: dict[str, Any] | None = None, + track_constraints: dict[str, Any] | None = None, time_limit: float | None = None, mode: Literal["send-receive", "receive"] = "send-receive", modality: Literal["video", "audio"] = "video", @@ -412,7 +432,24 @@ class WebRTC(Component): self.rtc_configuration = rtc_configuration self.mode = mode self.modality = modality - self.event_handler: Callable | None = None + if track_constraints is None and modality == "audio": + track_constraints = { + "echoCancellation": True, + "noiseSuppression": {"exact": True}, + "autoGainControl": {"exact": True}, + "sampleRate": {"ideal": 24000}, + "sampleSize": {"ideal": 16}, + "channelCount": {"exact": 1}, + } + if track_constraints is None and modality == "video": + track_constraints = { + "facingMode": "user", + "width": {"ideal": 500}, + "height": {"ideal": 500}, + "frameRate": {"ideal": 30}, + } + self.track_constraints = track_constraints + self.event_handler: Callable | StreamHandler | None = None super().__init__( label=label, every=every, @@ -456,6 +493,7 @@ class WebRTC(Component): ) elif self.mode == "receive": self.connections[webrtc_id].latest_args = list(args) + self.connections[webrtc_id].args_set.set() # type: ignore def stream( self, @@ -534,9 +572,9 @@ class WebRTC(Component): "In the receive mode stream event, the trigger parameter must be provided" ) trigger(lambda: "start_webrtc_stream", inputs=None, outputs=self) - self.tick( + self.tick( # type: ignore self.set_output, - inputs=[self] + inputs, + inputs=[self] + list(inputs), outputs=None, concurrency_id=concurrency_id, ) diff --git a/demo/app.py b/demo/app.py index 12e3204..1ff27ba 100644 --- a/demo/app.py +++ b/demo/app.py @@ -1,22 +1,178 @@ -import logging -# Configure the root logger to WARNING to suppress debug messages from other libraries -logging.basicConfig(level=logging.WARNING) +import os -# Create a console handler -console_handler = logging.StreamHandler() -console_handler.setLevel(logging.DEBUG) +import gradio as gr -# Create a formatter -formatter = logging.Formatter("%(name)s - %(levelname)s - %(message)s") -console_handler.setFormatter(formatter) - -# Configure the logger for your specific library -logger = logging.getLogger("gradio_webrtc") -logger.setLevel(logging.DEBUG) -logger.addHandler(console_handler) +_docs = {'WebRTC': + {'description': 'Stream audio/video with WebRTC', + 'members': {'__init__': + { + 'rtc_configuration': {'type': 'dict[str, Any] | None', 'default': 'None', 'description': "The configration dictionary to pass to the RTCPeerConnection constructor. If None, the default configuration is used."}, + 'height': {'type': 'int | str | None', 'default': 'None', 'description': 'The height of the component, specified in pixels if a number is passed, or in CSS units if a string is passed. This has no effect on the preprocessed video file, but will affect the displayed video.'}, + 'width': {'type': 'int | str | None', 'default': 'None', 'description': 'The width of the component, specified in pixels if a number is passed, or in CSS units if a string is passed. This has no effect on the preprocessed video file, but will affect the displayed video.'}, + 'label': {'type': 'str | None', 'default': 'None', 'description': 'the label for this component. Appears above the component and is also used as the header if there are a table of examples for this component. If None and used in a `gr.Interface`, the label will be the name of the parameter this component is assigned to.'}, + 'show_label': {'type': 'bool | None', 'default': 'None', 'description': 'if True, will display label.'}, 'container': {'type': 'bool', 'default': 'True', 'description': 'if True, will place the component in a container - providing some extra padding around the border.'}, + 'scale': {'type': 'int | None', 'default': 'None', 'description': 'relative size compared to adjacent Components. For example if Components A and B are in a Row, and A has scale=2, and B has scale=1, A will be twice as wide as B. Should be an integer. scale applies in Rows, and to top-level Components in Blocks where fill_height=True.'}, + 'min_width': {'type': 'int', 'default': '160', 'description': 'minimum pixel width, will wrap if not sufficient screen space to satisfy this value. If a certain scale value results in this Component being narrower than min_width, the min_width parameter will be respected first.'}, + 'interactive': {'type': 'bool | None', 'default': 'None', 'description': 'if True, will allow users to upload a video; if False, can only be used to display videos. If not provided, this is inferred based on whether the component is used as an input or output.'}, 'visible': {'type': 'bool', 'default': 'True', 'description': 'if False, component will be hidden.'}, + 'elem_id': {'type': 'str | None', 'default': 'None', 'description': 'an optional string that is assigned as the id of this component in the HTML DOM. Can be used for targeting CSS styles.'}, + 'elem_classes': {'type': 'list[str] | str | None', 'default': 'None', 'description': 'an optional list of strings that are assigned as the classes of this component in the HTML DOM. Can be used for targeting CSS styles.'}, + 'render': {'type': 'bool', 'default': 'True', 'description': 'if False, component will not render be rendered in the Blocks context. Should be used if the intention is to assign event listeners now but render the component later.'}, + 'key': {'type': 'int | str | None', 'default': 'None', 'description': 'if assigned, will be used to assume identity across a re-render. Components that have the same key across a re-render will have their value preserved.'}, + 'mirror_webcam': {'type': 'bool', 'default': 'True', 'description': 'if True webcam will be mirrored. Default is True.'}, + }, + 'events': {'tick': {'type': None, 'default': None, 'description': ''}}}, '__meta__': {'additional_interfaces': {}, 'user_fn_refs': {'WebRTC': []}}} +} +abs_path = os.path.join(os.path.dirname(__file__), "css.css") + +with gr.Blocks( + css_paths=abs_path, + theme=gr.themes.Default( + font_mono=[ + gr.themes.GoogleFont("Inconsolata"), + "monospace", + ], + ), +) as demo: + gr.Markdown( +""" +

Gradio WebRTC ⚡️

+ +
+Static Badge +Static Badge +
+""", elem_classes=["md-custom"], header_links=True) + gr.Markdown( +""" +## Installation + +```bash +pip install gradio_webrtc +``` + +## Examples: +1. [Object Detection from Webcam with YOLOv10](https://huggingface.co/spaces/freddyaboulton/webrtc-yolov10n) 📷 +2. [Streaming Object Detection from Video with RT-DETR](https://huggingface.co/spaces/freddyaboulton/rt-detr-object-detection-webrtc) 🎥 +3. [Text-to-Speech](https://huggingface.co/spaces/freddyaboulton/parler-tts-streaming-webrtc) 🗣️ +4. [Conversational AI]() + +## Usage + +The WebRTC component supports the following three use cases: +1. [Streaming video from the user webcam to the server and back](#h-streaming-video-from-the-user-webcam-to-the-server-and-back) +2. [Streaming Video from the server to the client](#h-streaming-video-from-the-server-to-the-client) +3. [Streaming Audio from the server to the client](#h-streaming-audio-from-the-server-to-the-client) +4. [Streaming Audio from the client to the server and back (conversational AI)](#h-conversational-ai) + + +## Streaming Video from the User Webcam to the Server and Back + +```python +import gradio as gr +from gradio_webrtc import WebRTC + + +def detection(image, conf_threshold=0.3): + ... your detection code here ... + + +with gr.Blocks() as demo: + image = WebRTC(label="Stream", mode="send-receive", modality="video") + conf_threshold = gr.Slider( + label="Confidence Threshold", + minimum=0.0, + maximum=1.0, + step=0.05, + value=0.30, + ) + image.stream( + fn=detection, + inputs=[image, conf_threshold], + outputs=[image], time_limit=10 + ) + +if __name__ == "__main__": + demo.launch() + +``` +* Set the `mode` parameter to `send-receive` and `modality` to "video". +* The `stream` event's `fn` parameter is a function that receives the next frame from the webcam +as a **numpy array** and returns the processed frame also as a **numpy array**. +* Numpy arrays are in (height, width, 3) format where the color channels are in RGB format. +* The `inputs` parameter should be a list where the first element is the WebRTC component. The only output allowed is the WebRTC component. +* The `time_limit` parameter is the maximum time in seconds the video stream will run. If the time limit is reached, the video stream will stop. + +## Streaming Video from the server to the client + +```python +import gradio as gr +from gradio_webrtc import WebRTC +import cv2 + +def generation(): + url = "https://download.tsi.telecom-paristech.fr/gpac/dataset/dash/uhd/mux_sources/hevcds_720p30_2M.mp4" + cap = cv2.VideoCapture(url) + iterating = True + while iterating: + iterating, frame = cap.read() + yield frame + +with gr.Blocks() as demo: + output_video = WebRTC(label="Video Stream", mode="receive", modality="video") + button = gr.Button("Start", variant="primary") + output_video.stream( + fn=generation, inputs=None, outputs=[output_video], + trigger=button.click + ) + +if __name__ == "__main__": + demo.launch() +``` + +* Set the "mode" parameter to "receive" and "modality" to "video". +* The `stream` event's `fn` parameter is a generator function that yields the next frame from the video as a **numpy array**. +* The only output allowed is the WebRTC component. +* The `trigger` parameter the gradio event that will trigger the webrtc connection. In this case, the button click event. + +## Streaming Audio from the Server to the Client + +```python +import gradio as gr +from pydub import AudioSegment + +def generation(num_steps): + for _ in range(num_steps): + segment = AudioSegment.from_file("/Users/freddy/sources/gradio/demo/audio_debugger/cantina.wav") + yield (segment.frame_rate, np.array(segment.get_array_of_samples()).reshape(1, -1)) + +with gr.Blocks() as demo: + audio = WebRTC(label="Stream", mode="receive", modality="audio") + num_steps = gr.Slider( + label="Number of Steps", + minimum=1, + maximum=10, + step=1, + value=5, + ) + button = gr.Button("Generate") + + audio.stream( + fn=generation, inputs=[num_steps], outputs=[audio], + trigger=button.click + ) +``` + +* Set the "mode" parameter to "receive" and "modality" to "audio". +* The `stream` event's `fn` parameter is a generator function that yields the next audio segment as a tuple of (frame_rate, audio_samples). +* The numpy array should be of shape (1, num_samples). +* The `outputs` parameter should be a list with the WebRTC component as the only element. + +## Conversational AI + +```python import gradio as gr import numpy as np from gradio_webrtc import WebRTC, StreamHandler @@ -26,6 +182,7 @@ import time class EchoHandler(StreamHandler): def __init__(self) -> None: + super().__init__() self.queue = Queue() def receive(self, frame: tuple[int, np.ndarray] | np.ndarray) -> None: @@ -35,20 +192,9 @@ class EchoHandler(StreamHandler): return self.queue.get() -css = """.my-group {max-width: 600px !important; max-height: 600 !important;} - .my-column {display: flex !important; justify-content: center !important; align-items: center !important};""" - - with gr.Blocks() as demo: - gr.HTML( - """ -

- Audio Streaming (Powered by WebRTC ⚡️) -

- """ - ) - with gr.Column(elem_classes=["my-column"]): - with gr.Group(elem_classes=["my-group"]): + with gr.Column(): + with gr.Group(): audio = WebRTC( label="Stream", rtc_configuration=None, @@ -61,3 +207,85 @@ with gr.Blocks() as demo: if __name__ == "__main__": demo.launch() +``` + +* Instead of passing a function to the `stream` event's `fn` parameter, pass a `StreamHandler` implementation. The `StreamHandler` above simply echoes the audio back to the client. +* The `StreamHandler` class has two methods: `receive` and `emit`. The `receive` method is called when a new frame is received from the client, and the `emit` method returns the next frame to send to the client. +* An audio frame is represented as a tuple of (frame_rate, audio_samples) where `audio_samples` is a numpy array of shape (num_channels, num_samples). +* You can also specify the audio layout ("mono" or "stereo") in the emit method by retuning it as the third element of the tuple. If not specified, the default is "mono". +* The `time_limit` parameter is the maximum time in seconds the conversation will run. If the time limit is reached, the audio stream will stop. + + +## Deployment + +When deploying in a cloud environment (like Hugging Face Spaces, EC2, etc), you need to set up a TURN server to relay the WebRTC traffic. +The easiest way to do this is to use a service like Twilio. + +```python +from twilio.rest import Client +import os + +account_sid = os.environ.get("TWILIO_ACCOUNT_SID") +auth_token = os.environ.get("TWILIO_AUTH_TOKEN") + +client = Client(account_sid, auth_token) + +token = client.tokens.create() + +rtc_configuration = { + "iceServers": token.ice_servers, + "iceTransportPolicy": "relay", +} + +with gr.Blocks() as demo: + ... + rtc = WebRTC(rtc_configuration=rtc_configuration, ...) + ... +``` +""", elem_classes=["md-custom"], header_links=True) + + + gr.Markdown(""" +## +""", elem_classes=["md-custom"], header_links=True) + + gr.ParamViewer(value=_docs["WebRTC"]["members"]["__init__"], linkify=[]) + + + demo.load(None, js=r"""function() { + const refs = {}; + const user_fn_refs = { + WebRTC: [], }; + requestAnimationFrame(() => { + + Object.entries(user_fn_refs).forEach(([key, refs]) => { + if (refs.length > 0) { + const el = document.querySelector(`.${key}-user-fn`); + if (!el) return; + refs.forEach(ref => { + el.innerHTML = el.innerHTML.replace( + new RegExp("\\b"+ref+"\\b", "g"), + `${ref}` + ); + }) + } + }) + + Object.entries(refs).forEach(([key, refs]) => { + if (refs.length > 0) { + const el = document.querySelector(`.${key}`); + if (!el) return; + refs.forEach(ref => { + el.innerHTML = el.innerHTML.replace( + new RegExp("\\b"+ref+"\\b", "g"), + `${ref}` + ); + }) + } + }) + }) +} + +""") + +demo.launch() \ No newline at end of file diff --git a/demo/app_orig.py b/demo/app_orig.py index 3489fab..31f3b3f 100644 --- a/demo/app_orig.py +++ b/demo/app_orig.py @@ -1,10 +1,11 @@ -import gradio as gr -import cv2 -from huggingface_hub import hf_hub_download -from gradio_webrtc import WebRTC -from twilio.rest import Client import os + +import cv2 +import gradio as gr +from gradio_webrtc import WebRTC +from huggingface_hub import hf_hub_download from inference import YOLOv10 +from twilio.rest import Client model_file = hf_hub_download( repo_id="onnx-community/yolov10n", filename="onnx/model.onnx" diff --git a/demo/audio_out.py b/demo/audio_out.py index b2da4a8..879ba67 100644 --- a/demo/audio_out.py +++ b/demo/audio_out.py @@ -1,10 +1,10 @@ +import os + import gradio as gr import numpy as np from gradio_webrtc import WebRTC -from twilio.rest import Client -import os from pydub import AudioSegment - +from twilio.rest import Client account_sid = os.environ.get("TWILIO_ACCOUNT_SID") auth_token = os.environ.get("TWILIO_AUTH_TOKEN") @@ -33,8 +33,6 @@ def generation(num_steps): segment.frame_rate, np.array(segment.get_array_of_samples()).reshape(1, -1), ) - time.sleep(3.5) - css = """.my-group {max-width: 600px !important; max-height: 600 !important;} .my-column {display: flex !important; justify-content: center !important; align-items: center !important};""" diff --git a/demo/audio_out_2.py b/demo/audio_out_2.py index ec885b3..8a8238f 100644 --- a/demo/audio_out_2.py +++ b/demo/audio_out_2.py @@ -1,10 +1,10 @@ +import os + import gradio as gr import numpy as np from gradio_webrtc import WebRTC -from twilio.rest import Client -import os from pydub import AudioSegment - +from twilio.rest import Client account_sid = os.environ.get("TWILIO_ACCOUNT_SID") auth_token = os.environ.get("TWILIO_AUTH_TOKEN") diff --git a/demo/inference.py b/demo/inference.py index 1e4d9a8..7b3fdff 100644 --- a/demo/inference.py +++ b/demo/inference.py @@ -1,8 +1,8 @@ import time + import cv2 import numpy as np import onnxruntime - from utils import draw_detections @@ -120,8 +120,9 @@ class YOLOv10: if __name__ == "__main__": - import requests import tempfile + + import requests from huggingface_hub import hf_hub_download model_file = hf_hub_download( diff --git a/demo/space.py b/demo/space.py index 36a54de..7b7bc06 100644 --- a/demo/space.py +++ b/demo/space.py @@ -1,6 +1,7 @@ -import gradio as gr import os +import gradio as gr + _docs = { "WebRTC": { "description": "Stream audio/video with WebRTC", diff --git a/demo/utils.py b/demo/utils.py index 8cdd1d2..04dadeb 100644 --- a/demo/utils.py +++ b/demo/utils.py @@ -1,5 +1,5 @@ -import numpy as np import cv2 +import numpy as np class_names = [ "person", diff --git a/demo/video_out.py b/demo/video_out.py index 298614d..e244b51 100644 --- a/demo/video_out.py +++ b/demo/video_out.py @@ -1,9 +1,9 @@ +import os + +import cv2 import gradio as gr from gradio_webrtc import WebRTC from twilio.rest import Client -import os -import cv2 - account_sid = os.environ.get("TWILIO_ACCOUNT_SID") auth_token = os.environ.get("TWILIO_AUTH_TOKEN") diff --git a/demo/video_out_stream.py b/demo/video_out_stream.py index 49461ed..a5689ca 100644 --- a/demo/video_out_stream.py +++ b/demo/video_out_stream.py @@ -1,9 +1,9 @@ +import os + +import cv2 import gradio as gr from gradio_webrtc import WebRTC from twilio.rest import Client -import os -import cv2 - account_sid = os.environ.get("TWILIO_ACCOUNT_SID") auth_token = os.environ.get("TWILIO_AUTH_TOKEN") diff --git a/frontend/Index.svelte b/frontend/Index.svelte index b3ba5db..242841e 100644 --- a/frontend/Index.svelte +++ b/frontend/Index.svelte @@ -32,6 +32,7 @@ export let time_limit: number | null = null; export let modality: "video" | "audio" = "video"; export let mode: "send-receive" | "receive" = "send-receive"; + export let track_constraints: MediaTrackConstraints = {}; let dragging = false; @@ -113,6 +114,7 @@ {server} {rtc_configuration} {time_limit} + {track_constraints} i18n={gradio.i18n} on:tick={() => gradio.dispatch("tick")} on:error={({ detail }) => gradio.dispatch("error", detail)} diff --git a/frontend/shared/AudioWave.svelte b/frontend/shared/AudioWave.svelte index 8d8c19e..81f3c34 100644 --- a/frontend/shared/AudioWave.svelte +++ b/frontend/shared/AudioWave.svelte @@ -25,7 +25,6 @@ }); function setupAudioContext() { - console.log("set up") audioContext = new (window.AudioContext || window.webkitAudioContext)(); analyser = audioContext.createAnalyser(); console.log("audio_source", audio_source.srcObject); @@ -50,16 +49,6 @@ animationId = requestAnimationFrame(updateBars); } - - function toggleMute() { - if (audio_source && audio_source.srcObject) { - const audioTracks = (audio_source.srcObject as MediaStream).getAudioTracks(); - audioTracks.forEach(track => { - track.enabled = !track.enabled; - }); - is_muted = !audioTracks[0].enabled; - } - } @@ -75,6 +64,8 @@ .waveContainer { position: relative; display: flex; + min-height: 100px; + max-height: 128px; } .boxContainer { diff --git a/frontend/shared/InteractiveAudio.svelte b/frontend/shared/InteractiveAudio.svelte index 6f7f043..5dd3ef3 100644 --- a/frontend/shared/InteractiveAudio.svelte +++ b/frontend/shared/InteractiveAudio.svelte @@ -24,6 +24,7 @@ export let rtc_configuration: Object | null = null; export let i18n: I18nFormatter; export let time_limit: number | null = null; + export let track_constraints: MediaTrackConstraints = {}; let _time_limit: number | null = null; $: console.log("time_limit", time_limit); @@ -87,14 +88,7 @@ let stream = null try { - stream = await navigator.mediaDevices.getUserMedia({ audio: { - echoCancellation: true, - noiseSuppression: {exact: true}, - autoGainControl: {exact: true}, - sampleRate: {ideal: 48000}, - sampleSize: {ideal: 16}, - channelCount: 2, - } }); + stream = await navigator.mediaDevices.getUserMedia({ audio: track_constraints }); } catch (err) { if (!navigator.mediaDevices) { dispatch("error", i18n("audio.no_device_support")); diff --git a/frontend/shared/InteractiveVideo.svelte b/frontend/shared/InteractiveVideo.svelte index 9239a7c..5794c70 100644 --- a/frontend/shared/InteractiveVideo.svelte +++ b/frontend/shared/InteractiveVideo.svelte @@ -20,6 +20,7 @@ offer: (body: any) => Promise; }; export let rtc_configuration: Object; + export let track_constraints: MediaTrackConstraints = {}; const dispatch = createEventDispatcher<{ change: FileData | null; @@ -48,6 +49,7 @@ {rtc_configuration} {include_audio} {time_limit} + {track_constraints} on:error on:start_recording on:stop_recording diff --git a/frontend/shared/StaticAudio.svelte b/frontend/shared/StaticAudio.svelte index 05cf77b..69b4f88 100644 --- a/frontend/shared/StaticAudio.svelte +++ b/frontend/shared/StaticAudio.svelte @@ -22,7 +22,7 @@ offer: (body: any) => Promise; }; - let stream_state: "open" | "closed" | "connecting" = "closed"; + let stream_state: "open" | "closed" | "waiting" = "closed"; let audio_player: HTMLAudioElement; let pc: RTCPeerConnection; let _webrtc_id = Math.random().toString(36).substring(2); @@ -35,7 +35,6 @@ stop: undefined; }>(); - onMount(() => { window.setInterval(() => { if (stream_state == "open") { @@ -45,10 +44,11 @@ } ) - async function start_stream(value: string): Promise { + async function start_stream(value: string): Promise { if( value === "start_webrtc_stream") { - stream_state = "connecting"; + stream_state = "waiting"; value = _webrtc_id; + console.log("set value to ", value); pc = new RTCPeerConnection(rtc_configuration); pc.addEventListener("connectionstatechange", async (event) => { @@ -74,9 +74,12 @@ dispatch("error", "Too many concurrent users. Come back later!"); }); } + return value; } - $: start_stream(value); + $: start_stream(value).then((val) => { + value = val; + }); @@ -97,23 +100,28 @@ on:play={() => dispatch("play")} /> {#if value !== "__webrtc_value__"} +
+
{/if} {#if value === "__webrtc_value__"} - - - + + + {/if}