deepgeek/gradio-webrtc
2
0
Fork 0
mirror of https://github.com/HumanAIGC-Engineering/gradio-webrtc.git synced 2026-02-05 18:09:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

make code

Browse Source
This commit is contained in:
freddyaboulton
2024-10-22 16:24:21 -07:00
parent cff6073df0
commit e7f3e63c79
20 changed files with 427 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/gradio_webrtc/__init__.py
View File

@@ -1,3 +1,3 @@
from .webrtc import WebRTC, StreamHandler from .webrtc import StreamHandler, WebRTC
__all__ = ["StreamHandler", "WebRTC"] __all__ = ["StreamHandler", "WebRTC"]

49
backend/gradio_webrtc/utils.py
View File

@@ -2,7 +2,6 @@ import asyncio
import fractions import fractions
import logging import logging
import threading import threading
import time
from typing import Callable from typing import Callable
import av import av
@@ -15,35 +14,44 @@ AUDIO_PTIME = 0.020
def player_worker_decode( def player_worker_decode(
loop, loop,
next: Callable, next_frame: Callable,
queue: asyncio.Queue, queue: asyncio.Queue,
throttle_playback: bool,
thread_quit: threading.Event, thread_quit: threading.Event,
quit_on_none: bool = False,
sample_rate: int = 48000,
frame_size: int = int(48000 * AUDIO_PTIME),
): ):
audio_sample_rate = 48000
audio_samples = 0 audio_samples = 0
audio_time_base = fractions.Fraction(1, audio_sample_rate) audio_time_base = fractions.Fraction(1, sample_rate)
audio_resampler = av.AudioResampler( audio_resampler = av.AudioResampler( # type: ignore
format="s16", format="s16",
layout="stereo", layout="stereo",
rate=audio_sample_rate, rate=sample_rate,
frame_size=int(audio_sample_rate * AUDIO_PTIME), frame_size=frame_size,
) )
frame_time = None
start_time = time.time()
while not thread_quit.is_set(): while not thread_quit.is_set():
frame = next() frame = next_frame()
logger.debug("emitted %s", frame) if frame is None:
# read up to 1 second ahead if quit_on_none:
if throttle_playback: asyncio.run_coroutine_threadsafe(queue.put(None), loop)
elapsed_time = time.time() - start_time continue
if frame_time and frame_time > elapsed_time + 1:
time.sleep(0.1) if len(frame) == 2:
sample_rate, audio_array = frame sample_rate, audio_array = frame
layout = "mono"
elif len(frame) == 3:
sample_rate, audio_array, layout = frame
logger.debug(
"received array with shape %s sample rate %s layout %s",
audio_array.shape,
sample_rate,
layout,
)
format = "s16" if audio_array.dtype == "int16" else "fltp" format = "s16" if audio_array.dtype == "int16" else "fltp"
frame = av.AudioFrame.from_ndarray(audio_array, format=format, layout="stereo")
frame = av.AudioFrame.from_ndarray(audio_array, format=format, layout=layout) # type: ignore
frame.sample_rate = sample_rate frame.sample_rate = sample_rate
for frame in audio_resampler.resample(frame): for frame in audio_resampler.resample(frame):
# fix timestamps # fix timestamps
@@ -51,5 +59,4 @@ def player_worker_decode(
frame.time_base = audio_time_base frame.time_base = audio_time_base
audio_samples += frame.samples audio_samples += frame.samples
frame_time = frame.time
asyncio.run_coroutine_threadsafe(queue.put(frame), loop) asyncio.run_coroutine_threadsafe(queue.put(frame), loop)

132
backend/gradio_webrtc/webrtc.py
View File

@@ -3,24 +3,25 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
from abc import ABC, abstractmethod
import logging import logging
import threading import threading
import time import time
import traceback import traceback
from abc import ABC, abstractmethod
from collections.abc import Callable from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Generator, Literal, Sequence, cast from typing import TYPE_CHECKING, Any, Generator, Literal, Sequence, cast
import anyio.to_thread import anyio.to_thread
import av
import numpy as np import numpy as np
from aiortc import ( from aiortc import (
AudioStreamTrack, AudioStreamTrack,
MediaStreamTrack,
RTCPeerConnection, RTCPeerConnection,
RTCSessionDescription, RTCSessionDescription,
VideoStreamTrack, VideoStreamTrack,
MediaStreamTrack,
) )
from aiortc.contrib.media import MediaRelay, AudioFrame, VideoFrame # type: ignore from aiortc.contrib.media import AudioFrame, MediaRelay, VideoFrame # type: ignore
from aiortc.mediastreams import MediaStreamError from aiortc.mediastreams import MediaStreamError
from gradio import wasm_utils from gradio import wasm_utils
from gradio.components.base import Component, server from gradio.components.base import Component, server
@@ -104,6 +105,27 @@ class VideoCallback(VideoStreamTrack):
class StreamHandler(ABC): class StreamHandler(ABC):
def __init__(
self,
expected_layout: Literal["mono", "stereo"] = "mono",
output_sample_rate: int = 24000,
output_frame_size: int = 960,
) -> None:
self.expected_layout = expected_layout
self.output_sample_rate = output_sample_rate
self.output_frame_size = output_frame_size
self._resampler = None
def resample(self, frame: AudioFrame) -> Generator[AudioFrame, None, None]:
if self._resampler is None:
self._resampler = av.AudioResampler( # type: ignore
format="s16",
layout=self.expected_layout,
rate=frame.sample_rate,
frame_size=frame.samples,
)
yield from self._resampler.resample(frame)
@abstractmethod @abstractmethod
def receive(self, frame: tuple[int, np.ndarray] | np.ndarray) -> None: def receive(self, frame: tuple[int, np.ndarray] | np.ndarray) -> None:
pass pass
@@ -124,24 +146,27 @@ class AudioCallback(AudioStreamTrack):
self.track = track self.track = track
self.event_handler = event_handler self.event_handler = event_handler
self.current_timestamp = 0 self.current_timestamp = 0
self.latest_args = "not_set" self.latest_args: str | list[Any] = "not_set"
self.queue = asyncio.Queue() self.queue = asyncio.Queue()
self.thread_quit = threading.Event() self.thread_quit = threading.Event()
self.__thread = None self.__thread = None
self._start: float | None = None self._start: float | None = None
self.has_started = False self.has_started = False
self.last_timestamp = 0
super().__init__() super().__init__()
async def process_input_frames(self) -> None: async def process_input_frames(self) -> None:
while not self.thread_quit.is_set(): while not self.thread_quit.is_set():
try: try:
frame = cast(AudioFrame, await self.track.recv()) frame = cast(AudioFrame, await self.track.recv())
numpy_array = frame.to_ndarray() for frame in self.event_handler.resample(frame):
logger.debug("numpy array shape %s", numpy_array.shape) numpy_array = frame.to_ndarray()
await anyio.to_thread.run_sync( logger.debug("numpy array shape %s", numpy_array.shape)
self.event_handler.receive, (frame.sample_rate, numpy_array) await anyio.to_thread.run_sync(
) self.event_handler.receive, (frame.sample_rate, numpy_array)
except MediaStreamError: )
except MediaStreamError as e:
print("MediaStreamError", e)
break break
def start(self): def start(self):
@@ -154,8 +179,10 @@ class AudioCallback(AudioStreamTrack):
asyncio.get_event_loop(), asyncio.get_event_loop(),
self.event_handler.emit, self.event_handler.emit,
self.queue, self.queue,
True,
self.thread_quit, self.thread_quit,
False,
self.event_handler.output_sample_rate,
self.event_handler.output_frame_size,
), ),
) )
self.__thread.start() self.__thread.start()
@@ -167,23 +194,25 @@ class AudioCallback(AudioStreamTrack):
raise MediaStreamError raise MediaStreamError
self.start() self.start()
data = await self.queue.get() frame = await self.queue.get()
logger.debug("data %s", data) logger.debug("frame %s", frame)
if data is None:
self.stop()
return
data_time = data.time data_time = frame.time
if time.time() - self.last_timestamp > 10 * (
self.event_handler.output_frame_size
/ self.event_handler.output_sample_rate
):
self._start = None
# control playback rate # control playback rate
if data_time is not None: if self._start is None:
if self._start is None: self._start = time.time() - data_time
self._start = time.time() - data_time else:
else: wait = self._start + data_time - time.time()
wait = self._start + data_time - time.time() await asyncio.sleep(wait)
await asyncio.sleep(wait) self.last_timestamp = time.time()
return frame
return data
except Exception as e: except Exception as e:
logger.debug(e) logger.debug(e)
exec = traceback.format_exc() exec = traceback.format_exc()
@@ -210,6 +239,7 @@ class ServerToClientVideo(VideoStreamTrack):
) -> None: ) -> None:
super().__init__() # don't forget this! super().__init__() # don't forget this!
self.event_handler = event_handler self.event_handler = event_handler
self.args_set = asyncio.Event()
self.latest_args: str | list[Any] = "not_set" self.latest_args: str | list[Any] = "not_set"
self.generator: Generator[Any, None, Any] | None = None self.generator: Generator[Any, None, Any] | None = None
@@ -219,12 +249,8 @@ class ServerToClientVideo(VideoStreamTrack):
async def recv(self): async def recv(self):
try: try:
pts, time_base = await self.next_timestamp() pts, time_base = await self.next_timestamp()
if self.latest_args == "not_set": await self.args_set.wait()
frame = self.array_to_frame(np.zeros((480, 640, 3), dtype=np.uint8)) if self.generator is None:
frame.pts = pts
frame.time_base = time_base
return frame
elif self.generator is None:
self.generator = cast( self.generator = cast(
Generator[Any, None, Any], self.event_handler(*self.latest_args) Generator[Any, None, Any], self.event_handler(*self.latest_args)
) )
@@ -255,7 +281,8 @@ class ServerToClientAudio(AudioStreamTrack):
self.generator: Generator[Any, None, Any] | None = None self.generator: Generator[Any, None, Any] | None = None
self.event_handler = event_handler self.event_handler = event_handler
self.current_timestamp = 0 self.current_timestamp = 0
self.latest_args = "not_set" self.latest_args: str | list[Any] = "not_set"
self.args_set = threading.Event()
self.queue = asyncio.Queue() self.queue = asyncio.Queue()
self.thread_quit = threading.Event() self.thread_quit = threading.Event()
self.__thread = None self.__thread = None
@@ -263,23 +290,15 @@ class ServerToClientAudio(AudioStreamTrack):
super().__init__() super().__init__()
def next(self) -> tuple[int, np.ndarray] | None: def next(self) -> tuple[int, np.ndarray] | None:
import anyio self.args_set.wait()
if self.latest_args == "not_set":
return
if self.generator is None: if self.generator is None:
self.generator = self.event_handler(*self.latest_args) self.generator = self.event_handler(*self.latest_args)
if self.generator is not None: if self.generator is not None:
try: try:
frame = next(self.generator) frame = next(self.generator)
return frame return frame
except Exception as exc: except StopIteration:
if isinstance(exc, StopIteration): pass
logger.debug("Stopping audio stream")
asyncio.run_coroutine_threadsafe(
self.queue.put(None), asyncio.get_event_loop()
)
self.thread_quit.set()
def start(self): def start(self):
if self.__thread is None: if self.__thread is None:
@@ -290,8 +309,8 @@ class ServerToClientAudio(AudioStreamTrack):
asyncio.get_event_loop(), asyncio.get_event_loop(),
self.next, self.next,
self.queue, self.queue,
False,
self.thread_quit, self.thread_quit,
True,
), ),
) )
self.__thread.start() self.__thread.start()
@@ -370,6 +389,7 @@ class WebRTC(Component):
key: int | str | None = None, key: int | str | None = None,
mirror_webcam: bool = True, mirror_webcam: bool = True,
rtc_configuration: dict[str, Any] | None = None, rtc_configuration: dict[str, Any] | None = None,
track_constraints: dict[str, Any] | None = None,
time_limit: float | None = None, time_limit: float | None = None,
mode: Literal["send-receive", "receive"] = "send-receive", mode: Literal["send-receive", "receive"] = "send-receive",
modality: Literal["video", "audio"] = "video", modality: Literal["video", "audio"] = "video",
@@ -412,7 +432,24 @@ class WebRTC(Component):
self.rtc_configuration = rtc_configuration self.rtc_configuration = rtc_configuration
self.mode = mode self.mode = mode
self.modality = modality self.modality = modality
self.event_handler: Callable | None = None if track_constraints is None and modality == "audio":
track_constraints = {
"echoCancellation": True,
"noiseSuppression": {"exact": True},
"autoGainControl": {"exact": True},
"sampleRate": {"ideal": 24000},
"sampleSize": {"ideal": 16},
"channelCount": {"exact": 1},
}
if track_constraints is None and modality == "video":
track_constraints = {
"facingMode": "user",
"width": {"ideal": 500},
"height": {"ideal": 500},
"frameRate": {"ideal": 30},
}
self.track_constraints = track_constraints
self.event_handler: Callable | StreamHandler | None = None
super().__init__( super().__init__(
label=label, label=label,
every=every, every=every,
@@ -456,6 +493,7 @@ class WebRTC(Component):
) )
elif self.mode == "receive": elif self.mode == "receive":
self.connections[webrtc_id].latest_args = list(args) self.connections[webrtc_id].latest_args = list(args)
self.connections[webrtc_id].args_set.set() # type: ignore
def stream( def stream(
self, self,
@@ -534,9 +572,9 @@ class WebRTC(Component):
"In the receive mode stream event, the trigger parameter must be provided" "In the receive mode stream event, the trigger parameter must be provided"
) )
trigger(lambda: "start_webrtc_stream", inputs=None, outputs=self) trigger(lambda: "start_webrtc_stream", inputs=None, outputs=self)
self.tick( self.tick( # type: ignore
self.set_output, self.set_output,
inputs=[self] + inputs, inputs=[self] + list(inputs),
outputs=None, outputs=None,
concurrency_id=concurrency_id, concurrency_id=concurrency_id,
) )

282
demo/app.py
View File

@@ -1,22 +1,178 @@
import logging
# Configure the root logger to WARNING to suppress debug messages from other libraries import os
logging.basicConfig(level=logging.WARNING)
# Create a console handler import gradio as gr
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.DEBUG)
# Create a formatter _docs = {'WebRTC':
formatter = logging.Formatter("%(name)s - %(levelname)s - %(message)s") {'description': 'Stream audio/video with WebRTC',
console_handler.setFormatter(formatter) 'members': {'__init__':
{
# Configure the logger for your specific library 'rtc_configuration': {'type': 'dict[str, Any] | None', 'default': 'None', 'description': "The configration dictionary to pass to the RTCPeerConnection constructor. If None, the default configuration is used."},
logger = logging.getLogger("gradio_webrtc") 'height': {'type': 'int | str | None', 'default': 'None', 'description': 'The height of the component, specified in pixels if a number is passed, or in CSS units if a string is passed. This has no effect on the preprocessed video file, but will affect the displayed video.'},
logger.setLevel(logging.DEBUG) 'width': {'type': 'int | str | None', 'default': 'None', 'description': 'The width of the component, specified in pixels if a number is passed, or in CSS units if a string is passed. This has no effect on the preprocessed video file, but will affect the displayed video.'},
logger.addHandler(console_handler) 'label': {'type': 'str | None', 'default': 'None', 'description': 'the label for this component. Appears above the component and is also used as the header if there are a table of examples for this component. If None and used in a `gr.Interface`, the label will be the name of the parameter this component is assigned to.'},
'show_label': {'type': 'bool | None', 'default': 'None', 'description': 'if True, will display label.'}, 'container': {'type': 'bool', 'default': 'True', 'description': 'if True, will place the component in a container - providing some extra padding around the border.'},
'scale': {'type': 'int | None', 'default': 'None', 'description': 'relative size compared to adjacent Components. For example if Components A and B are in a Row, and A has scale=2, and B has scale=1, A will be twice as wide as B. Should be an integer. scale applies in Rows, and to top-level Components in Blocks where fill_height=True.'},
'min_width': {'type': 'int', 'default': '160', 'description': 'minimum pixel width, will wrap if not sufficient screen space to satisfy this value. If a certain scale value results in this Component being narrower than min_width, the min_width parameter will be respected first.'},
'interactive': {'type': 'bool | None', 'default': 'None', 'description': 'if True, will allow users to upload a video; if False, can only be used to display videos. If not provided, this is inferred based on whether the component is used as an input or output.'}, 'visible': {'type': 'bool', 'default': 'True', 'description': 'if False, component will be hidden.'},
'elem_id': {'type': 'str | None', 'default': 'None', 'description': 'an optional string that is assigned as the id of this component in the HTML DOM. Can be used for targeting CSS styles.'},
'elem_classes': {'type': 'list[str] | str | None', 'default': 'None', 'description': 'an optional list of strings that are assigned as the classes of this component in the HTML DOM. Can be used for targeting CSS styles.'},
'render': {'type': 'bool', 'default': 'True', 'description': 'if False, component will not render be rendered in the Blocks context. Should be used if the intention is to assign event listeners now but render the component later.'},
'key': {'type': 'int | str | None', 'default': 'None', 'description': 'if assigned, will be used to assume identity across a re-render. Components that have the same key across a re-render will have their value preserved.'},
'mirror_webcam': {'type': 'bool', 'default': 'True', 'description': 'if True webcam will be mirrored. Default is True.'},
},
'events': {'tick': {'type': None, 'default': None, 'description': ''}}}, '__meta__': {'additional_interfaces': {}, 'user_fn_refs': {'WebRTC': []}}}
}
abs_path = os.path.join(os.path.dirname(__file__), "css.css")
with gr.Blocks(
css_paths=abs_path,
theme=gr.themes.Default(
font_mono=[
gr.themes.GoogleFont("Inconsolata"),
"monospace",
],
),
) as demo:
gr.Markdown(
"""
<h1 style='text-align: center; margin-bottom: 1rem'> Gradio WebRTC ⚡️ </h1>
<div style="display: flex; flex-direction: row; justify-content: center">
<img style="display: block; padding-right: 5px; height: 20px;" alt="Static Badge" src="https://img.shields.io/badge/version%20-%200.0.6%20-%20orange">
<a href="https://github.com/freddyaboulton/gradio-webrtc" target="_blank"><img alt="Static Badge" src="https://img.shields.io/badge/github-white?logo=github&logoColor=black"></a>
</div>
""", elem_classes=["md-custom"], header_links=True)
gr.Markdown(
"""
## Installation
```bash
pip install gradio_webrtc
```
## Examples:
1. [Object Detection from Webcam with YOLOv10](https://huggingface.co/spaces/freddyaboulton/webrtc-yolov10n) 📷
2. [Streaming Object Detection from Video with RT-DETR](https://huggingface.co/spaces/freddyaboulton/rt-detr-object-detection-webrtc) 🎥
3. [Text-to-Speech](https://huggingface.co/spaces/freddyaboulton/parler-tts-streaming-webrtc) 🗣️
4. [Conversational AI]()
## Usage
The WebRTC component supports the following three use cases:
1. [Streaming video from the user webcam to the server and back](#h-streaming-video-from-the-user-webcam-to-the-server-and-back)
2. [Streaming Video from the server to the client](#h-streaming-video-from-the-server-to-the-client)
3. [Streaming Audio from the server to the client](#h-streaming-audio-from-the-server-to-the-client)
4. [Streaming Audio from the client to the server and back (conversational AI)](#h-conversational-ai)
## Streaming Video from the User Webcam to the Server and Back
```python
import gradio as gr
from gradio_webrtc import WebRTC
def detection(image, conf_threshold=0.3):
... your detection code here ...
with gr.Blocks() as demo:
image = WebRTC(label="Stream", mode="send-receive", modality="video")
conf_threshold = gr.Slider(
label="Confidence Threshold",
minimum=0.0,
maximum=1.0,
step=0.05,
value=0.30,
)
image.stream(
fn=detection,
inputs=[image, conf_threshold],
outputs=[image], time_limit=10
)
if __name__ == "__main__":
demo.launch()
```
* Set the `mode` parameter to `send-receive` and `modality` to "video".
* The `stream` event's `fn` parameter is a function that receives the next frame from the webcam
as a **numpy array** and returns the processed frame also as a **numpy array**.
* Numpy arrays are in (height, width, 3) format where the color channels are in RGB format.
* The `inputs` parameter should be a list where the first element is the WebRTC component. The only output allowed is the WebRTC component.
* The `time_limit` parameter is the maximum time in seconds the video stream will run. If the time limit is reached, the video stream will stop.
## Streaming Video from the server to the client
```python
import gradio as gr
from gradio_webrtc import WebRTC
import cv2
def generation():
url = "https://download.tsi.telecom-paristech.fr/gpac/dataset/dash/uhd/mux_sources/hevcds_720p30_2M.mp4"
cap = cv2.VideoCapture(url)
iterating = True
while iterating:
iterating, frame = cap.read()
yield frame
with gr.Blocks() as demo:
output_video = WebRTC(label="Video Stream", mode="receive", modality="video")
button = gr.Button("Start", variant="primary")
output_video.stream(
fn=generation, inputs=None, outputs=[output_video],
trigger=button.click
)
if __name__ == "__main__":
demo.launch()
```
* Set the "mode" parameter to "receive" and "modality" to "video".
* The `stream` event's `fn` parameter is a generator function that yields the next frame from the video as a **numpy array**.
* The only output allowed is the WebRTC component.
* The `trigger` parameter the gradio event that will trigger the webrtc connection. In this case, the button click event.
## Streaming Audio from the Server to the Client
```python
import gradio as gr
from pydub import AudioSegment
def generation(num_steps):
for _ in range(num_steps):
segment = AudioSegment.from_file("/Users/freddy/sources/gradio/demo/audio_debugger/cantina.wav")
yield (segment.frame_rate, np.array(segment.get_array_of_samples()).reshape(1, -1))
with gr.Blocks() as demo:
audio = WebRTC(label="Stream", mode="receive", modality="audio")
num_steps = gr.Slider(
label="Number of Steps",
minimum=1,
maximum=10,
step=1,
value=5,
)
button = gr.Button("Generate")
audio.stream(
fn=generation, inputs=[num_steps], outputs=[audio],
trigger=button.click
)
```
* Set the "mode" parameter to "receive" and "modality" to "audio".
* The `stream` event's `fn` parameter is a generator function that yields the next audio segment as a tuple of (frame_rate, audio_samples).
* The numpy array should be of shape (1, num_samples).
* The `outputs` parameter should be a list with the WebRTC component as the only element.
## Conversational AI
```python
import gradio as gr import gradio as gr
import numpy as np import numpy as np
from gradio_webrtc import WebRTC, StreamHandler from gradio_webrtc import WebRTC, StreamHandler
@@ -26,6 +182,7 @@ import time
class EchoHandler(StreamHandler): class EchoHandler(StreamHandler):
def __init__(self) -> None: def __init__(self) -> None:
super().__init__()
self.queue = Queue() self.queue = Queue()
def receive(self, frame: tuple[int, np.ndarray] | np.ndarray) -> None: def receive(self, frame: tuple[int, np.ndarray] | np.ndarray) -> None:
@@ -35,20 +192,9 @@ class EchoHandler(StreamHandler):
return self.queue.get() return self.queue.get()
css = """.my-group {max-width: 600px !important; max-height: 600 !important;}
.my-column {display: flex !important; justify-content: center !important; align-items: center !important};"""
with gr.Blocks() as demo: with gr.Blocks() as demo:
gr.HTML( with gr.Column():
""" with gr.Group():
<h1 style='text-align: center'>
Audio Streaming (Powered by WebRTC ⚡️)
</h1>
"""
)
with gr.Column(elem_classes=["my-column"]):
with gr.Group(elem_classes=["my-group"]):
audio = WebRTC( audio = WebRTC(
label="Stream", label="Stream",
rtc_configuration=None, rtc_configuration=None,
@@ -61,3 +207,85 @@ with gr.Blocks() as demo:
if __name__ == "__main__": if __name__ == "__main__":
demo.launch() demo.launch()
```
* Instead of passing a function to the `stream` event's `fn` parameter, pass a `StreamHandler` implementation. The `StreamHandler` above simply echoes the audio back to the client.
* The `StreamHandler` class has two methods: `receive` and `emit`. The `receive` method is called when a new frame is received from the client, and the `emit` method returns the next frame to send to the client.
* An audio frame is represented as a tuple of (frame_rate, audio_samples) where `audio_samples` is a numpy array of shape (num_channels, num_samples).
* You can also specify the audio layout ("mono" or "stereo") in the emit method by retuning it as the third element of the tuple. If not specified, the default is "mono".
* The `time_limit` parameter is the maximum time in seconds the conversation will run. If the time limit is reached, the audio stream will stop.
## Deployment
When deploying in a cloud environment (like Hugging Face Spaces, EC2, etc), you need to set up a TURN server to relay the WebRTC traffic.
The easiest way to do this is to use a service like Twilio.
```python
from twilio.rest import Client
import os
account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
client = Client(account_sid, auth_token)
token = client.tokens.create()
rtc_configuration = {
"iceServers": token.ice_servers,
"iceTransportPolicy": "relay",
}
with gr.Blocks() as demo:
...
rtc = WebRTC(rtc_configuration=rtc_configuration, ...)
...
```
""", elem_classes=["md-custom"], header_links=True)
gr.Markdown("""
##
""", elem_classes=["md-custom"], header_links=True)
gr.ParamViewer(value=_docs["WebRTC"]["members"]["__init__"], linkify=[])
demo.load(None, js=r"""function() {
const refs = {};
const user_fn_refs = {
WebRTC: [], };
requestAnimationFrame(() => {
Object.entries(user_fn_refs).forEach(([key, refs]) => {
if (refs.length > 0) {
const el = document.querySelector(`.${key}-user-fn`);
if (!el) return;
refs.forEach(ref => {
el.innerHTML = el.innerHTML.replace(
new RegExp("\\b"+ref+"\\b", "g"),
`<a href="#h-${ref.toLowerCase()}">${ref}</a>`
);
})
}
})
Object.entries(refs).forEach(([key, refs]) => {
if (refs.length > 0) {
const el = document.querySelector(`.${key}`);
if (!el) return;
refs.forEach(ref => {
el.innerHTML = el.innerHTML.replace(
new RegExp("\\b"+ref+"\\b", "g"),
`<a href="#h-${ref.toLowerCase()}">${ref}</a>`
);
})
}
})
})
}
""")
demo.launch()

11
demo/app_orig.py
View File

@@ -1,10 +1,11 @@
import gradio as gr
import cv2
from huggingface_hub import hf_hub_download
from gradio_webrtc import WebRTC
from twilio.rest import Client
import os import os
import cv2
import gradio as gr
from gradio_webrtc import WebRTC
from huggingface_hub import hf_hub_download
from inference import YOLOv10 from inference import YOLOv10
from twilio.rest import Client
model_file = hf_hub_download( model_file = hf_hub_download(
repo_id="onnx-community/yolov10n", filename="onnx/model.onnx" repo_id="onnx-community/yolov10n", filename="onnx/model.onnx"

8
demo/audio_out.py
View File

@@ -1,10 +1,10 @@
import os
import gradio as gr import gradio as gr
import numpy as np import numpy as np
from gradio_webrtc import WebRTC from gradio_webrtc import WebRTC
from twilio.rest import Client
import os
from pydub import AudioSegment from pydub import AudioSegment
from twilio.rest import Client
account_sid = os.environ.get("TWILIO_ACCOUNT_SID") account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
auth_token = os.environ.get("TWILIO_AUTH_TOKEN") auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
@@ -33,8 +33,6 @@ def generation(num_steps):
segment.frame_rate, segment.frame_rate,
np.array(segment.get_array_of_samples()).reshape(1, -1), np.array(segment.get_array_of_samples()).reshape(1, -1),
) )
time.sleep(3.5)
css = """.my-group {max-width: 600px !important; max-height: 600 !important;} css = """.my-group {max-width: 600px !important; max-height: 600 !important;}
.my-column {display: flex !important; justify-content: center !important; align-items: center !important};""" .my-column {display: flex !important; justify-content: center !important; align-items: center !important};"""

6
demo/audio_out_2.py
View File

@@ -1,10 +1,10 @@
import os
import gradio as gr import gradio as gr
import numpy as np import numpy as np
from gradio_webrtc import WebRTC from gradio_webrtc import WebRTC
from twilio.rest import Client
import os
from pydub import AudioSegment from pydub import AudioSegment
from twilio.rest import Client
account_sid = os.environ.get("TWILIO_ACCOUNT_SID") account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
auth_token = os.environ.get("TWILIO_AUTH_TOKEN") auth_token = os.environ.get("TWILIO_AUTH_TOKEN")

5
demo/inference.py
View File

@@ -1,8 +1,8 @@
import time import time
import cv2 import cv2
import numpy as np import numpy as np
import onnxruntime import onnxruntime
from utils import draw_detections from utils import draw_detections
@@ -120,8 +120,9 @@ class YOLOv10:
if __name__ == "__main__": if __name__ == "__main__":
import requests
import tempfile import tempfile
import requests
from huggingface_hub import hf_hub_download from huggingface_hub import hf_hub_download
model_file = hf_hub_download( model_file = hf_hub_download(

3
demo/space.py
View File

@@ -1,6 +1,7 @@
import gradio as gr
import os import os
import gradio as gr
_docs = { _docs = {
"WebRTC": { "WebRTC": {
"description": "Stream audio/video with WebRTC", "description": "Stream audio/video with WebRTC",

2
demo/utils.py
View File

@@ -1,5 +1,5 @@
import numpy as np
import cv2 import cv2
import numpy as np
class_names = [ class_names = [
"person", "person",

6
demo/video_out.py
View File

@@ -1,9 +1,9 @@
import os
import cv2
import gradio as gr import gradio as gr
from gradio_webrtc import WebRTC from gradio_webrtc import WebRTC
from twilio.rest import Client from twilio.rest import Client
import os
import cv2
account_sid = os.environ.get("TWILIO_ACCOUNT_SID") account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
auth_token = os.environ.get("TWILIO_AUTH_TOKEN") auth_token = os.environ.get("TWILIO_AUTH_TOKEN")

6
demo/video_out_stream.py
View File

@@ -1,9 +1,9 @@
import os
import cv2
import gradio as gr import gradio as gr
from gradio_webrtc import WebRTC from gradio_webrtc import WebRTC
from twilio.rest import Client from twilio.rest import Client
import os
import cv2
account_sid = os.environ.get("TWILIO_ACCOUNT_SID") account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
auth_token = os.environ.get("TWILIO_AUTH_TOKEN") auth_token = os.environ.get("TWILIO_AUTH_TOKEN")

2
frontend/Index.svelte
View File

@@ -32,6 +32,7 @@
export let time_limit: number | null = null; export let time_limit: number | null = null;
export let modality: "video" | "audio" = "video"; export let modality: "video" | "audio" = "video";
export let mode: "send-receive" | "receive" = "send-receive"; export let mode: "send-receive" | "receive" = "send-receive";
export let track_constraints: MediaTrackConstraints = {};
let dragging = false; let dragging = false;
@@ -113,6 +114,7 @@
{server} {server}
{rtc_configuration} {rtc_configuration}
{time_limit} {time_limit}
{track_constraints}
i18n={gradio.i18n} i18n={gradio.i18n}
on:tick={() => gradio.dispatch("tick")} on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)} on:error={({ detail }) => gradio.dispatch("error", detail)}

13
frontend/shared/AudioWave.svelte
View File

@@ -25,7 +25,6 @@
}); });
function setupAudioContext() { function setupAudioContext() {
console.log("set up")
audioContext = new (window.AudioContext || window.webkitAudioContext)(); audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser(); analyser = audioContext.createAnalyser();
console.log("audio_source", audio_source.srcObject); console.log("audio_source", audio_source.srcObject);
@@ -50,16 +49,6 @@
animationId = requestAnimationFrame(updateBars); animationId = requestAnimationFrame(updateBars);
} }
function toggleMute() {
if (audio_source && audio_source.srcObject) {
const audioTracks = (audio_source.srcObject as MediaStream).getAudioTracks();
audioTracks.forEach(track => {
track.enabled = !track.enabled;
});
is_muted = !audioTracks[0].enabled;
}
}
</script> </script>
@@ -75,6 +64,8 @@
<style> <style>
.waveContainer { .waveContainer {
position: relative; position: relative;
display: flex;
min-height: 100px;
max-height: 128px; max-height: 128px;
} }

10
frontend/shared/InteractiveAudio.svelte
View File

@@ -24,6 +24,7 @@
export let rtc_configuration: Object | null = null; export let rtc_configuration: Object | null = null;
export let i18n: I18nFormatter; export let i18n: I18nFormatter;
export let time_limit: number | null = null; export let time_limit: number | null = null;
export let track_constraints: MediaTrackConstraints = {};
let _time_limit: number | null = null; let _time_limit: number | null = null;
$: console.log("time_limit", time_limit); $: console.log("time_limit", time_limit);
@@ -87,14 +88,7 @@
let stream = null let stream = null
try { try {
stream = await navigator.mediaDevices.getUserMedia({ audio: { stream = await navigator.mediaDevices.getUserMedia({ audio: track_constraints });
echoCancellation: true,
noiseSuppression: {exact: true},
autoGainControl: {exact: true},
sampleRate: {ideal: 48000},
sampleSize: {ideal: 16},
channelCount: 2,
} });
} catch (err) { } catch (err) {
if (!navigator.mediaDevices) { if (!navigator.mediaDevices) {
dispatch("error", i18n("audio.no_device_support")); dispatch("error", i18n("audio.no_device_support"));

2
frontend/shared/InteractiveVideo.svelte
View File

@@ -20,6 +20,7 @@
offer: (body: any) => Promise<any>; offer: (body: any) => Promise<any>;
}; };
export let rtc_configuration: Object; export let rtc_configuration: Object;
export let track_constraints: MediaTrackConstraints = {};
const dispatch = createEventDispatcher<{ const dispatch = createEventDispatcher<{
change: FileData | null; change: FileData | null;
@@ -48,6 +49,7 @@
{rtc_configuration} {rtc_configuration}
{include_audio} {include_audio}
{time_limit} {time_limit}
{track_constraints}
on:error on:error
on:start_recording on:start_recording
on:stop_recording on:stop_recording

32
frontend/shared/StaticAudio.svelte
View File

@@ -22,7 +22,7 @@
offer: (body: any) => Promise<any>; offer: (body: any) => Promise<any>;
}; };
let stream_state: "open" | "closed" | "connecting" = "closed"; let stream_state: "open" | "closed" | "waiting" = "closed";
let audio_player: HTMLAudioElement; let audio_player: HTMLAudioElement;
let pc: RTCPeerConnection; let pc: RTCPeerConnection;
let _webrtc_id = Math.random().toString(36).substring(2); let _webrtc_id = Math.random().toString(36).substring(2);
@@ -35,7 +35,6 @@
stop: undefined; stop: undefined;
}>(); }>();
onMount(() => { onMount(() => {
window.setInterval(() => { window.setInterval(() => {
if (stream_state == "open") { if (stream_state == "open") {
@@ -45,10 +44,11 @@
} }
) )
async function start_stream(value: string): Promise<void> { async function start_stream(value: string): Promise<string> {
if( value === "start_webrtc_stream") { if( value === "start_webrtc_stream") {
stream_state = "connecting"; stream_state = "waiting";
value = _webrtc_id; value = _webrtc_id;
console.log("set value to ", value);
pc = new RTCPeerConnection(rtc_configuration); pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange", pc.addEventListener("connectionstatechange",
async (event) => { async (event) => {
@@ -74,9 +74,12 @@
dispatch("error", "Too many concurrent users. Come back later!"); dispatch("error", "Too many concurrent users. Come back later!");
}); });
} }
return value;
} }
$: start_stream(value); $: start_stream(value).then((val) => {
value = val;
});
@@ -97,23 +100,28 @@
on:play={() => dispatch("play")} on:play={() => dispatch("play")}
/> />
{#if value !== "__webrtc_value__"} {#if value !== "__webrtc_value__"}
<div class="audio-container">
<AudioWave audio_source={audio_player} {stream_state}/> <AudioWave audio_source={audio_player} {stream_state}/>
</div>
{/if} {/if}
{#if value === "__webrtc_value__"} {#if value === "__webrtc_value__"}
<Empty size="small"> <Empty size="small">
<Music /> <Music />
</Empty> </Empty>
{/if} {/if}
<style> <style>
:global(::part(wrapper)) { .audio-container {
margin-bottom: var(--size-2); display: flex;
} height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
.standard-player { .standard-player {
width: 100%; width: 100%;
padding: var(--size-2);
} }
.hidden { .hidden {

3
frontend/shared/Webcam.svelte
View File

@@ -40,6 +40,7 @@
}; };
let canvas: HTMLCanvasElement; let canvas: HTMLCanvasElement;
export let track_constraints: MediaTrackConstraints | null = null;
export let rtc_configuration: Object; export let rtc_configuration: Object;
export let stream_every = 1; export let stream_every = 1;
export let server: { export let server: {
@@ -63,7 +64,7 @@
const target = event.target as HTMLInputElement; const target = event.target as HTMLInputElement;
const device_id = target.value; const device_id = target.value;
await get_video_stream(include_audio, video_source, device_id).then( await get_video_stream(include_audio, video_source, device_id, track_constraints).then(
async (local_stream) => { async (local_stream) => {
stream = local_stream; stream = local_stream;
selected_device = selected_device =

7
frontend/shared/stream_utils.ts
View File

@@ -18,15 +18,16 @@ export function set_local_stream(
export async function get_video_stream( export async function get_video_stream(
include_audio: boolean, include_audio: boolean,
video_source: HTMLVideoElement, video_source: HTMLVideoElement,
device_id?: string device_id?: string,
track_constraints?: MediaTrackConstraints,
): Promise<MediaStream> { ): Promise<MediaStream> {
const size = { const fallback_constraints = track_constraints || {
width: { ideal: 500 }, width: { ideal: 500 },
height: { ideal: 500 } height: { ideal: 500 }
}; };
const constraints = { const constraints = {
video: device_id ? { deviceId: { exact: device_id }, ...size } : size, video: device_id ? { deviceId: { exact: device_id }, ...fallback_constraints } : fallback_constraints,
audio: include_audio audio: include_audio
}; };

2
frontend/shared/webrtc_utils.ts
View File

@@ -48,8 +48,6 @@ export async function start(stream, pc: RTCPeerConnection, node, server_fn, webr
pc = createPeerConnection(pc, node); pc = createPeerConnection(pc, node);
if (stream) { if (stream) {
stream.getTracks().forEach((track) => { stream.getTracks().forEach((track) => {
if(modality == "video") track.applyConstraints({ frameRate: { max: 30 } });
else if(modality == "audio") track.applyConstraints({ sampleRate: 48000, channelCount: 1 });
console.debug("Track stream callback", track); console.debug("Track stream callback", track);
pc.addTrack(track, stream); pc.addTrack(track, stream);
}); });