diff --git a/backend/gradio_webrtc/__init__.py b/backend/gradio_webrtc/__init__.py
index 2903fed..af3f6cd 100644
--- a/backend/gradio_webrtc/__init__.py
+++ b/backend/gradio_webrtc/__init__.py
@@ -1,3 +1,3 @@
-from .webrtc import WebRTC, StreamHandler
+from .webrtc import StreamHandler, WebRTC
__all__ = ["StreamHandler", "WebRTC"]
diff --git a/backend/gradio_webrtc/utils.py b/backend/gradio_webrtc/utils.py
index d7f6d4f..1f9cb55 100644
--- a/backend/gradio_webrtc/utils.py
+++ b/backend/gradio_webrtc/utils.py
@@ -2,7 +2,6 @@ import asyncio
import fractions
import logging
import threading
-import time
from typing import Callable
import av
@@ -15,35 +14,44 @@ AUDIO_PTIME = 0.020
def player_worker_decode(
loop,
- next: Callable,
+ next_frame: Callable,
queue: asyncio.Queue,
- throttle_playback: bool,
thread_quit: threading.Event,
+ quit_on_none: bool = False,
+ sample_rate: int = 48000,
+ frame_size: int = int(48000 * AUDIO_PTIME),
):
- audio_sample_rate = 48000
audio_samples = 0
- audio_time_base = fractions.Fraction(1, audio_sample_rate)
- audio_resampler = av.AudioResampler(
+ audio_time_base = fractions.Fraction(1, sample_rate)
+ audio_resampler = av.AudioResampler( # type: ignore
format="s16",
layout="stereo",
- rate=audio_sample_rate,
- frame_size=int(audio_sample_rate * AUDIO_PTIME),
+ rate=sample_rate,
+ frame_size=frame_size,
)
- frame_time = None
- start_time = time.time()
-
while not thread_quit.is_set():
- frame = next()
- logger.debug("emitted %s", frame)
- # read up to 1 second ahead
- if throttle_playback:
- elapsed_time = time.time() - start_time
- if frame_time and frame_time > elapsed_time + 1:
- time.sleep(0.1)
- sample_rate, audio_array = frame
+ frame = next_frame()
+ if frame is None:
+ if quit_on_none:
+ asyncio.run_coroutine_threadsafe(queue.put(None), loop)
+ continue
+
+ if len(frame) == 2:
+ sample_rate, audio_array = frame
+ layout = "mono"
+ elif len(frame) == 3:
+ sample_rate, audio_array, layout = frame
+
+ logger.debug(
+ "received array with shape %s sample rate %s layout %s",
+ audio_array.shape,
+ sample_rate,
+ layout,
+ )
format = "s16" if audio_array.dtype == "int16" else "fltp"
- frame = av.AudioFrame.from_ndarray(audio_array, format=format, layout="stereo")
+
+ frame = av.AudioFrame.from_ndarray(audio_array, format=format, layout=layout) # type: ignore
frame.sample_rate = sample_rate
for frame in audio_resampler.resample(frame):
# fix timestamps
@@ -51,5 +59,4 @@ def player_worker_decode(
frame.time_base = audio_time_base
audio_samples += frame.samples
- frame_time = frame.time
asyncio.run_coroutine_threadsafe(queue.put(frame), loop)
diff --git a/backend/gradio_webrtc/webrtc.py b/backend/gradio_webrtc/webrtc.py
index 955a541..92faa92 100644
--- a/backend/gradio_webrtc/webrtc.py
+++ b/backend/gradio_webrtc/webrtc.py
@@ -3,24 +3,25 @@
from __future__ import annotations
import asyncio
-from abc import ABC, abstractmethod
import logging
import threading
import time
import traceback
+from abc import ABC, abstractmethod
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Generator, Literal, Sequence, cast
import anyio.to_thread
+import av
import numpy as np
from aiortc import (
AudioStreamTrack,
+ MediaStreamTrack,
RTCPeerConnection,
RTCSessionDescription,
VideoStreamTrack,
- MediaStreamTrack,
)
-from aiortc.contrib.media import MediaRelay, AudioFrame, VideoFrame # type: ignore
+from aiortc.contrib.media import AudioFrame, MediaRelay, VideoFrame # type: ignore
from aiortc.mediastreams import MediaStreamError
from gradio import wasm_utils
from gradio.components.base import Component, server
@@ -104,6 +105,27 @@ class VideoCallback(VideoStreamTrack):
class StreamHandler(ABC):
+ def __init__(
+ self,
+ expected_layout: Literal["mono", "stereo"] = "mono",
+ output_sample_rate: int = 24000,
+ output_frame_size: int = 960,
+ ) -> None:
+ self.expected_layout = expected_layout
+ self.output_sample_rate = output_sample_rate
+ self.output_frame_size = output_frame_size
+ self._resampler = None
+
+ def resample(self, frame: AudioFrame) -> Generator[AudioFrame, None, None]:
+ if self._resampler is None:
+ self._resampler = av.AudioResampler( # type: ignore
+ format="s16",
+ layout=self.expected_layout,
+ rate=frame.sample_rate,
+ frame_size=frame.samples,
+ )
+ yield from self._resampler.resample(frame)
+
@abstractmethod
def receive(self, frame: tuple[int, np.ndarray] | np.ndarray) -> None:
pass
@@ -124,24 +146,27 @@ class AudioCallback(AudioStreamTrack):
self.track = track
self.event_handler = event_handler
self.current_timestamp = 0
- self.latest_args = "not_set"
+ self.latest_args: str | list[Any] = "not_set"
self.queue = asyncio.Queue()
self.thread_quit = threading.Event()
self.__thread = None
self._start: float | None = None
self.has_started = False
+ self.last_timestamp = 0
super().__init__()
async def process_input_frames(self) -> None:
while not self.thread_quit.is_set():
try:
frame = cast(AudioFrame, await self.track.recv())
- numpy_array = frame.to_ndarray()
- logger.debug("numpy array shape %s", numpy_array.shape)
- await anyio.to_thread.run_sync(
- self.event_handler.receive, (frame.sample_rate, numpy_array)
- )
- except MediaStreamError:
+ for frame in self.event_handler.resample(frame):
+ numpy_array = frame.to_ndarray()
+ logger.debug("numpy array shape %s", numpy_array.shape)
+ await anyio.to_thread.run_sync(
+ self.event_handler.receive, (frame.sample_rate, numpy_array)
+ )
+ except MediaStreamError as e:
+ print("MediaStreamError", e)
break
def start(self):
@@ -154,8 +179,10 @@ class AudioCallback(AudioStreamTrack):
asyncio.get_event_loop(),
self.event_handler.emit,
self.queue,
- True,
self.thread_quit,
+ False,
+ self.event_handler.output_sample_rate,
+ self.event_handler.output_frame_size,
),
)
self.__thread.start()
@@ -167,23 +194,25 @@ class AudioCallback(AudioStreamTrack):
raise MediaStreamError
self.start()
- data = await self.queue.get()
- logger.debug("data %s", data)
- if data is None:
- self.stop()
- return
+ frame = await self.queue.get()
+ logger.debug("frame %s", frame)
- data_time = data.time
+ data_time = frame.time
+
+ if time.time() - self.last_timestamp > 10 * (
+ self.event_handler.output_frame_size
+ / self.event_handler.output_sample_rate
+ ):
+ self._start = None
# control playback rate
- if data_time is not None:
- if self._start is None:
- self._start = time.time() - data_time
- else:
- wait = self._start + data_time - time.time()
- await asyncio.sleep(wait)
-
- return data
+ if self._start is None:
+ self._start = time.time() - data_time
+ else:
+ wait = self._start + data_time - time.time()
+ await asyncio.sleep(wait)
+ self.last_timestamp = time.time()
+ return frame
except Exception as e:
logger.debug(e)
exec = traceback.format_exc()
@@ -210,6 +239,7 @@ class ServerToClientVideo(VideoStreamTrack):
) -> None:
super().__init__() # don't forget this!
self.event_handler = event_handler
+ self.args_set = asyncio.Event()
self.latest_args: str | list[Any] = "not_set"
self.generator: Generator[Any, None, Any] | None = None
@@ -219,12 +249,8 @@ class ServerToClientVideo(VideoStreamTrack):
async def recv(self):
try:
pts, time_base = await self.next_timestamp()
- if self.latest_args == "not_set":
- frame = self.array_to_frame(np.zeros((480, 640, 3), dtype=np.uint8))
- frame.pts = pts
- frame.time_base = time_base
- return frame
- elif self.generator is None:
+ await self.args_set.wait()
+ if self.generator is None:
self.generator = cast(
Generator[Any, None, Any], self.event_handler(*self.latest_args)
)
@@ -255,7 +281,8 @@ class ServerToClientAudio(AudioStreamTrack):
self.generator: Generator[Any, None, Any] | None = None
self.event_handler = event_handler
self.current_timestamp = 0
- self.latest_args = "not_set"
+ self.latest_args: str | list[Any] = "not_set"
+ self.args_set = threading.Event()
self.queue = asyncio.Queue()
self.thread_quit = threading.Event()
self.__thread = None
@@ -263,23 +290,15 @@ class ServerToClientAudio(AudioStreamTrack):
super().__init__()
def next(self) -> tuple[int, np.ndarray] | None:
- import anyio
-
- if self.latest_args == "not_set":
- return
+ self.args_set.wait()
if self.generator is None:
self.generator = self.event_handler(*self.latest_args)
if self.generator is not None:
try:
frame = next(self.generator)
return frame
- except Exception as exc:
- if isinstance(exc, StopIteration):
- logger.debug("Stopping audio stream")
- asyncio.run_coroutine_threadsafe(
- self.queue.put(None), asyncio.get_event_loop()
- )
- self.thread_quit.set()
+ except StopIteration:
+ pass
def start(self):
if self.__thread is None:
@@ -290,8 +309,8 @@ class ServerToClientAudio(AudioStreamTrack):
asyncio.get_event_loop(),
self.next,
self.queue,
- False,
self.thread_quit,
+ True,
),
)
self.__thread.start()
@@ -370,6 +389,7 @@ class WebRTC(Component):
key: int | str | None = None,
mirror_webcam: bool = True,
rtc_configuration: dict[str, Any] | None = None,
+ track_constraints: dict[str, Any] | None = None,
time_limit: float | None = None,
mode: Literal["send-receive", "receive"] = "send-receive",
modality: Literal["video", "audio"] = "video",
@@ -412,7 +432,24 @@ class WebRTC(Component):
self.rtc_configuration = rtc_configuration
self.mode = mode
self.modality = modality
- self.event_handler: Callable | None = None
+ if track_constraints is None and modality == "audio":
+ track_constraints = {
+ "echoCancellation": True,
+ "noiseSuppression": {"exact": True},
+ "autoGainControl": {"exact": True},
+ "sampleRate": {"ideal": 24000},
+ "sampleSize": {"ideal": 16},
+ "channelCount": {"exact": 1},
+ }
+ if track_constraints is None and modality == "video":
+ track_constraints = {
+ "facingMode": "user",
+ "width": {"ideal": 500},
+ "height": {"ideal": 500},
+ "frameRate": {"ideal": 30},
+ }
+ self.track_constraints = track_constraints
+ self.event_handler: Callable | StreamHandler | None = None
super().__init__(
label=label,
every=every,
@@ -456,6 +493,7 @@ class WebRTC(Component):
)
elif self.mode == "receive":
self.connections[webrtc_id].latest_args = list(args)
+ self.connections[webrtc_id].args_set.set() # type: ignore
def stream(
self,
@@ -534,9 +572,9 @@ class WebRTC(Component):
"In the receive mode stream event, the trigger parameter must be provided"
)
trigger(lambda: "start_webrtc_stream", inputs=None, outputs=self)
- self.tick(
+ self.tick( # type: ignore
self.set_output,
- inputs=[self] + inputs,
+ inputs=[self] + list(inputs),
outputs=None,
concurrency_id=concurrency_id,
)
diff --git a/demo/app.py b/demo/app.py
index 12e3204..1ff27ba 100644
--- a/demo/app.py
+++ b/demo/app.py
@@ -1,22 +1,178 @@
-import logging
-# Configure the root logger to WARNING to suppress debug messages from other libraries
-logging.basicConfig(level=logging.WARNING)
+import os
-# Create a console handler
-console_handler = logging.StreamHandler()
-console_handler.setLevel(logging.DEBUG)
+import gradio as gr
-# Create a formatter
-formatter = logging.Formatter("%(name)s - %(levelname)s - %(message)s")
-console_handler.setFormatter(formatter)
-
-# Configure the logger for your specific library
-logger = logging.getLogger("gradio_webrtc")
-logger.setLevel(logging.DEBUG)
-logger.addHandler(console_handler)
+_docs = {'WebRTC':
+ {'description': 'Stream audio/video with WebRTC',
+ 'members': {'__init__':
+ {
+ 'rtc_configuration': {'type': 'dict[str, Any] | None', 'default': 'None', 'description': "The configration dictionary to pass to the RTCPeerConnection constructor. If None, the default configuration is used."},
+ 'height': {'type': 'int | str | None', 'default': 'None', 'description': 'The height of the component, specified in pixels if a number is passed, or in CSS units if a string is passed. This has no effect on the preprocessed video file, but will affect the displayed video.'},
+ 'width': {'type': 'int | str | None', 'default': 'None', 'description': 'The width of the component, specified in pixels if a number is passed, or in CSS units if a string is passed. This has no effect on the preprocessed video file, but will affect the displayed video.'},
+ 'label': {'type': 'str | None', 'default': 'None', 'description': 'the label for this component. Appears above the component and is also used as the header if there are a table of examples for this component. If None and used in a `gr.Interface`, the label will be the name of the parameter this component is assigned to.'},
+ 'show_label': {'type': 'bool | None', 'default': 'None', 'description': 'if True, will display label.'}, 'container': {'type': 'bool', 'default': 'True', 'description': 'if True, will place the component in a container - providing some extra padding around the border.'},
+ 'scale': {'type': 'int | None', 'default': 'None', 'description': 'relative size compared to adjacent Components. For example if Components A and B are in a Row, and A has scale=2, and B has scale=1, A will be twice as wide as B. Should be an integer. scale applies in Rows, and to top-level Components in Blocks where fill_height=True.'},
+ 'min_width': {'type': 'int', 'default': '160', 'description': 'minimum pixel width, will wrap if not sufficient screen space to satisfy this value. If a certain scale value results in this Component being narrower than min_width, the min_width parameter will be respected first.'},
+ 'interactive': {'type': 'bool | None', 'default': 'None', 'description': 'if True, will allow users to upload a video; if False, can only be used to display videos. If not provided, this is inferred based on whether the component is used as an input or output.'}, 'visible': {'type': 'bool', 'default': 'True', 'description': 'if False, component will be hidden.'},
+ 'elem_id': {'type': 'str | None', 'default': 'None', 'description': 'an optional string that is assigned as the id of this component in the HTML DOM. Can be used for targeting CSS styles.'},
+ 'elem_classes': {'type': 'list[str] | str | None', 'default': 'None', 'description': 'an optional list of strings that are assigned as the classes of this component in the HTML DOM. Can be used for targeting CSS styles.'},
+ 'render': {'type': 'bool', 'default': 'True', 'description': 'if False, component will not render be rendered in the Blocks context. Should be used if the intention is to assign event listeners now but render the component later.'},
+ 'key': {'type': 'int | str | None', 'default': 'None', 'description': 'if assigned, will be used to assume identity across a re-render. Components that have the same key across a re-render will have their value preserved.'},
+ 'mirror_webcam': {'type': 'bool', 'default': 'True', 'description': 'if True webcam will be mirrored. Default is True.'},
+ },
+ 'events': {'tick': {'type': None, 'default': None, 'description': ''}}}, '__meta__': {'additional_interfaces': {}, 'user_fn_refs': {'WebRTC': []}}}
+}
+abs_path = os.path.join(os.path.dirname(__file__), "css.css")
+
+with gr.Blocks(
+ css_paths=abs_path,
+ theme=gr.themes.Default(
+ font_mono=[
+ gr.themes.GoogleFont("Inconsolata"),
+ "monospace",
+ ],
+ ),
+) as demo:
+ gr.Markdown(
+"""
+ Gradio WebRTC ⚡️
+
+
+
+
+
- Audio Streaming (Powered by WebRTC ⚡️)
-
- """
- )
- with gr.Column(elem_classes=["my-column"]):
- with gr.Group(elem_classes=["my-group"]):
+ with gr.Column():
+ with gr.Group():
audio = WebRTC(
label="Stream",
rtc_configuration=None,
@@ -61,3 +207,85 @@ with gr.Blocks() as demo:
if __name__ == "__main__":
demo.launch()
+```
+
+* Instead of passing a function to the `stream` event's `fn` parameter, pass a `StreamHandler` implementation. The `StreamHandler` above simply echoes the audio back to the client.
+* The `StreamHandler` class has two methods: `receive` and `emit`. The `receive` method is called when a new frame is received from the client, and the `emit` method returns the next frame to send to the client.
+* An audio frame is represented as a tuple of (frame_rate, audio_samples) where `audio_samples` is a numpy array of shape (num_channels, num_samples).
+* You can also specify the audio layout ("mono" or "stereo") in the emit method by retuning it as the third element of the tuple. If not specified, the default is "mono".
+* The `time_limit` parameter is the maximum time in seconds the conversation will run. If the time limit is reached, the audio stream will stop.
+
+
+## Deployment
+
+When deploying in a cloud environment (like Hugging Face Spaces, EC2, etc), you need to set up a TURN server to relay the WebRTC traffic.
+The easiest way to do this is to use a service like Twilio.
+
+```python
+from twilio.rest import Client
+import os
+
+account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
+auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
+
+client = Client(account_sid, auth_token)
+
+token = client.tokens.create()
+
+rtc_configuration = {
+ "iceServers": token.ice_servers,
+ "iceTransportPolicy": "relay",
+}
+
+with gr.Blocks() as demo:
+ ...
+ rtc = WebRTC(rtc_configuration=rtc_configuration, ...)
+ ...
+```
+""", elem_classes=["md-custom"], header_links=True)
+
+
+ gr.Markdown("""
+##
+""", elem_classes=["md-custom"], header_links=True)
+
+ gr.ParamViewer(value=_docs["WebRTC"]["members"]["__init__"], linkify=[])
+
+
+ demo.load(None, js=r"""function() {
+ const refs = {};
+ const user_fn_refs = {
+ WebRTC: [], };
+ requestAnimationFrame(() => {
+
+ Object.entries(user_fn_refs).forEach(([key, refs]) => {
+ if (refs.length > 0) {
+ const el = document.querySelector(`.${key}-user-fn`);
+ if (!el) return;
+ refs.forEach(ref => {
+ el.innerHTML = el.innerHTML.replace(
+ new RegExp("\\b"+ref+"\\b", "g"),
+ `${ref}`
+ );
+ })
+ }
+ })
+
+ Object.entries(refs).forEach(([key, refs]) => {
+ if (refs.length > 0) {
+ const el = document.querySelector(`.${key}`);
+ if (!el) return;
+ refs.forEach(ref => {
+ el.innerHTML = el.innerHTML.replace(
+ new RegExp("\\b"+ref+"\\b", "g"),
+ `${ref}`
+ );
+ })
+ }
+ })
+ })
+}
+
+""")
+
+demo.launch()
\ No newline at end of file
diff --git a/demo/app_orig.py b/demo/app_orig.py
index 3489fab..31f3b3f 100644
--- a/demo/app_orig.py
+++ b/demo/app_orig.py
@@ -1,10 +1,11 @@
-import gradio as gr
-import cv2
-from huggingface_hub import hf_hub_download
-from gradio_webrtc import WebRTC
-from twilio.rest import Client
import os
+
+import cv2
+import gradio as gr
+from gradio_webrtc import WebRTC
+from huggingface_hub import hf_hub_download
from inference import YOLOv10
+from twilio.rest import Client
model_file = hf_hub_download(
repo_id="onnx-community/yolov10n", filename="onnx/model.onnx"
diff --git a/demo/audio_out.py b/demo/audio_out.py
index b2da4a8..879ba67 100644
--- a/demo/audio_out.py
+++ b/demo/audio_out.py
@@ -1,10 +1,10 @@
+import os
+
import gradio as gr
import numpy as np
from gradio_webrtc import WebRTC
-from twilio.rest import Client
-import os
from pydub import AudioSegment
-
+from twilio.rest import Client
account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
@@ -33,8 +33,6 @@ def generation(num_steps):
segment.frame_rate,
np.array(segment.get_array_of_samples()).reshape(1, -1),
)
- time.sleep(3.5)
-
css = """.my-group {max-width: 600px !important; max-height: 600 !important;}
.my-column {display: flex !important; justify-content: center !important; align-items: center !important};"""
diff --git a/demo/audio_out_2.py b/demo/audio_out_2.py
index ec885b3..8a8238f 100644
--- a/demo/audio_out_2.py
+++ b/demo/audio_out_2.py
@@ -1,10 +1,10 @@
+import os
+
import gradio as gr
import numpy as np
from gradio_webrtc import WebRTC
-from twilio.rest import Client
-import os
from pydub import AudioSegment
-
+from twilio.rest import Client
account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
diff --git a/demo/inference.py b/demo/inference.py
index 1e4d9a8..7b3fdff 100644
--- a/demo/inference.py
+++ b/demo/inference.py
@@ -1,8 +1,8 @@
import time
+
import cv2
import numpy as np
import onnxruntime
-
from utils import draw_detections
@@ -120,8 +120,9 @@ class YOLOv10:
if __name__ == "__main__":
- import requests
import tempfile
+
+ import requests
from huggingface_hub import hf_hub_download
model_file = hf_hub_download(
diff --git a/demo/space.py b/demo/space.py
index 36a54de..7b7bc06 100644
--- a/demo/space.py
+++ b/demo/space.py
@@ -1,6 +1,7 @@
-import gradio as gr
import os
+import gradio as gr
+
_docs = {
"WebRTC": {
"description": "Stream audio/video with WebRTC",
diff --git a/demo/utils.py b/demo/utils.py
index 8cdd1d2..04dadeb 100644
--- a/demo/utils.py
+++ b/demo/utils.py
@@ -1,5 +1,5 @@
-import numpy as np
import cv2
+import numpy as np
class_names = [
"person",
diff --git a/demo/video_out.py b/demo/video_out.py
index 298614d..e244b51 100644
--- a/demo/video_out.py
+++ b/demo/video_out.py
@@ -1,9 +1,9 @@
+import os
+
+import cv2
import gradio as gr
from gradio_webrtc import WebRTC
from twilio.rest import Client
-import os
-import cv2
-
account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
diff --git a/demo/video_out_stream.py b/demo/video_out_stream.py
index 49461ed..a5689ca 100644
--- a/demo/video_out_stream.py
+++ b/demo/video_out_stream.py
@@ -1,9 +1,9 @@
+import os
+
+import cv2
import gradio as gr
from gradio_webrtc import WebRTC
from twilio.rest import Client
-import os
-import cv2
-
account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
diff --git a/frontend/Index.svelte b/frontend/Index.svelte
index b3ba5db..242841e 100644
--- a/frontend/Index.svelte
+++ b/frontend/Index.svelte
@@ -32,6 +32,7 @@
export let time_limit: number | null = null;
export let modality: "video" | "audio" = "video";
export let mode: "send-receive" | "receive" = "send-receive";
+ export let track_constraints: MediaTrackConstraints = {};
let dragging = false;
@@ -113,6 +114,7 @@
{server}
{rtc_configuration}
{time_limit}
+ {track_constraints}
i18n={gradio.i18n}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
diff --git a/frontend/shared/AudioWave.svelte b/frontend/shared/AudioWave.svelte
index 8d8c19e..81f3c34 100644
--- a/frontend/shared/AudioWave.svelte
+++ b/frontend/shared/AudioWave.svelte
@@ -25,7 +25,6 @@
});
function setupAudioContext() {
- console.log("set up")
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
console.log("audio_source", audio_source.srcObject);
@@ -50,16 +49,6 @@
animationId = requestAnimationFrame(updateBars);
}
-
- function toggleMute() {
- if (audio_source && audio_source.srcObject) {
- const audioTracks = (audio_source.srcObject as MediaStream).getAudioTracks();
- audioTracks.forEach(track => {
- track.enabled = !track.enabled;
- });
- is_muted = !audioTracks[0].enabled;
- }
- }
@@ -75,6 +64,8 @@
.waveContainer {
position: relative;
display: flex;
+ min-height: 100px;
+ max-height: 128px;
}
.boxContainer {
diff --git a/frontend/shared/InteractiveAudio.svelte b/frontend/shared/InteractiveAudio.svelte
index 6f7f043..5dd3ef3 100644
--- a/frontend/shared/InteractiveAudio.svelte
+++ b/frontend/shared/InteractiveAudio.svelte
@@ -24,6 +24,7 @@
export let rtc_configuration: Object | null = null;
export let i18n: I18nFormatter;
export let time_limit: number | null = null;
+ export let track_constraints: MediaTrackConstraints = {};
let _time_limit: number | null = null;
$: console.log("time_limit", time_limit);
@@ -87,14 +88,7 @@
let stream = null
try {
- stream = await navigator.mediaDevices.getUserMedia({ audio: {
- echoCancellation: true,
- noiseSuppression: {exact: true},
- autoGainControl: {exact: true},
- sampleRate: {ideal: 48000},
- sampleSize: {ideal: 16},
- channelCount: 2,
- } });
+ stream = await navigator.mediaDevices.getUserMedia({ audio: track_constraints });
} catch (err) {
if (!navigator.mediaDevices) {
dispatch("error", i18n("audio.no_device_support"));
diff --git a/frontend/shared/InteractiveVideo.svelte b/frontend/shared/InteractiveVideo.svelte
index 9239a7c..5794c70 100644
--- a/frontend/shared/InteractiveVideo.svelte
+++ b/frontend/shared/InteractiveVideo.svelte
@@ -20,6 +20,7 @@
offer: (body: any) => Promise;
};
export let rtc_configuration: Object;
+ export let track_constraints: MediaTrackConstraints = {};
const dispatch = createEventDispatcher<{
change: FileData | null;
@@ -48,6 +49,7 @@
{rtc_configuration}
{include_audio}
{time_limit}
+ {track_constraints}
on:error
on:start_recording
on:stop_recording
diff --git a/frontend/shared/StaticAudio.svelte b/frontend/shared/StaticAudio.svelte
index 05cf77b..69b4f88 100644
--- a/frontend/shared/StaticAudio.svelte
+++ b/frontend/shared/StaticAudio.svelte
@@ -22,7 +22,7 @@
offer: (body: any) => Promise;
};
- let stream_state: "open" | "closed" | "connecting" = "closed";
+ let stream_state: "open" | "closed" | "waiting" = "closed";
let audio_player: HTMLAudioElement;
let pc: RTCPeerConnection;
let _webrtc_id = Math.random().toString(36).substring(2);
@@ -35,7 +35,6 @@
stop: undefined;
}>();
-
onMount(() => {
window.setInterval(() => {
if (stream_state == "open") {
@@ -45,10 +44,11 @@
}
)
- async function start_stream(value: string): Promise {
+ async function start_stream(value: string): Promise {
if( value === "start_webrtc_stream") {
- stream_state = "connecting";
+ stream_state = "waiting";
value = _webrtc_id;
+ console.log("set value to ", value);
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange",
async (event) => {
@@ -74,9 +74,12 @@
dispatch("error", "Too many concurrent users. Come back later!");
});
}
+ return value;
}
- $: start_stream(value);
+ $: start_stream(value).then((val) => {
+ value = val;
+ });
@@ -97,23 +100,28 @@
on:play={() => dispatch("play")}
/>
{#if value !== "__webrtc_value__"}
+
{/if}
{#if value === "__webrtc_value__"}
-
-
-
+
+
+
{/if}