deepgeek/gradio-webrtc
2
0
Fork 0
mirror of https://github.com/HumanAIGC-Engineering/gradio-webrtc.git synced 2026-02-04 17:39:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

add code

Browse Source
This commit is contained in:
freddyaboulton
2024-10-10 16:04:54 -07:00
parent 3777bfe777
commit 85cf0d726e
4 changed files with 172 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/gradio_webrtc/utils.py
View File

@@ -31,7 +31,6 @@ def player_worker_decode(
generator = None
while not thread_quit.is_set():
print("stream.latest_args", stream.latest_args)
if stream.latest_args == "not_set":
continue
if generator is None:
@@ -41,7 +40,7 @@ def player_worker_decode(
except Exception as exc:
if isinstance(exc, StopIteration):
print("Not iterating")
asyncio.run_coroutine_threadsafe(queue.put(frame), loop)
asyncio.run_coroutine_threadsafe(queue.put(None), loop)
thread_quit.set()
break
@@ -51,7 +50,8 @@ def player_worker_decode(
if frame_time and frame_time > elapsed_time + 1:
time.sleep(0.1)
sample_rate, audio_array = frame
frame = av.AudioFrame.from_ndarray(audio_array, format="s16", layout="mono")
format = "s16" if audio_array.dtype == "int16" else "fltp"
frame = av.AudioFrame.from_ndarray(audio_array, format=format, layout="mono")
frame.sample_rate = sample_rate
for frame in audio_resampler.resample(frame):
# fix timestamps

86
backend/gradio_webrtc/webrtc.py
View File

@@ -135,7 +135,9 @@ class ServerToClientVideo(VideoStreamTrack):
frame.time_base = time_base
return frame
elif self.generator is None:
self.generator = cast(Generator[Any, None, Any], self.event_handler(*self.latest_args))
self.generator = cast(
Generator[Any, None, Any], self.event_handler(*self.latest_args)
)
try:
next_array = next(self.generator)
@@ -150,6 +152,7 @@ class ServerToClientVideo(VideoStreamTrack):
except Exception as e:
print(e)
import traceback
traceback.print_exc()
@@ -189,41 +192,38 @@ class ServerToClientAudio(AudioStreamTrack):
self.current_timestamp += samples
frame.pts = self.current_timestamp
return frame
def start(self):
if self.__thread is None:
self.__thread = threading.Thread(
name="generator-runner",
target=player_worker_decode,
args=(
asyncio.get_event_loop(),
self.event_handler,
self,
self.queue,
False,
self.thread_quit
),
)
self.__thread.start()
def start(self):
if self.__thread is None:
self.__thread = threading.Thread(
name="generator-runner",
target=player_worker_decode,
args=(
asyncio.get_event_loop(),
self.event_handler,
self,
self.queue,
False,
self.thread_quit,
),
)
self.__thread.start()
async def recv(self):
try:
if self.readyState != "live":
raise MediaStreamError
self.start()
data = await self.queue.get()
if data is None:
self.stop()
raise MediaStreamError
return
data_time = data.time
# control playback rate
if (
data_time is not None
):
if data_time is not None:
if self._start is None:
self._start = time.time() - data_time
else:
@@ -238,35 +238,35 @@ class ServerToClientAudio(AudioStreamTrack):
traceback.print_exc()
def stop(self):
super().stop()
self.thread_quit.set()
if self.__thread is not None:
self.__thread.join()
self.__thread = None
super().stop()
# next_frame = await super().recv()
# print("next frame", next_frame)
# return next_frame
#try:
# if self.latest_args == "not_set":
# frame = await self.empty_frame()
# try:
# if self.latest_args == "not_set":
# frame = await self.empty_frame()
# # await self.modify_frame(frame)
# await asyncio.sleep(100 / 22050)
# print("next_frame not set", frame)
# return frame
# if self.generator is None:
# self.generator = cast(
# Generator[Any, None, Any], self.event_handler(*self.latest_args)
# )
# # await self.modify_frame(frame)
# await asyncio.sleep(100 / 22050)
# print("next_frame not set", frame)
# return frame
# if self.generator is None:
# self.generator = cast(
# Generator[Any, None, Any], self.event_handler(*self.latest_args)
# )
# try:
# next_array = next(self.generator)
# print("iteration")
# except StopIteration:
# print("exception")
# self.stop() # type: ignore
# return
# try:
# next_array = next(self.generator)
# print("iteration")
# except StopIteration:
# print("exception")
# self.stop() # type: ignore
# return
# next_frame = self.array_to_frame(next_array)
# # await self.modify_frame(next_frame)
# print("next frame", next_frame)
@@ -525,6 +525,7 @@ class WebRTC(Component):
cb = ServerToClientVideo(cast(Callable, self.event_handler))
pc.addTrack(cb)
self.connections[body["webrtc_id"]] = cb
cb.on("ended", lambda: self.connections.pop(body["webrtc_id"], None))
if self.mode == "receive" and self.modality == "audio":
print("adding")
cb = ServerToClientAudio(cast(Callable, self.event_handler))
@@ -534,6 +535,7 @@ class WebRTC(Component):
# pc.addTrack(player.audio)
pc.addTrack(cb)
self.connections[body["webrtc_id"]] = cb
cb.on("ended", lambda: self.connections.pop(body["webrtc_id"], None))
print("here")
# handle offer

121
frontend/shared/AudioWave.svelte Normal file
View File

@@ -0,0 +1,121 @@
<script lang="ts">
import { onMount, onDestroy } from 'svelte';
export let numBars = 16;
export let stream_state: "open" | "closed" = "closed";
export let audio_source: HTMLAudioElement;
let audioContext: AudioContext;
let analyser;
let dataArray;
let animationId;
let is_muted = false;
$: containerWidth = `calc((var(--boxSize) + var(--gutter)) * ${numBars})`;
$: if(stream_state === "open") setupAudioContext()
onDestroy(() => {
if (animationId) {
cancelAnimationFrame(animationId);
}
if (audioContext) {
audioContext.close();
}
});
function setupAudioContext() {
console.log("set up")
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
console.log("audio_source", audio_source.srcObject);
const source = audioContext.createMediaStreamSource(audio_source.srcObject);
source.connect(analyser);
analyser.connect(audioContext.destination);
analyser.fftSize = 64;
dataArray = new Uint8Array(analyser.frequencyBinCount);
updateBars();
}
function updateBars() {
analyser.getByteFrequencyData(dataArray);
const bars = document.querySelectorAll('.box');
for (let i = 0; i < bars.length; i++) {
const barHeight = (dataArray[i] / 255) * 2; // Amplify the effect
bars[i].style.transform = `scaleY(${Math.max(0.1, barHeight)})`;
}
animationId = requestAnimationFrame(updateBars);
}
function togglePlayPause() {
if (is_muted) {
audio_source.muted = false;
} else {
audio_source.muted = true;
}
is_muted = !is_muted;
}
</script>
<div class="waveContainer">
<div class="boxContainer" style:width={containerWidth}>
{#each Array(numBars) as _}
<div class="box"></div>
{/each}
</div>
<button class="playPauseButton" on:click={togglePlayPause}>
{is_muted ? '🔈' : '🔇'}
</button>
</div>
<style>
.waveContainer {
position: relative;
display: flex;
flex-direction: column;
align-items: center;
}
.boxContainer {
display: flex;
justify-content: space-between;
height: 64px;
--boxSize: 8px;
--gutter: 4px;
}
.box {
height: 100%;
width: var(--boxSize);
background: var(--color-accent);
border-radius: 8px;
transition: transform 0.05s ease;
}
.playPauseButton {
margin-top: 10px;
padding: 10px 20px;
font-size: 24px;
cursor: pointer;
background: none;
border: none;
border-radius: 5px;
color: black;
}
:global(body) {
display: flex;
justify-content: center;
background: black;
margin: 0;
padding: 0;
align-items: center;
height: 100vh;
color: white;
font-family: Arial, sans-serif;
}

6
frontend/shared/StaticAudio.svelte
View File

@@ -9,6 +9,7 @@
import { onMount } from "svelte";
import { start, stop } from "./webrtc_utils";
import AudioWave from "./AudioWave.svelte";
export let value: string | null = null;
@@ -56,7 +57,6 @@
]
};
pc = new RTCPeerConnection(rtc_configuration);
console.log("config", pc.getConfiguration());
pc.addEventListener("connectionstatechange",
async (event) => {
switch(pc.connectionState) {
@@ -95,12 +95,14 @@
<audio
class="standard-player"
class:hidden={value === "__webrtc_value__"}
controls
on:load
bind:this={audio_player}
on:ended={() => dispatch("stop")}
on:play={() => dispatch("play")}
/>
{#if value !== "__webrtc_value__"}
<AudioWave audio_source={audio_player} {stream_state}/>
{/if}
{#if value === "__webrtc_value__"}
<Empty size="small">
<Music />