deepgeek/gradio-webrtc
2
0
Fork 0
mirror of https://github.com/HumanAIGC-Engineering/gradio-webrtc.git synced 2026-02-04 17:39:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Customizable icon also fix a bug where you could not import the lib without silero (#39)

Browse Source 
* commit

* Add code

* Add docs
This commit is contained in:
Freddy Boulton
2024-12-13 16:53:35 -08:00
committed by GitHub
parent b97712bc0d
commit e92efb1c7d
8 changed files with 229 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
frontend/Index.svelte
View File

@@ -34,6 +34,9 @@
export let mode: "send-receive" | "receive" | "send" = "send-receive";
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let track_constraints: MediaTrackConstraints = {};
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
const on_change_cb = (msg: "change" | "tick") => {
gradio.dispatch(msg === "change" ? "state_change" : "tick");
@@ -84,6 +87,9 @@
{show_label}
{server}
{rtc_configuration}
{icon}
{icon_button_color}
{pulse_color}
i18n={gradio.i18n}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
@@ -130,6 +136,9 @@
{mode}
{rtp_params}
i18n={gradio.i18n}
{icon}
{icon_button_color}
{pulse_color}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
/>

222
frontend/shared/AudioWave.svelte
View File

@@ -1,85 +1,179 @@
<script lang="ts">
import { onMount, onDestroy } from 'svelte';
export let numBars = 16;
export let stream_state: "open" | "closed" | "waiting" = "closed";
export let audio_source_callback: () => MediaStream;
let audioContext: AudioContext;
let analyser: AnalyserNode;
let dataArray: Uint8Array;
let animationId: number;
$: containerWidth = `calc((var(--boxSize) + var(--gutter)) * ${numBars})`;
import { onDestroy } from 'svelte';
$: if(stream_state === "open") setupAudioContext()
onDestroy(() => {
if (animationId) {
cancelAnimationFrame(animationId);
}
if (audioContext) {
audioContext.close();
}
});
function setupAudioContext() {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(audio_source_callback());
// Only connect to analyser, not to destination
source.connect(analyser);
// Configure analyser
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.8; // Add smoothing to make visualization less jittery
dataArray = new Uint8Array(analyser.frequencyBinCount);
updateBars();
export let numBars = 16;
export let stream_state: "open" | "closed" | "waiting" = "closed";
export let audio_source_callback: () => MediaStream;
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
let audioContext: AudioContext;
let analyser: AnalyserNode;
let dataArray: Uint8Array;
let animationId: number;
let pulseScale = 1;
let pulseIntensity = 0;
$: containerWidth = icon
? "128px"
: `calc((var(--boxSize) + var(--gutter)) * ${numBars})`;
$: if(stream_state === "open") setupAudioContext();
onDestroy(() => {
if (animationId) {
cancelAnimationFrame(animationId);
}
function updateBars() {
analyser.getByteFrequencyData(dataArray);
if (audioContext) {
audioContext.close();
}
});
function setupAudioContext() {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(audio_source_callback());
source.connect(analyser);
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.8;
dataArray = new Uint8Array(analyser.frequencyBinCount);
updateVisualization();
}
function updateVisualization() {
analyser.getByteFrequencyData(dataArray);
if (icon) {
// Calculate average amplitude for pulse effect
const average = Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
const normalizedAverage = average / 255;
pulseScale = 1 + (normalizedAverage * 0.15);
pulseIntensity = normalizedAverage;
} else {
// Update bars
const bars = document.querySelectorAll('.gradio-webrtc-waveContainer .gradio-webrtc-box');
for (let i = 0; i < bars.length; i++) {
const barHeight = (dataArray[i] / 255) * 2; // Amplify the effect
const barHeight = (dataArray[i] / 255) * 2;
bars[i].style.transform = `scaleY(${Math.max(0.1, barHeight)})`;
}
animationId = requestAnimationFrame(updateBars);
}
animationId = requestAnimationFrame(updateVisualization);
}
</script>
<div class="gradio-webrtc-waveContainer">
{#if icon}
<div class="gradio-webrtc-icon-container">
{#if pulseIntensity > 0}
{#each Array(3) as _, i}
<div
class="pulse-ring"
style:background={pulse_color}
style:animation-delay={`${i * 0.4}s`}
/>
{/each}
{/if}
<div
class="gradio-webrtc-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
<img
src={icon}
alt="Audio visualization icon"
class="icon-image"
/>
</div>
</div>
{:else}
<div class="gradio-webrtc-boxContainer" style:width={containerWidth}>
{#each Array(numBars) as _}
<div class="gradio-webrtc-box"></div>
{/each}
</div>
{/if}
</div>
<style>
.gradio-webrtc-waveContainer {
position: relative;
display: flex;
min-height: 100px;
max-height: 128px;
}
.gradio-webrtc-waveContainer {
position: relative;
display: flex;
min-height: 100px;
max-height: 128px;
justify-content: center;
align-items: center;
}
.gradio-webrtc-boxContainer {
display: flex;
justify-content: space-between;
height: 64px;
--boxSize: 8px;
--gutter: 4px;
}
.gradio-webrtc-boxContainer {
display: flex;
justify-content: space-between;
height: 64px;
--boxSize: 8px;
--gutter: 4px;
}
.gradio-webrtc-box {
height: 100%;
width: var(--boxSize);
background: var(--color-accent);
border-radius: 8px;
transition: transform 0.05s ease;
.gradio-webrtc-box {
height: 100%;
width: var(--boxSize);
background: var(--color-accent);
border-radius: 8px;
transition: transform 0.05s ease;
}
.gradio-webrtc-icon-container {
position: relative;
width: 128px;
height: 128px;
display: flex;
justify-content: center;
align-items: center;
}
.gradio-webrtc-icon {
position: relative;
width: 48px;
height: 48px;
border-radius: 50%;
transition: transform 0.1s ease;
display: flex;
justify-content: center;
align-items: center;
z-index: 2;
}
.icon-image {
width: 32px;
height: 32px;
object-fit: contain;
filter: brightness(0) invert(1);
}
.pulse-ring {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
width: 48px;
height: 48px;
border-radius: 50%;
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
opacity: 0.5;
}
@keyframes pulse {
0% {
transform: translate(-50%, -50%) scale(1);
opacity: 0.5;
}
100% {
transform: translate(-50%, -50%) scale(3);
opacity: 0;
}
}
</style>

5
frontend/shared/InteractiveAudio.svelte
View File

@@ -31,6 +31,9 @@
export let track_constraints: MediaTrackConstraints = {};
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let on_change_cb: (mg: "tick" | "change") => void;
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
let stopword_recognized = false;
@@ -240,7 +243,7 @@
<WebcamPermissions icon={Microphone} on:click={async () => access_mic()} />
</div>
{:else}
<AudioWave {audio_source_callback} {stream_state}/>
<AudioWave {audio_source_callback} {stream_state} {icon} {icon_button_color} {pulse_color}/>
<StreamingBar time_limit={_time_limit} />
<div class="button-wrap" class:pulse={stopword_recognized}>
<button

5
frontend/shared/StaticAudio.svelte
View File

@@ -18,6 +18,9 @@
export let rtc_configuration: Object | null = null;
export let i18n: I18nFormatter;
export let on_change_cb: (msg: "change" | "tick") => void;
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let server: {
offer: (body: any) => Promise<any>;
@@ -103,7 +106,7 @@
/>
{#if value !== "__webrtc_value__"}
<div class="audio-container">
<AudioWave audio_source_callback={() => audio_player.srcObject} {stream_state}/>
<AudioWave audio_source_callback={() => audio_player.srcObject} {stream_state} {icon} {icon_button_color} {pulse_color}/>
</div>
{/if}
{#if value === "__webrtc_value__"}

8
frontend/shared/webrtc_utils.ts
View File

@@ -64,7 +64,11 @@ export async function start(
data_channel.onmessage = (event) => {
console.debug("Received message:", event.data);
if (event.data === "change" || event.data === "tick" || event.data === "stopword") {
if (
event.data === "change" ||
event.data === "tick" ||
event.data === "stopword"
) {
console.debug(`${event.data} event received`);
on_change_cb(event.data);
}
@@ -76,7 +80,7 @@ export async function start(
const sender = pc.addTrack(track, stream);
const params = sender.getParameters();
const updated_params = { ...params, ...rtp_params };
await sender.setParameters(updated_params)
await sender.setParameters(updated_params);
console.debug("sender params", sender.getParameters());
});
} else {