deepgeek/gradio-webrtc
2
0
Fork 0
mirror of https://github.com/HumanAIGC-Engineering/gradio-webrtc.git synced 2026-02-04 17:39:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

code (#48)

Browse Source
This commit is contained in:
Freddy Boulton
2025-01-10 17:14:47 -05:00
committed by GitHub
parent b64e019323
commit 4d16634307
12 changed files with 431 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
frontend/shared/AudioWave.svelte
View File

@@ -1,10 +1,13 @@
<script lang="ts">
import { onDestroy } from 'svelte';
import type {ComponentType} from 'svelte';
import PulsingIcon from './PulsingIcon.svelte';
export let numBars = 16;
export let stream_state: "open" | "closed" | "waiting" = "closed";
export let audio_source_callback: () => MediaStream;
export let icon: string | undefined = undefined;
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
@@ -13,7 +16,6 @@
let dataArray: Uint8Array;
let animationId: number;
let pulseScale = 1;
let pulseIntensity = 0;
$: containerWidth = icon
? "128px"
@@ -47,53 +49,31 @@
function updateVisualization() {
analyser.getByteFrequencyData(dataArray);
if (icon) {
// Calculate average amplitude for pulse effect
const average = Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
const normalizedAverage = average / 255;
pulseScale = 1 + (normalizedAverage * 0.15);
pulseIntensity = normalizedAverage;
} else {
// Update bars
const bars = document.querySelectorAll('.gradio-webrtc-waveContainer .gradio-webrtc-box');
for (let i = 0; i < bars.length; i++) {
const barHeight = (dataArray[i] / 255) * 2;
bars[i].style.transform = `scaleY(${Math.max(0.1, barHeight)})`;
}
}
animationId = requestAnimationFrame(updateVisualization);
}
$: maxPulseScale = 1 + (pulseIntensity * 10); // Scale from 1x to 3x based on intensity
</script>
<div class="gradio-webrtc-waveContainer">
{#if icon}
<div class="gradio-webrtc-icon-container">
{#if pulseIntensity > 0}
{#each Array(3) as _, i}
<div
class="pulse-ring"
style:background={pulse_color}
style:animation-delay={`${i * 0.4}s`}
style:--max-scale={maxPulseScale}
style:opacity={0.5 * pulseIntensity}
/>
{/each}
{/if}
<div
class="gradio-webrtc-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
<img
src={icon}
alt="Audio visualization icon"
class="icon-image"
/>
<PulsingIcon
{stream_state}
{pulse_color}
{icon}
{icon_button_color}
{audio_source_callback}/>
</div>
</div>
{:else}

7
frontend/shared/InteractiveVideo.svelte
View File

@@ -1,5 +1,6 @@
<script lang="ts">
import { createEventDispatcher } from "svelte";
import type { ComponentType } from "svelte";
import type { FileData, Client } from "@gradio/client";
import { BlockLabel } from "@gradio/atoms";
import Webcam from "./Webcam.svelte";
@@ -24,6 +25,9 @@
export let mode: "send" | "send-receive";
export let on_change_cb: (msg: "change" | "tick") => void;
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
const dispatch = createEventDispatcher<{
change: FileData | null;
@@ -56,6 +60,9 @@
{mode}
{rtp_params}
{on_change_cb}
{icon}
{icon_button_color}
{pulse_color}
on:error
on:start_recording
on:stop_recording

151
frontend/shared/PulsingIcon.svelte Normal file
View File

@@ -0,0 +1,151 @@
<script lang="ts">
import { onDestroy } from 'svelte';
import type {ComponentType} from 'svelte';
export let stream_state: "open" | "closed" | "waiting" = "closed";
export let audio_source_callback: () => MediaStream;
export let icon: string | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
let audioContext: AudioContext;
let analyser: AnalyserNode;
let dataArray: Uint8Array;
let animationId: number;
let pulseScale = 1;
let pulseIntensity = 0;
$: if(stream_state === "open") setupAudioContext();
onDestroy(() => {
if (animationId) {
cancelAnimationFrame(animationId);
}
if (audioContext) {
audioContext.close();
}
});
function setupAudioContext() {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(audio_source_callback());
source.connect(analyser);
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.8;
dataArray = new Uint8Array(analyser.frequencyBinCount);
updateVisualization();
}
function updateVisualization() {
analyser.getByteFrequencyData(dataArray);
// Calculate average amplitude for pulse effect
const average = Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
const normalizedAverage = average / 255;
pulseScale = 1 + (normalizedAverage * 0.15);
pulseIntensity = normalizedAverage;
animationId = requestAnimationFrame(updateVisualization);
}
$: maxPulseScale = 1 + (pulseIntensity * 10); // Scale from 1x to 3x based on intensity
</script>
<div class="gradio-webrtc-icon-wrapper">
<div class="gradio-webrtc-pulsing-icon-container">
{#if pulseIntensity > 0}
{#each Array(3) as _, i}
<div
class="pulse-ring"
style:background={pulse_color}
style:animation-delay={`${i * 0.4}s`}
style:--max-scale={maxPulseScale}
style:opacity={0.5 * pulseIntensity}
/>
{/each}
{/if}
<div
class="gradio-webrtc-pulsing-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
{#if typeof icon === "string"}
<img
src={icon}
alt="Audio visualization icon"
class="icon-image"
/>
{:else}
<svelte:component this={icon} />
{/if}
</div>
</div>
</div>
<style>
.gradio-webrtc-icon-wrapper {
position: relative;
display: flex;
max-height: 128px;
justify-content: center;
align-items: center;
}
.gradio-webrtc-pulsing-icon-container {
position: relative;
width: 100%;
height: 100%;
display: flex;
justify-content: center;
align-items: center;
}
.gradio-webrtc-pulsing-icon {
position: relative;
width: 100%;
height: 100%;
border-radius: 50%;
transition: transform 0.1s ease;
display: flex;
justify-content: center;
align-items: center;
z-index: 2;
}
.icon-image {
width: 100%;
height: 100%;
object-fit: contain;
filter: brightness(0) invert(1);
}
.pulse-ring {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
width: 100%;
height: 100%;
border-radius: 50%;
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
opacity: 0.5;
}
@keyframes pulse {
0% {
transform: translate(-50%, -50%) scale(1);
opacity: 0.5;
}
100% {
transform: translate(-50%, -50%) scale(var(--max-scale, 3));
opacity: 0;
}
}

2
frontend/shared/StaticAudio.svelte
View File

@@ -98,7 +98,7 @@
/>
<audio
class="standard-player"
class:hidden={value === "__webrtc_value__"}
class:hidden={true}
on:load
bind:this={audio_player}
on:ended={() => dispatch("stop")}

44
frontend/shared/Webcam.svelte
View File

@@ -1,10 +1,12 @@
<script lang="ts">
import { createEventDispatcher, onMount } from "svelte";
import type { ComponentType } from "svelte";
import {
Circle,
Square,
DropdownArrow,
Spinner
Spinner,
Microphone as Mic
} from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
import { StreamingBar } from "@gradio/statustracker";
@@ -15,8 +17,8 @@
get_video_stream,
set_available_devices
} from "./stream_utils";
import { start, stop } from "./webrtc_utils";
import PulsingIcon from "./PulsingIcon.svelte";
let video_source: HTMLVideoElement;
let available_video_devices: MediaDeviceInfo[] = [];
@@ -28,6 +30,9 @@
export let mode: "send-receive" | "send";
const _webrtc_id = Math.random().toString(36).substring(2);
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export const modify_stream: (state: "open" | "closed" | "waiting") => void = (
state: "open" | "closed" | "waiting"
@@ -156,14 +161,13 @@
_time_limit = null;
await access_webcam();
}
}
window.setInterval(() => {
if (stream_state == "open") {
dispatch("tick");
}
}, stream_every * 1000);
// window.setInterval(() => {
// if (stream_state == "open") {
// dispatch("tick");
// }
// }, stream_every * 1000);
let options_open = false;
@@ -192,16 +196,29 @@
event.stopPropagation();
options_open = false;
}
const audio_source_callback = () => video_source.srcObject as MediaStream;
</script>
<div class="wrap">
<StreamingBar time_limit={_time_limit} />
{#if stream_state === "open" && include_audio}
<div class="audio-indicator">
<PulsingIcon
stream_state={stream_state}
audio_source_callback={audio_source_callback}
icon={icon || Mic}
icon_button_color={icon_button_color}
pulse_color={pulse_color}
/>
</div>
{/if}
<!-- svelte-ignore a11y-media-has-caption -->
<!-- need to suppress for video streaming https://github.com/sveltejs/svelte/issues/5967 -->
<video
bind:this={video_source}
class:hide={!webcam_accessed}
class:flip={(stream_state != "open")}
class:flip={(stream_state != "open") || (stream_state === "open" && include_audio)}
autoplay={true}
playsinline={true}
/>
@@ -324,6 +341,15 @@
justify-content: space-evenly;
}
.audio-indicator {
position: absolute;
top: var(--size-2);
right: var(--size-2);
z-index: var(--layer-2);
height: var(--size-5);
width: var(--size-5);
}
@media (--screen-md) {
button {
bottom: var(--size-4);

4
frontend/shared/webrtc_utils.ts
View File

@@ -68,14 +68,14 @@ export async function start(
try {
event_json = JSON.parse(event.data);
} catch (e) {
console.debug("Error parsing JSON")
console.debug("Error parsing JSON");
}
console.log("event_json", event_json);
if (
event.data === "change" ||
event.data === "tick" ||
event.data === "stopword" ||
event_json?.type === "warning" ||
event_json?.type === "warning" ||
event_json?.type === "error"
) {
console.debug(`${event.data} event received`);