deepgeek/gradio-webrtc
2
0
Fork 0
mirror of https://github.com/HumanAIGC-Engineering/gradio-webrtc.git synced 2026-02-05 18:09:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Rebrand to FastRTC (#60)

Browse Source 
* Add code

* add code

* add code

* Rename messages

* rename

* add code

* Add demo

* docs + demos + bug fixes

* add code

* styles

* user guide

* Styles

* Add code

* misc docs updates

* print nit

* whisper + pr

* url for images

* whsiper update

* Fix bugs

* remove demo files

* version number

* Fix pypi readme

* Fix

* demos

* Add llama code editor

* Update llama code editor and object detection cookbook

* Add more cookbook demos

* add code

* Fix links for PR deploys

* add code

* Fix the install

* add tts

* TTS docs

* Typo

* Pending bubbles for reply on pause

* Stream redesign (#63)

* better error handling

* Websocket error handling

* add code

---------

Co-authored-by: Freddy Boulton <freddyboulton@hf-freddy.local>

* remove docs from dist

* Some docs typos

* more typos

* upload changes + docs

* docs

* better phone

* update docs

* add code

* Make demos better

* fix docs + websocket start_up

* remove mention of FastAPI app

* fastphone tweaks

* add code

* ReplyOnStopWord fixes

* Fix cookbook

* Fix pypi readme

* add code

* bump versions

* sambanova cookbook

* Fix tags

* Llm voice chat

* kyutai tag

* Add error message to all index.html

* STT module uses Moonshine

* Not required from typing extensions

* fix llm voice chat

* Add vpn warning

* demo fixes

* demos

* Add more ui args and gemini audio-video

* update cookbook

* version 9

---------

Co-authored-by: Freddy Boulton <freddyboulton@hf-freddy.local>
This commit is contained in:
Freddy Boulton
2025-02-24 01:13:42 -05:00
committed by GitHub
parent 36190066ec
commit 853d6a06b5
131 changed files with 12349 additions and 4741 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
frontend/Index.svelte
View File

@@ -13,7 +13,7 @@
export let elem_classes: string[] = [];
export let visible = true;
export let value: string = "__webrtc_value__";
export let button_labels: {start: string, stop: string, waiting: string};
export let button_labels: { start: string; stop: string; waiting: string };
export let label: string;
export let root: string;
@@ -40,16 +40,45 @@
export let pulse_color: string = "var(--color-accent)";
const on_change_cb = (msg: "change" | "tick" | any) => {
if (msg?.type === "info" || msg?.type === "warning" || msg?.type === "error") {
console.log("dispatching info", msg.message);
gradio.dispatch(msg?.type === "error"? "error": "warning", msg.message);
if (
msg?.type === "info" ||
msg?.type === "warning" ||
msg?.type === "error"
) {
gradio.dispatch(
msg?.type === "error" ? "error" : "warning",
msg.message,
);
} else if (msg?.type === "fetch_output") {
gradio.dispatch("state_change");
} else if (msg?.type === "send_input") {
gradio.dispatch("tick");
} else if (msg?.type === "connection_timeout") {
gradio.dispatch(
"warning",
"Taking a while to connect. Are you on a VPN?",
);
}
gradio.dispatch(msg === "change" ? "state_change" : "tick");
}
if (msg.type === "state_change") {
gradio.dispatch(msg === "change" ? "state_change" : "tick");
}
};
const reject_cb = (msg: object) => {
if (
msg.status === "failed" &&
msg.meta?.error === "concurrency_limit_reached"
) {
gradio.dispatch(
"error",
`Too many concurrent connections. Please try again later!`,
);
} else {
gradio.dispatch("error", "Unexpected server error");
}
};
let dragging = false;
$: console.log("value", value);
</script>
<Block
@@ -65,17 +94,17 @@
{scale}
{min_width}
allow_overflow={false}
>
<StatusTracker
autoscroll={gradio.autoscroll}
i18n={gradio.i18n}
{...loading_status}
on:clear_status={() => gradio.dispatch("clear_status", loading_status)}
/>
>
<StatusTracker
autoscroll={gradio.autoscroll}
i18n={gradio.i18n}
{...loading_status}
on:clear_status={() => gradio.dispatch("clear_status", loading_status)}
/>
{#if mode == "receive" && modality === "video"}
<StaticVideo
bind:value={value}
bind:value
{on_change_cb}
{label}
{show_label}
@@ -86,7 +115,7 @@
/>
{:else if mode == "receive" && modality === "audio"}
<StaticAudio
bind:value={value}
bind:value
{on_change_cb}
{label}
{show_label}
@@ -98,11 +127,10 @@
i18n={gradio.i18n}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
/>
{:else if (mode === "send-receive" || mode == "send") && (modality === "video" || modality == "audio-video")}
<Video
bind:value={value}
bind:value
{label}
{show_label}
active_source={"webcam"}
@@ -114,6 +142,7 @@
{track_constraints}
{rtp_params}
{on_change_cb}
{reject_cb}
{icon}
{icon_button_color}
{pulse_color}
@@ -135,7 +164,7 @@
</Video>
{:else if (mode === "send-receive" || mode === "send") && modality === "audio"}
<InteractiveAudio
bind:value={value}
bind:value
{on_change_cb}
{label}
{show_label}
@@ -147,6 +176,7 @@
{rtp_params}
i18n={gradio.i18n}
{icon}
{reject_cb}
{icon_button_color}
{pulse_color}
{button_labels}

241
frontend/shared/AudioWave.svelte
View File

@@ -1,8 +1,8 @@
<script lang="ts">
import { onDestroy } from 'svelte';
import type {ComponentType} from 'svelte';
import { onDestroy } from "svelte";
import type { ComponentType } from "svelte";
import PulsingIcon from './PulsingIcon.svelte';
import PulsingIcon from "./PulsingIcon.svelte";
export let numBars = 16;
export let stream_state: "open" | "closed" | "waiting" = "closed";
@@ -10,18 +10,19 @@
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let pending: boolean = false;
let audioContext: AudioContext;
let analyser: AnalyserNode;
let dataArray: Uint8Array;
let animationId: number;
let pulseScale = 1;
export let pulseScale = 1;
$: containerWidth = icon
$: containerWidth = icon
? "128px"
: `calc((var(--boxSize) + var(--gutter)) * ${numBars})`;
$: if(stream_state === "open") setupAudioContext();
$: if (stream_state === "open") setupAudioContext();
onDestroy(() => {
if (animationId) {
@@ -35,10 +36,12 @@
function setupAudioContext() {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(audio_source_callback());
const source = audioContext.createMediaStreamSource(
audio_source_callback(),
);
source.connect(analyser);
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.8;
dataArray = new Uint8Array(analyser.frequencyBinCount);
@@ -48,110 +51,119 @@
function updateVisualization() {
analyser.getByteFrequencyData(dataArray);
// Update bars
const bars = document.querySelectorAll('.gradio-webrtc-waveContainer .gradio-webrtc-box');
for (let i = 0; i < bars.length; i++) {
const barHeight = (dataArray[i] / 255) * 2;
bars[i].style.transform = `scaleY(${Math.max(0.1, barHeight)})`;
}
// Update bars
const bars = document.querySelectorAll(
".gradio-webrtc-waveContainer .gradio-webrtc-box",
);
for (let i = 0; i < bars.length; i++) {
const barHeight = (dataArray[i] / 255) * 2;
bars[i].style.transform = `scaleY(${Math.max(0.1, barHeight)})`;
}
animationId = requestAnimationFrame(updateVisualization);
}
</script>
<div class="gradio-webrtc-waveContainer">
{#if icon}
<div class="gradio-webrtc-icon-container">
<div
class="gradio-webrtc-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
<PulsingIcon
{stream_state}
{pulse_color}
{icon}
{icon_button_color}
{audio_source_callback}/>
{#if icon && !pending}
<div class="gradio-webrtc-icon-container">
<div
class="gradio-webrtc-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
<PulsingIcon
{stream_state}
{pulse_color}
{icon}
{icon_button_color}
{audio_source_callback}
/>
</div>
</div>
</div>
{:else}
<div class="gradio-webrtc-boxContainer" style:width={containerWidth}>
{#each Array(numBars) as _}
<div class="gradio-webrtc-box"></div>
{/each}
</div>
{/if}
{:else if pending}
<div class="dots">
<div class="dot" style:background-color={pulse_color} />
<div class="dot" style:background-color={pulse_color} />
<div class="dot" style:background-color={pulse_color} />
</div>
{:else}
<div class="gradio-webrtc-boxContainer" style:width={containerWidth}>
{#each Array(numBars) as _}
<div class="gradio-webrtc-box"></div>
{/each}
</div>
{/if}
</div>
<style>
.gradio-webrtc-waveContainer {
position: relative;
display: flex;
min-height: 100px;
max-height: 128px;
justify-content: center;
align-items: center;
}
.gradio-webrtc-waveContainer {
position: relative;
display: flex;
min-height: 100px;
max-height: 128px;
justify-content: center;
align-items: center;
}
.gradio-webrtc-boxContainer {
display: flex;
justify-content: space-between;
height: 64px;
--boxSize: 8px;
--gutter: 4px;
}
.gradio-webrtc-boxContainer {
display: flex;
justify-content: space-between;
height: 64px;
--boxSize: 8px;
--gutter: 4px;
}
.gradio-webrtc-box {
height: 100%;
width: var(--boxSize);
background: var(--color-accent);
border-radius: 8px;
transition: transform 0.05s ease;
}
.gradio-webrtc-box {
height: 100%;
width: var(--boxSize);
background: var(--color-accent);
border-radius: 8px;
transition: transform 0.05s ease;
}
.gradio-webrtc-icon-container {
position: relative;
width: 128px;
height: 128px;
display: flex;
justify-content: center;
align-items: center;
}
.gradio-webrtc-icon-container {
position: relative;
width: 128px;
height: 128px;
display: flex;
justify-content: center;
align-items: center;
}
.gradio-webrtc-icon {
position: relative;
width: 48px;
height: 48px;
border-radius: 50%;
transition: transform 0.1s ease;
display: flex;
justify-content: center;
align-items: center;
z-index: 2;
}
.gradio-webrtc-icon {
position: relative;
width: 48px;
height: 48px;
border-radius: 50%;
transition: transform 0.1s ease;
display: flex;
justify-content: center;
align-items: center;
z-index: 2;
}
.icon-image {
width: 32px;
height: 32px;
object-fit: contain;
filter: brightness(0) invert(1);
}
.icon-image {
width: 32px;
height: 32px;
object-fit: contain;
filter: brightness(0) invert(1);
}
.pulse-ring {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
width: 48px;
height: 48px;
border-radius: 50%;
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
opacity: 0.5;
}
.pulse-ring {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
width: 48px;
height: 48px;
border-radius: 50%;
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
opacity: 0.5;
}
@keyframes pulse {
@keyframes pulse {
0% {
transform: translate(-50%, -50%) scale(1);
opacity: 0.5;
@@ -161,4 +173,39 @@
opacity: 0;
}
}
</style>
.dots {
display: flex;
gap: 8px;
align-items: center;
height: 64px;
}
.dot {
width: 12px;
height: 12px;
border-radius: 50%;
opacity: 0.5;
animation: pulse 1.5s infinite;
}
.dot:nth-child(2) {
animation-delay: 0.2s;
}
.dot:nth-child(3) {
animation-delay: 0.4s;
}
@keyframes pulse {
0%,
100% {
opacity: 0.4;
transform: scale(1);
}
50% {
opacity: 1;
transform: scale(1.1);
}
}
</style>

528
frontend/shared/InteractiveAudio.svelte
View File

@@ -1,26 +1,24 @@
<script lang="ts">
import {
BlockLabel,
} from "@gradio/atoms";
import type { I18nFormatter } from "@gradio/utils";
import { createEventDispatcher } from "svelte";
import { BlockLabel } from "@gradio/atoms";
import type { I18nFormatter } from "@gradio/utils";
import { createEventDispatcher } from "svelte";
import { onMount } from "svelte";
import { fade } from "svelte/transition";
import { StreamingBar } from "@gradio/statustracker";
import {
Circle,
Square,
Spinner,
Circle,
Square,
Spinner,
Music,
DropdownArrow,
Microphone
} from "@gradio/icons";
Microphone,
} from "@gradio/icons";
import { start, stop } from "./webrtc_utils";
import { get_devices, set_available_devices } from "./stream_utils";
import AudioWave from "./AudioWave.svelte";
import WebcamPermissions from "./WebcamPermissions.svelte";
import PulsingIcon from "./PulsingIcon.svelte";
export let mode: "send-receive" | "send";
export let value: string | null = null;
export let label: string | undefined = undefined;
@@ -31,10 +29,12 @@
export let track_constraints: MediaTrackConstraints = {};
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let on_change_cb: (mg: "tick" | "change") => void;
export let reject_cb: (msg: object) => void;
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let button_labels: {start: string, stop: string, waiting: string};
export let button_labels: { start: string; stop: string; waiting: string };
let pending = false;
let stopword_recognized = false;
@@ -42,20 +42,20 @@
onMount(() => {
if (value === "__webrtc_value__") {
notification_sound = new Audio("https://huggingface.co/datasets/freddyaboulton/bucket/resolve/main/pop-sounds.mp3");
notification_sound = new Audio(
"https://huggingface.co/datasets/freddyaboulton/bucket/resolve/main/pop-sounds.mp3",
);
}
});
let _on_change_cb = (msg: "change" | "tick" | "stopword") => {
console.log("msg", msg);
if (msg === "stopword") {
console.log("stopword recognized");
stopword_recognized = true;
setTimeout(() => {
stopword_recognized = false;
}, 3000);
} else {
console.log("calling on_change_cb with msg", msg);
console.debug("calling on_change_cb with msg", msg);
on_change_cb(msg);
}
};
@@ -63,7 +63,7 @@
let options_open = false;
let _time_limit: number | null = null;
export let server: {
offer: (body: any) => Promise<any>;
};
@@ -78,26 +78,29 @@
let mic_accessed = false;
const audio_source_callback = () => {
console.log("stream in callback", stream);
if(mode==="send") return stream;
else return audio_player.srcObject as MediaStream
}
if (mode === "send") return stream;
else return audio_player.srcObject as MediaStream;
};
const dispatch = createEventDispatcher<{
tick: undefined;
state_change: undefined;
error: string
error: string;
play: undefined;
stop: undefined;
}>();
}>();
async function access_mic(): Promise<void> {
try {
const constraints = selected_device ? { deviceId: { exact: selected_device.deviceId }, ...track_constraints } : track_constraints;
const stream_ = await navigator.mediaDevices.getUserMedia({ audio: constraints });
const constraints = selected_device
? {
deviceId: { exact: selected_device.deviceId },
...track_constraints,
}
: track_constraints;
const stream_ = await navigator.mediaDevices.getUserMedia({
audio: constraints,
});
stream = stream_;
} catch (err) {
if (!navigator.mediaDevices) {
@@ -110,123 +113,155 @@
}
throw err;
}
available_audio_devices = set_available_devices(await get_devices(), "audioinput");
available_audio_devices = set_available_devices(
await get_devices(),
"audioinput",
);
mic_accessed = true;
const used_devices = stream
.getTracks()
.map((track) => track.getSettings()?.deviceId)[0];
.getTracks()
.map((track) => track.getSettings()?.deviceId)[0];
selected_device = used_devices
? available_audio_devices.find((device) => device.deviceId === used_devices) ||
available_audio_devices[0]
? available_audio_devices.find(
(device) => device.deviceId === used_devices,
) || available_audio_devices[0]
: available_audio_devices[0];
}
async function start_stream(): Promise<void> {
if( stream_state === "open"){
if (stream_state === "open") {
stop(pc);
stream_state = "closed";
_time_limit = null;
await access_mic();
return;
}
_webrtc_id = Math.random().toString(36).substring(2);
value = _webrtc_id;
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange",
async (event) => {
switch(pc.connectionState) {
case "connected":
console.info("connected");
stream_state = "open";
_time_limit = time_limit;
break;
case "disconnected":
console.info("closed");
stream_state = "closed";
_time_limit = null;
stop(pc);
break;
default:
break;
}
}
)
stream_state = "waiting"
stream = null
try {
await access_mic();
} catch (err) {
if (!navigator.mediaDevices) {
dispatch("error", i18n("audio.no_device_support"));
return;
}
if (err instanceof DOMException && err.name == "NotAllowedError") {
dispatch("error", i18n("audio.allow_recording_access"));
return;
}
throw err;
_webrtc_id = Math.random().toString(36).substring(2);
value = _webrtc_id;
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange", async (event) => {
switch (pc.connectionState) {
case "connected":
console.info("connected");
stream_state = "open";
_time_limit = time_limit;
break;
case "disconnected":
console.info("closed");
stream_state = "closed";
_time_limit = null;
stop(pc);
break;
default:
break;
}
if (stream == null) return;
});
stream_state = "waiting";
stream = null;
start(stream, pc, mode === "send" ? null: audio_player, server.offer, _webrtc_id, "audio", _on_change_cb, rtp_params).then((connection) => {
pc = connection;
}).catch(() => {
console.info("catching")
dispatch("error", "Too many concurrent users. Come back later!");
});
try {
await access_mic();
} catch (err) {
if (!navigator.mediaDevices) {
dispatch("error", i18n("audio.no_device_support"));
return;
}
if (err instanceof DOMException && err.name == "NotAllowedError") {
dispatch("error", i18n("audio.allow_recording_access"));
return;
}
throw err;
}
if (stream == null) return;
function handle_click_outside(event: MouseEvent): void {
event.preventDefault();
event.stopPropagation();
options_open = false;
}
const additional_message_cb = (msg: object) => {
// @ts-ignore
if (msg.type === "log" && msg.data === "pause_detected") {
pending = true;
// @ts-ignore
} else if (msg.type === "log" && msg.data === "response_starting") {
pending = false;
}
};
function click_outside(node: Node, cb: any): any {
const handle_click = (event: MouseEvent): void => {
if (
node &&
!node.contains(event.target as Node) &&
!event.defaultPrevented
) {
cb(event);
}
};
const timeoutId = setTimeout(() => {
// @ts-ignore
_on_change_cb({ type: "connection_timeout" });
}, 5000);
document.addEventListener("click", handle_click, true);
return {
destroy() {
document.removeEventListener("click", handle_click, true);
}
};
}
const handle_device_change = async (event: InputEvent): Promise<void> => {
const target = event.target as HTMLInputElement;
const device_id = target.value;
stream = await navigator.mediaDevices.getUserMedia({ audio: {deviceId: { exact: device_id }, ...track_constraints }});
selected_device =
available_audio_devices.find(
(device) => device.deviceId === device_id
) || null;
options_open = false;
};
$: if(stopword_recognized){
notification_sound.play();
start(
stream,
pc,
mode === "send" ? null : audio_player,
server.offer,
_webrtc_id,
"audio",
_on_change_cb,
rtp_params,
additional_message_cb,
reject_cb,
)
.then((connection) => {
clearTimeout(timeoutId);
pc = connection;
})
.catch(() => {
console.info("catching");
stream_state = "closed";
});
}
function handle_click_outside(event: MouseEvent): void {
event.preventDefault();
event.stopPropagation();
options_open = false;
}
function click_outside(node: Node, cb: any): any {
const handle_click = (event: MouseEvent): void => {
if (
node &&
!node.contains(event.target as Node) &&
!event.defaultPrevented
) {
cb(event);
}
};
document.addEventListener("click", handle_click, true);
return {
destroy() {
document.removeEventListener("click", handle_click, true);
},
};
}
const handle_device_change = async (event: InputEvent): Promise<void> => {
const target = event.target as HTMLInputElement;
const device_id = target.value;
stream = await navigator.mediaDevices.getUserMedia({
audio: { deviceId: { exact: device_id }, ...track_constraints },
});
selected_device =
available_audio_devices.find(
(device) => device.deviceId === device_id,
) || null;
options_open = false;
};
$: if (stopword_recognized) {
notification_sound.play();
}
</script>
<BlockLabel
{show_label}
Icon={Music}
float={false}
label={label || i18n("audio.audio")}
{show_label}
Icon={Music}
float={false}
label={label || i18n("audio.audio")}
/>
<div class="audio-container">
<audio
@@ -243,27 +278,40 @@
title="grant webcam access"
style="height: 100%"
>
<WebcamPermissions icon={Microphone} on:click={async () => access_mic()} />
<WebcamPermissions
icon={Microphone}
on:click={async () => access_mic()}
/>
</div>
{:else}
<AudioWave {audio_source_callback} {stream_state} {icon} {icon_button_color} {pulse_color}/>
<AudioWave
{audio_source_callback}
{stream_state}
{icon}
{icon_button_color}
{pulse_color}
{pending}
/>
<StreamingBar time_limit={_time_limit} />
<div class="button-wrap" class:pulse={stopword_recognized}>
<button
on:click={start_stream}
aria-label={"start stream"}
>
<button on:click={start_stream} aria-label={"start stream"}>
{#if stream_state === "waiting"}
<div class="icon-with-text">
<div class="icon color-primary" title="spinner">
<Spinner />
</div>
{button_labels.waiting || i18n("audio.waiting")}
{button_labels.waiting || "Connecting..."}
</div>
{:else if stream_state === "open"}
<div class="icon-with-text">
<div class="icon color-primary" title="stop recording">
<Square />
<PulsingIcon
audio_source_callback={() => stream}
stream_state={"open"}
icon={Circle}
{icon_button_color}
{pulse_color}
/>
</div>
{button_labels.stop || i18n("audio.stop")}
</div>
@@ -286,46 +334,45 @@
</button>
{/if}
{#if options_open && selected_device}
<select
class="select-wrap"
aria-label="select source"
use:click_outside={handle_click_outside}
on:change={handle_device_change}
>
<button
class="inset-icon"
on:click|stopPropagation={() => (options_open = false)}
<select
class="select-wrap"
aria-label="select source"
use:click_outside={handle_click_outside}
on:change={handle_device_change}
>
<DropdownArrow />
</button>
{#if available_audio_devices.length === 0}
<option value="">{i18n("common.no_devices")}</option>
{:else}
{#each available_audio_devices as device}
<option
value={device.deviceId}
selected={selected_device.deviceId === device.deviceId}
>
{device.label}
</option>
{/each}
{/if}
</select>
{/if}
<button
class="inset-icon"
on:click|stopPropagation={() => (options_open = false)}
>
<DropdownArrow />
</button>
{#if available_audio_devices.length === 0}
<option value="">{i18n("common.no_devices")}</option>
{:else}
{#each available_audio_devices as device}
<option
value={device.deviceId}
selected={selected_device.deviceId ===
device.deviceId}
>
{device.label}
</option>
{/each}
{/if}
</select>
{/if}
</div>
{/if}
</div>
<style>
.audio-container {
display: flex;
height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
display: flex;
height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
:global(::part(wrapper)) {
margin-bottom: var(--size-2);
@@ -340,33 +387,32 @@
display: none;
}
.button-wrap {
.button-wrap {
margin-top: var(--size-2);
margin-bottom: var(--size-2);
background-color: var(--block-background-fill);
border: 1px solid var(--border-color-primary);
border-radius: var(--radius-xl);
padding: var(--size-1-5);
display: flex;
bottom: var(--size-2);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
line-height: var(--size-3);
color: var(--button-secondary-text-color);
}
background-color: var(--block-background-fill);
border: 1px solid var(--border-color-primary);
border-radius: var(--radius-xl);
padding: var(--size-1-5);
display: flex;
bottom: var(--size-2);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
line-height: var(--size-3);
color: var(--button-secondary-text-color);
}
@keyframes pulse {
0% {
transform: scale(1);
box-shadow: 0 0 0 0 rgba(var(--primary-500-rgb), 0.7);
}
70% {
transform: scale(1.25);
box-shadow: 0 0 0 10px rgba(var(--primary-500-rgb), 0);
}
100% {
transform: scale(1);
box-shadow: 0 0 0 0 rgba(var(--primary-500-rgb), 0);
@@ -377,78 +423,78 @@
animation: pulse 1s infinite;
}
.icon-with-text {
min-width: var(--size-16);
align-items: center;
margin: 0 var(--spacing-xl);
display: flex;
justify-content: space-evenly;
.icon-with-text {
min-width: var(--size-16);
align-items: center;
margin: 0 var(--spacing-xl);
display: flex;
justify-content: space-evenly;
gap: var(--size-2);
}
}
@media (--screen-md) {
button {
bottom: var(--size-4);
}
}
@media (--screen-md) {
button {
bottom: var(--size-4);
}
}
@media (--screen-xl) {
button {
bottom: var(--size-8);
}
}
@media (--screen-xl) {
button {
bottom: var(--size-8);
}
}
.icon {
width: 18px;
height: 18px;
display: flex;
justify-content: space-between;
align-items: center;
}
.icon {
width: 18px;
height: 18px;
display: flex;
justify-content: space-between;
align-items: center;
}
.color-primary {
fill: var(--primary-600);
stroke: var(--primary-600);
color: var(--primary-600);
}
.color-primary {
fill: var(--primary-600);
stroke: var(--primary-600);
color: var(--primary-600);
}
.select-wrap {
-webkit-appearance: none;
-moz-appearance: none;
appearance: none;
color: var(--button-secondary-text-color);
background-color: transparent;
width: 95%;
font-size: var(--text-md);
position: absolute;
bottom: var(--size-2);
background-color: var(--block-background-fill);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
z-index: var(--layer-top);
border: 1px solid var(--border-color-primary);
text-align: left;
line-height: var(--size-4);
white-space: nowrap;
text-overflow: ellipsis;
left: 50%;
transform: translate(-50%, 0);
max-width: var(--size-52);
}
-webkit-appearance: none;
-moz-appearance: none;
appearance: none;
color: var(--button-secondary-text-color);
background-color: transparent;
width: 95%;
font-size: var(--text-md);
position: absolute;
bottom: var(--size-2);
background-color: var(--block-background-fill);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
z-index: var(--layer-top);
border: 1px solid var(--border-color-primary);
text-align: left;
line-height: var(--size-4);
white-space: nowrap;
text-overflow: ellipsis;
left: 50%;
transform: translate(-50%, 0);
max-width: var(--size-52);
}
.select-wrap > option {
padding: 0.25rem 0.5rem;
border-bottom: 1px solid var(--border-color-accent);
padding-right: var(--size-8);
text-overflow: ellipsis;
overflow: hidden;
}
.select-wrap > option {
padding: 0.25rem 0.5rem;
border-bottom: 1px solid var(--border-color-accent);
padding-right: var(--size-8);
text-overflow: ellipsis;
overflow: hidden;
}
.select-wrap > option:hover {
background-color: var(--color-accent);
}
.select-wrap > option:hover {
background-color: var(--color-accent);
}
.select-wrap > option:last-child {
border: none;
}
</style>
.select-wrap > option:last-child {
border: none;
}
</style>

13
frontend/shared/InteractiveVideo.svelte
View File

@@ -3,7 +3,7 @@
import type { ComponentType } from "svelte";
import type { FileData, Client } from "@gradio/client";
import { BlockLabel } from "@gradio/atoms";
import Webcam from "./Webcam.svelte";
import Webcam from "./Webcam.svelte";
import { Video } from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
@@ -17,7 +17,7 @@
export let handle_reset_value: () => void = () => {};
export let stream_handler: Client["stream"];
export let time_limit: number | null = null;
export let button_labels: {start: string, stop: string, waiting: string};
export let button_labels: { start: string; stop: string; waiting: string };
export let server: {
offer: (body: any) => Promise<any>;
};
@@ -25,10 +25,11 @@
export let track_constraints: MediaTrackConstraints = {};
export let mode: "send" | "send-receive";
export let on_change_cb: (msg: "change" | "tick") => void;
export let reject_cb: (msg: object) => void;
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
const dispatch = createEventDispatcher<{
change: FileData | null;
@@ -46,9 +47,6 @@
let dragging = false;
$: dispatch("drag", dragging);
$: console.log("value", value)
</script>
<BlockLabel {show_label} Icon={Video} label={label || "Video"} />
@@ -73,6 +71,7 @@
stream_every={0.5}
{server}
bind:webrtc_id={value}
{reject_cb}
/>
<!-- <SelectSource {sources} bind:active_source /> -->

198
frontend/shared/PulsingIcon.svelte
View File

@@ -1,95 +1,95 @@
<script lang="ts">
import { onDestroy } from 'svelte';
import type {ComponentType} from 'svelte';
export let stream_state: "open" | "closed" | "waiting" = "closed";
export let audio_source_callback: () => MediaStream;
export let icon: string | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
import { onDestroy } from "svelte";
import type { ComponentType } from "svelte";
let audioContext: AudioContext;
let analyser: AnalyserNode;
let dataArray: Uint8Array;
let animationId: number;
let pulseScale = 1;
let pulseIntensity = 0;
$: if(stream_state === "open") setupAudioContext();
onDestroy(() => {
if (animationId) {
cancelAnimationFrame(animationId);
}
if (audioContext) {
audioContext.close();
}
});
function setupAudioContext() {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(audio_source_callback());
source.connect(analyser);
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.8;
dataArray = new Uint8Array(analyser.frequencyBinCount);
updateVisualization();
}
function updateVisualization() {
analyser.getByteFrequencyData(dataArray);
export let stream_state: "open" | "closed" | "waiting" = "closed";
export let audio_source_callback: () => MediaStream;
export let icon: string | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
// Calculate average amplitude for pulse effect
const average = Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
const normalizedAverage = average / 255;
pulseScale = 1 + (normalizedAverage * 0.15);
pulseIntensity = normalizedAverage;
animationId = requestAnimationFrame(updateVisualization);
let audioContext: AudioContext;
let analyser: AnalyserNode;
let dataArray: Uint8Array;
let animationId: number;
let pulseScale = 1;
let pulseIntensity = 0;
$: if (stream_state === "open") setupAudioContext();
onDestroy(() => {
if (animationId) {
cancelAnimationFrame(animationId);
}
$: maxPulseScale = 1 + (pulseIntensity * 10); // Scale from 1x to 3x based on intensity
</script>
if (audioContext) {
audioContext.close();
}
});
function setupAudioContext() {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(
audio_source_callback(),
);
source.connect(analyser);
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.8;
dataArray = new Uint8Array(analyser.frequencyBinCount);
updateVisualization();
}
function updateVisualization() {
analyser.getByteFrequencyData(dataArray);
// Calculate average amplitude for pulse effect
const average =
Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
const normalizedAverage = average / 255;
pulseScale = 1 + normalizedAverage * 0.15;
pulseIntensity = normalizedAverage;
animationId = requestAnimationFrame(updateVisualization);
}
$: maxPulseScale = 1 + pulseIntensity * 10; // Scale from 1x to 3x based on intensity
</script>
<div class="gradio-webrtc-icon-wrapper">
<div class="gradio-webrtc-icon-wrapper">
<div class="gradio-webrtc-pulsing-icon-container">
{#if pulseIntensity > 0}
{#each Array(3) as _, i}
<div
class="pulse-ring"
style:background={pulse_color}
style:animation-delay={`${i * 0.4}s`}
style:--max-scale={maxPulseScale}
style:opacity={0.5 * pulseIntensity}
/>
{/each}
{/if}
<div
class="gradio-webrtc-pulsing-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
{#if typeof icon === "string"}
<img
src={icon}
alt="Audio visualization icon"
class="icon-image"
/>
{:else}
<svelte:component this={icon} />
<div class="gradio-webrtc-pulsing-icon-container">
{#if pulseIntensity > 0}
{#each Array(3) as _, i}
<div
class="pulse-ring"
style:background={pulse_color}
style:animation-delay={`${i * 0.4}s`}
style:--max-scale={maxPulseScale}
style:opacity={0.5 * pulseIntensity}
/>
{/each}
{/if}
<div
class="gradio-webrtc-pulsing-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
{#if typeof icon === "string"}
<img src={icon} alt="Audio visualization icon" class="icon-image" />
{:else if icon === undefined}
<div></div>
{:else}
<div>
<svelte:component this={icon} />
</div>
{/if}
</div>
</div>
</div>
</div>
<style>
.gradio-webrtc-icon-wrapper {
position: relative;
display: flex;
@@ -97,7 +97,7 @@
justify-content: center;
align-items: center;
}
}
.gradio-webrtc-pulsing-icon-container {
position: relative;
width: 100%;
@@ -106,7 +106,7 @@
justify-content: center;
align-items: center;
}
}
.gradio-webrtc-pulsing-icon {
position: relative;
width: 100%;
@@ -118,14 +118,14 @@
align-items: center;
z-index: 2;
}
}
.icon-image {
width: 100%;
height: 100%;
object-fit: contain;
filter: brightness(0) invert(1);
}
}
.pulse-ring {
position: absolute;
top: 50%;
@@ -136,16 +136,18 @@
border-radius: 50%;
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
opacity: 0.5;
min-width: 18px;
min-height: 18px;
}
}
@keyframes pulse {
@keyframes pulse {
0% {
transform: translate(-50%, -50%) scale(1);
opacity: 0.5;
}
100% {
transform: translate(-50%, -50%) scale(var(--max-scale, 3));
opacity: 0;
0% {
transform: translate(-50%, -50%) scale(1);
opacity: 0.5;
}
}
100% {
transform: translate(-50%, -50%) scale(var(--max-scale, 3));
opacity: 0;
}
}
</style>

119
frontend/shared/StaticAudio.svelte
View File

@@ -1,17 +1,14 @@
<script lang="ts">
import { Empty } from "@gradio/atoms";
import {
BlockLabel,
} from "@gradio/atoms";
import { Music } from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
import { createEventDispatcher } from "svelte";
import { Empty } from "@gradio/atoms";
import { BlockLabel } from "@gradio/atoms";
import { Music } from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
import { createEventDispatcher } from "svelte";
import { onMount } from "svelte";
import { start, stop } from "./webrtc_utils";
import AudioWave from "./AudioWave.svelte";
export let value: string | null = null;
export let label: string | undefined = undefined;
export let show_label = true;
@@ -21,7 +18,7 @@
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let server: {
offer: (body: any) => Promise<any>;
};
@@ -31,13 +28,12 @@
let pc: RTCPeerConnection;
let _webrtc_id = Math.random().toString(36).substring(2);
const dispatch = createEventDispatcher<{
tick: undefined;
error: string
error: string;
play: undefined;
stop: undefined;
}>();
}>();
onMount(() => {
window.setInterval(() => {
@@ -45,38 +41,53 @@
dispatch("tick");
}
}, 1000);
}
)
});
async function start_stream(value: string): Promise<string> {
if( value === "start_webrtc_stream") {
if (value === "start_webrtc_stream") {
stream_state = "waiting";
_webrtc_id = Math.random().toString(36).substring(2)
_webrtc_id = Math.random().toString(36).substring(2);
value = _webrtc_id;
console.log("set value to ", value);
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange",
async (event) => {
switch(pc.connectionState) {
case "connected":
console.info("connected");
stream_state = "open";
break;
case "disconnected":
console.info("closed");
stop(pc);
break;
default:
break;
}
pc.addEventListener("connectionstatechange", async (event) => {
switch (pc.connectionState) {
case "connected":
console.info("connected");
stream_state = "open";
break;
case "disconnected":
console.info("closed");
stop(pc);
break;
default:
break;
}
)
});
let stream = null;
start(stream, pc, audio_player, server.offer, _webrtc_id, "audio", on_change_cb).then((connection) => {
const timeoutId = setTimeout(() => {
on_change_cb({ type: "connection_timeout" });
}, 5000);
start(
stream,
pc,
audio_player,
server.offer,
_webrtc_id,
"audio",
on_change_cb,
)
.then((connection) => {
clearTimeout(timeoutId);
pc = connection;
}).catch(() => {
console.info("catching")
dispatch("error", "Too many concurrent users. Come back later!");
})
.catch(() => {
clearTimeout(timeoutId);
console.info("catching");
dispatch(
"error",
"Too many concurrent users. Come back later!",
);
});
}
return value;
@@ -85,16 +96,13 @@
$: start_stream(value).then((val) => {
value = val;
});
</script>
<BlockLabel
{show_label}
Icon={Music}
float={false}
label={label || i18n("audio.audio")}
{show_label}
Icon={Music}
float={false}
label={label || i18n("audio.audio")}
/>
<audio
class="standard-player"
@@ -106,8 +114,14 @@
/>
{#if value !== "__webrtc_value__"}
<div class="audio-container">
<AudioWave audio_source_callback={() => audio_player.srcObject} {stream_state} {icon} {icon_button_color} {pulse_color}/>
</div>
<AudioWave
audio_source_callback={() => audio_player.srcObject}
{stream_state}
{icon}
{icon_button_color}
{pulse_color}
/>
</div>
{/if}
{#if value === "__webrtc_value__"}
<Empty size="small">
@@ -115,15 +129,14 @@
</Empty>
{/if}
<style>
.audio-container {
display: flex;
height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
display: flex;
height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
.standard-player {
width: 100%;
@@ -132,4 +145,4 @@
.hidden {
display: none;
}
</style>
</style>

117
frontend/shared/StaticVideo.svelte
View File

@@ -1,14 +1,10 @@
<script lang="ts">
import { createEventDispatcher, onMount} from "svelte";
import {
BlockLabel,
Empty
} from "@gradio/atoms";
import { createEventDispatcher, onMount } from "svelte";
import { BlockLabel, Empty } from "@gradio/atoms";
import { Video } from "@gradio/icons";
import { start, stop } from "./webrtc_utils";
export let value: string | null = null;
export let label: string | undefined = undefined;
export let show_label = true;
@@ -18,17 +14,17 @@
offer: (body: any) => Promise<any>;
};
let video_element: HTMLVideoElement;
let video_element: HTMLVideoElement;
let _webrtc_id = Math.random().toString(36).substring(2);
let pc: RTCPeerConnection;
const dispatch = createEventDispatcher<{
error: string;
tick: undefined;
}>();
let stream_state = "closed";
onMount(() => {
@@ -36,39 +32,52 @@
if (stream_state == "open") {
dispatch("tick");
}
}, 1000);
}
)
}, 1000);
});
$: if( value === "start_webrtc_stream") {
$: if (value === "start_webrtc_stream") {
_webrtc_id = Math.random().toString(36).substring(2);
value = _webrtc_id;
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange",
async (event) => {
switch(pc.connectionState) {
case "connected":
console.log("connected");
stream_state = "open";
break;
case "disconnected":
console.log("closed");
stop(pc);
break;
default:
break;
}
pc.addEventListener("connectionstatechange", async (event) => {
switch (pc.connectionState) {
case "connected":
stream_state = "open";
break;
case "disconnected":
stop(pc);
break;
default:
break;
}
)
start(null, pc, video_element, server.offer, _webrtc_id, "video", on_change_cb).then((connection) => {
pc = connection;
}).catch(() => {
console.log("catching")
dispatch("error", "Too many concurrent users. Come back later!");
});
const timeoutId = setTimeout(() => {
// @ts-ignore
on_change_cb({ type: "connection_timeout" });
}, 5000);
start(
null,
pc,
video_element,
server.offer,
_webrtc_id,
"video",
on_change_cb,
)
.then((connection) => {
clearTimeout(timeoutId);
pc = connection;
})
.catch(() => {
clearTimeout(timeoutId);
dispatch(
"error",
"Too many concurrent users. Come back later!",
);
});
}
</script>
<BlockLabel {show_label} Icon={Video} label={label || "Video"} />
@@ -78,28 +87,26 @@
{/if}
<div class="wrap">
<video
class:hidden={value === "__webrtc_value__"}
bind:this={video_element}
autoplay={true}
on:loadeddata={dispatch.bind(null, "loadeddata")}
on:click={dispatch.bind(null, "click")}
on:play={dispatch.bind(null, "play")}
on:pause={dispatch.bind(null, "pause")}
on:ended={dispatch.bind(null, "ended")}
on:mouseover={dispatch.bind(null, "mouseover")}
on:mouseout={dispatch.bind(null, "mouseout")}
on:focus={dispatch.bind(null, "focus")}
on:blur={dispatch.bind(null, "blur")}
on:load
data-testid={$$props["data-testid"]}
crossorigin="anonymous"
>
<track kind="captions" />
</video>
class:hidden={value === "__webrtc_value__"}
bind:this={video_element}
autoplay={true}
on:loadeddata={dispatch.bind(null, "loadeddata")}
on:click={dispatch.bind(null, "click")}
on:play={dispatch.bind(null, "play")}
on:pause={dispatch.bind(null, "pause")}
on:ended={dispatch.bind(null, "ended")}
on:mouseover={dispatch.bind(null, "mouseover")}
on:mouseout={dispatch.bind(null, "mouseout")}
on:focus={dispatch.bind(null, "focus")}
on:blur={dispatch.bind(null, "blur")}
on:load
data-testid={$$props["data-testid"]}
crossorigin="anonymous"
>
<track kind="captions" />
</video>
</div>
<style>
.hidden {
display: none;

239
frontend/shared/Webcam.svelte
View File

@@ -6,7 +6,7 @@
Square,
DropdownArrow,
Spinner,
Microphone as Mic
Microphone as Mic,
} from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
import { StreamingBar } from "@gradio/statustracker";
@@ -15,29 +15,30 @@
import {
get_devices,
get_video_stream,
set_available_devices
set_available_devices,
} from "./stream_utils";
import { start, stop } from "./webrtc_utils";
import PulsingIcon from "./PulsingIcon.svelte";
import { start, stop } from "./webrtc_utils";
import PulsingIcon from "./PulsingIcon.svelte";
let video_source: HTMLVideoElement;
let available_video_devices: MediaDeviceInfo[] = [];
let selected_device: MediaDeviceInfo | null = null;
let _time_limit: number | null = null;
export let time_limit: number | null = null;
export let time_limit: number | null = null;
let stream_state: "open" | "waiting" | "closed" = "closed";
export let on_change_cb: (msg: "tick" | "change") => void;
export let reject_cb: (msg: object) => void;
export let mode: "send-receive" | "send";
const _webrtc_id = Math.random().toString(36).substring(2);
const _webrtc_id = Math.random().toString(36).substring(2);
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let button_labels: {start: string, stop: string, waiting: string};
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let button_labels: { start: string; stop: string; waiting: string };
export const modify_stream: (state: "open" | "closed" | "waiting") => void = (
state: "open" | "closed" | "waiting"
) => {
export const modify_stream: (
state: "open" | "closed" | "waiting",
) => void = (state: "open" | "closed" | "waiting") => {
if (state === "closed") {
_time_limit = null;
stream_state = "closed";
@@ -50,7 +51,7 @@
let canvas: HTMLCanvasElement;
export let track_constraints: MediaTrackConstraints | null = null;
export let rtc_configuration: Object;
export let rtc_configuration: Object;
export let stream_every = 1;
export let server: {
offer: (body: any) => Promise<any>;
@@ -73,21 +74,29 @@
const target = event.target as HTMLInputElement;
const device_id = target.value;
await get_video_stream(include_audio, video_source, device_id, track_constraints).then(
async (local_stream) => {
stream = local_stream;
selected_device =
available_video_devices.find(
(device) => device.deviceId === device_id
) || null;
options_open = false;
}
);
await get_video_stream(
include_audio,
video_source,
device_id,
track_constraints,
).then(async (local_stream) => {
stream = local_stream;
selected_device =
available_video_devices.find(
(device) => device.deviceId === device_id,
) || null;
options_open = false;
});
};
async function access_webcam(): Promise<void> {
try {
get_video_stream(include_audio, video_source, null, track_constraints)
get_video_stream(
include_audio,
video_source,
null,
track_constraints,
)
.then(async (local_stream) => {
webcam_accessed = true;
available_video_devices = await get_devices();
@@ -102,12 +111,16 @@
.map((track) => track.getSettings()?.deviceId)[0];
selected_device = used_devices
? devices.find((device) => device.deviceId === used_devices) ||
available_video_devices[0]
? devices.find(
(device) => device.deviceId === used_devices,
) || available_video_devices[0]
: available_video_devices[0];
});
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
if (
!navigator.mediaDevices ||
!navigator.mediaDevices.getUserMedia
) {
dispatch("error", i18n("image.no_webcam_support"));
}
} catch (err) {
@@ -123,46 +136,63 @@
let stream: MediaStream;
let webcam_accessed = false;
let pc: RTCPeerConnection;
let pc: RTCPeerConnection;
export let webrtc_id;
async function start_webrtc(): Promise<void> {
if (stream_state === 'closed') {
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange",
async (event) => {
switch(pc.connectionState) {
case "connected":
stream_state = "open";
_time_limit = time_limit;
dispatch("tick");
break;
case "disconnected":
stream_state = "closed";
_time_limit = null;
stop(pc);
await access_webcam();
break;
default:
break;
}
}
)
stream_state = "waiting"
if (stream_state === "closed") {
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange", async (event) => {
switch (pc.connectionState) {
case "connected":
stream_state = "open";
_time_limit = time_limit;
dispatch("tick");
break;
case "disconnected":
stream_state = "closed";
_time_limit = null;
stop(pc);
await access_webcam();
break;
default:
break;
}
});
stream_state = "waiting";
webrtc_id = Math.random().toString(36).substring(2);
start(stream, pc, mode === "send" ? null: video_source, server.offer, webrtc_id, "video", on_change_cb, rtp_params).then((connection) => {
pc = connection;
}).catch(() => {
console.info("catching")
stream_state = "closed";
dispatch("error", "Too many concurrent users. Come back later!");
});
} else {
stop(pc);
stream_state = "closed";
const timeoutId = setTimeout(() => {
// @ts-ignore
on_change_cb({ type: "connection_timeout" });
}, 5000);
start(
stream,
pc,
mode === "send" ? null : video_source,
server.offer,
webrtc_id,
"video",
on_change_cb,
rtp_params,
undefined,
reject_cb,
)
.then((connection) => {
clearTimeout(timeoutId);
pc = connection;
})
.catch(() => {
clearTimeout(timeoutId);
console.info("catching");
stream_state = "closed";
});
} else {
stop(pc);
stream_state = "closed";
_time_limit = null;
await access_webcam();
}
await access_webcam();
}
}
let options_open = false;
@@ -183,7 +213,7 @@
return {
destroy() {
document.removeEventListener("click", handle_click, true);
}
},
};
}
@@ -201,11 +231,11 @@
{#if stream_state === "open" && include_audio}
<div class="audio-indicator">
<PulsingIcon
stream_state={stream_state}
audio_source_callback={audio_source_callback}
{stream_state}
{audio_source_callback}
icon={icon || Mic}
icon_button_color={icon_button_color}
pulse_color={pulse_color}
{icon_button_color}
{pulse_color}
/>
</div>
{/if}
@@ -214,8 +244,9 @@
<video
bind:this={video_source}
class:hide={!webcam_accessed}
class:flip={(stream_state != "open") || (stream_state === "open" && include_audio)}
autoplay={true}
class:flip={stream_state != "open" ||
(stream_state === "open" && include_audio)}
autoplay={true}
playsinline={true}
/>
<!-- svelte-ignore a11y-missing-attribute -->
@@ -229,32 +260,29 @@
</div>
{:else}
<div class="button-wrap">
<button
on:click={start_webrtc}
aria-label={"start stream"}
>
{#if stream_state === "waiting"}
<div class="icon-with-text">
<div class="icon color-primary" title="spinner">
<Spinner />
</div>
{button_labels.waiting || i18n("audio.waiting")}
</div>
{:else if stream_state === "open"}
<div class="icon-with-text">
<div class="icon color-primary" title="stop recording">
<Square />
</div>
{button_labels.stop || i18n("audio.stop")}
</div>
{:else}
<div class="icon-with-text">
<div class="icon color-primary" title="start recording">
<Circle />
</div>
{button_labels.start || i18n("audio.record")}
</div>
{/if}
<button on:click={start_webrtc} aria-label={"start stream"}>
{#if stream_state === "waiting"}
<div class="icon-with-text">
<div class="icon color-primary" title="spinner">
<Spinner />
</div>
{button_labels.waiting || i18n("audio.waiting")}
</div>
{:else if stream_state === "open"}
<div class="icon-with-text">
<div class="icon color-primary" title="stop recording">
<Square />
</div>
{button_labels.stop || i18n("audio.stop")}
</div>
{:else}
<div class="icon-with-text">
<div class="icon color-primary" title="start recording">
<Circle />
</div>
{button_labels.start || i18n("audio.record")}
</div>
{/if}
</button>
{#if !recording}
<button
@@ -285,7 +313,8 @@
{#each available_video_devices as device}
<option
value={device.deviceId}
selected={selected_device.deviceId === device.deviceId}
selected={selected_device.deviceId ===
device.deviceId}
>
{device.label}
</option>
@@ -334,19 +363,19 @@
align-items: center;
margin: 0 var(--spacing-xl);
display: flex;
justify-content: space-evenly;
/* Add gap between icon and text */
gap: var(--size-2);
justify-content: space-evenly;
/* Add gap between icon and text */
gap: var(--size-2);
}
.audio-indicator {
position: absolute;
top: var(--size-2);
right: var(--size-2);
.audio-indicator {
position: absolute;
top: var(--size-2);
right: var(--size-2);
z-index: var(--layer-2);
height: var(--size-5);
width: var(--size-5);
}
}
@media (--screen-md) {
button {
@@ -432,4 +461,4 @@
font-size: var(--text-lg);
}
}
</style>
</style>

20
frontend/shared/webrtc_utils.ts
View File

@@ -51,8 +51,10 @@ export async function start(
server_fn,
webrtc_id,
modality: "video" | "audio" = "video",
on_change_cb: (msg: "change" | "tick") => void = () => {},
on_change_cb: (msg: "change" | "tick") => void = () => { },
rtp_params = {},
additional_message_cb: (msg: object) => void = () => { },
reject_cb: (msg: object) => void = () => { },
) {
pc = createPeerConnection(pc, node);
const data_channel = pc.createDataChannel("text");
@@ -70,17 +72,19 @@ export async function start(
} catch (e) {
console.debug("Error parsing JSON");
}
console.log("event_json", event_json);
if (
event.data === "change" ||
event.data === "tick" ||
event.data === "stopword" ||
event_json?.type === "warning" ||
event_json?.type === "error"
event_json?.type === "error" ||
event_json?.type === "send_input" ||
event_json?.type === "fetch_output" ||
event_json?.type === "stopword"
) {
console.debug(`${event.data} event received`);
on_change_cb(event_json ?? event.data);
}
additional_message_cb(event_json ?? event.data);
};
if (stream) {
@@ -97,15 +101,16 @@ export async function start(
pc.addTransceiver(modality, { direction: "recvonly" });
}
await negotiate(pc, server_fn, webrtc_id);
await negotiate(pc, server_fn, webrtc_id, reject_cb);
return pc;
}
function make_offer(server_fn: any, body): Promise<object> {
function make_offer(server_fn: any, body, reject_cb: (msg: object) => void = () => { }): Promise<object> {
return new Promise((resolve, reject) => {
server_fn(body).then((data) => {
console.debug("data", data);
if (data?.status === "failed") {
reject_cb(data);
console.debug("rejecting");
reject("error");
}
@@ -118,6 +123,7 @@ async function negotiate(
pc: RTCPeerConnection,
server_fn: any,
webrtc_id: string,
reject_cb: (msg: object) => void = () => { },
): Promise<void> {
return pc
.createOffer()
@@ -148,7 +154,7 @@ async function negotiate(
sdp: offer.sdp,
type: offer.type,
webrtc_id: webrtc_id,
});
}, reject_cb);
})
.then((response) => {
return response;