deepgeek/gradio-webrtc
2
0
Fork 0
mirror of https://github.com/HumanAIGC-Engineering/gradio-webrtc.git synced 2026-02-05 09:59:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'freddyaboulton:main' into audio-track-constraints

Browse Source
This commit is contained in:
bingochaos
2025-04-02 13:45:52 +08:00
committed by GitHub
parent 28f9914c3e 06885d06c4
commit ea6839cc0f
207 changed files with 41317 additions and 6004 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
frontend/.prettierrc Normal file
View File

@@ -0,0 +1,11 @@
{
"plugins": ["prettier-plugin-svelte"],
"overrides": [
{
"files": "*.svelte",
"options": {
"parser": "svelte"
}
}
]
}

120
frontend/Example.svelte
View File

@@ -1,73 +1,73 @@
<script lang="ts">
import { playable } from "./shared/utils";
import { type FileData } from "@gradio/client";
import { playable } from "./shared/utils";
import { type FileData } from "@gradio/client";
export let type: "gallery" | "table";
export let selected = false;
export let value: { video: FileData; subtitles: FileData | null } | null;
export let loop: boolean;
let video: HTMLVideoElement;
export let type: "gallery" | "table";
export let selected = false;
export let value: { video: FileData; subtitles: FileData | null } | null;
export let loop: boolean;
let video: HTMLVideoElement;
async function init(): Promise<void> {
video.muted = true;
video.playsInline = true;
video.controls = false;
video.setAttribute("muted", "");
async function init(): Promise<void> {
video.muted = true;
video.playsInline = true;
video.controls = false;
video.setAttribute("muted", "");
await video.play();
video.pause();
}
await video.play();
video.pause();
}
</script>
{#if value}
{#if playable()}
<div
class="container"
class:table={type === "table"}
class:gallery={type === "gallery"}
class:selected
>
<video
bind:this={video}
on:loadeddata={init}
on:mouseover={video.play.bind(video)}
on:mouseout={video.pause.bind(video)}
src={value?.video.url}
/>
</div>
{:else}
<div>{value}</div>
{/if}
{#if playable()}
<div
class="container"
class:table={type === "table"}
class:gallery={type === "gallery"}
class:selected
>
<video
bind:this={video}
on:loadeddata={init}
on:mouseover={video.play.bind(video)}
on:mouseout={video.pause.bind(video)}
src={value?.video.url}
/>
</div>
{:else}
<div>{value}</div>
{/if}
{/if}
<style>
.container {
flex: none;
max-width: none;
}
.container :global(video) {
width: var(--size-full);
height: var(--size-full);
object-fit: cover;
}
.container {
flex: none;
max-width: none;
}
.container :global(video) {
width: var(--size-full);
height: var(--size-full);
object-fit: cover;
}
.container:hover,
.container.selected {
border-color: var(--border-color-accent);
}
.container.table {
margin: 0 auto;
border: 2px solid var(--border-color-primary);
border-radius: var(--radius-lg);
overflow: hidden;
width: var(--size-20);
height: var(--size-20);
object-fit: cover;
}
.container:hover,
.container.selected {
border-color: var(--border-color-accent);
}
.container.table {
margin: 0 auto;
border: 2px solid var(--border-color-primary);
border-radius: var(--radius-lg);
overflow: hidden;
width: var(--size-20);
height: var(--size-20);
object-fit: cover;
}
.container.gallery {
height: var(--size-20);
max-height: var(--size-20);
object-fit: cover;
}
.container.gallery {
height: var(--size-20);
max-height: var(--size-20);
object-fit: cover;
}
</style>

321
frontend/Index.svelte
View File

@@ -1,158 +1,191 @@
<svelte:options accessors={true} />
<script lang="ts">
import { Block, UploadText } from "@gradio/atoms";
import Video from "./shared/InteractiveVideo.svelte";
import { StatusTracker } from "@gradio/statustracker";
import type { LoadingStatus } from "@gradio/statustracker";
import StaticVideo from "./shared/StaticVideo.svelte";
import StaticAudio from "./shared/StaticAudio.svelte";
import InteractiveAudio from "./shared/InteractiveAudio.svelte";
import { Block, UploadText } from "@gradio/atoms";
import Video from "./shared/InteractiveVideo.svelte";
import { StatusTracker } from "@gradio/statustracker";
import type { LoadingStatus } from "@gradio/statustracker";
import StaticVideo from "./shared/StaticVideo.svelte";
import StaticAudio from "./shared/StaticAudio.svelte";
import InteractiveAudio from "./shared/InteractiveAudio.svelte";
export let elem_id = "";
export let elem_classes: string[] = [];
export let visible = true;
export let value: string = "__webrtc_value__";
export let button_labels: {start: string, stop: string, waiting: string};
export let elem_id = "";
export let elem_classes: string[] = [];
export let visible = true;
export let value: string = "__webrtc_value__";
export let button_labels: { start: string; stop: string; waiting: string };
export let label: string;
export let root: string;
export let show_label: boolean;
export let loading_status: LoadingStatus;
export let height: number | undefined;
export let width: number | undefined;
export let server: {
offer: (body: any) => Promise<any>;
};
export let label: string;
export let root: string;
export let show_label: boolean;
export let loading_status: LoadingStatus;
export let height: number | undefined;
export let width: number | undefined;
export let server: {
offer: (body: any) => Promise<any>;
};
export let container = false;
export let scale: number | null = null;
export let min_width: number | undefined = undefined;
export let gradio;
export let rtc_configuration: Object;
export let time_limit: number | null = null;
export let modality: "video" | "audio" | "audio-video" = "video";
export let mode: "send-receive" | "receive" | "send" = "send-receive";
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let track_constraints: MediaTrackConstraints = {};
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let container = false;
export let scale: number | null = null;
export let min_width: number | undefined = undefined;
export let gradio;
export let rtc_configuration: Object;
export let time_limit: number | null = null;
export let modality: "video" | "audio" | "audio-video" = "video";
export let mode: "send-receive" | "receive" | "send" = "send-receive";
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let track_constraints: MediaTrackConstraints = {};
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let icon_radius: number = 50;
const on_change_cb = (msg: "change" | "tick" | any) => {
if (msg?.type === "info" || msg?.type === "warning" || msg?.type === "error") {
console.log("dispatching info", msg.message);
gradio.dispatch(msg?.type === "error"? "error": "warning", msg.message);
}
gradio.dispatch(msg === "change" ? "state_change" : "tick");
}
const on_change_cb = (msg: "change" | "tick" | any) => {
if (
msg?.type === "info" ||
msg?.type === "warning" ||
msg?.type === "error"
) {
gradio.dispatch(msg?.type === "error" ? "error" : "warning", msg.message);
} else if (msg?.type === "end_stream") {
gradio.dispatch("warning", msg.data);
} else if (msg?.type === "fetch_output") {
gradio.dispatch("state_change");
} else if (msg?.type === "send_input") {
gradio.dispatch("tick");
} else if (msg?.type === "connection_timeout") {
gradio.dispatch(
"warning",
"Taking a while to connect. Are you on a VPN?",
);
}
if (msg.type === "state_change") {
gradio.dispatch(msg === "change" ? "state_change" : "tick");
}
};
let dragging = false;
const reject_cb = (msg: object) => {
if (
msg.status === "failed" &&
msg.meta?.error === "concurrency_limit_reached"
) {
gradio.dispatch(
"error",
`Too many concurrent connections. Please try again later!`,
);
} else {
gradio.dispatch("error", "Unexpected server error");
}
};
$: console.log("value", value);
let dragging = false;
</script>
<Block
{visible}
variant={"solid"}
border_mode={dragging ? "focus" : "base"}
padding={false}
{elem_id}
{elem_classes}
{height}
{width}
{container}
{scale}
{min_width}
allow_overflow={false}
>
<StatusTracker
autoscroll={gradio.autoscroll}
i18n={gradio.i18n}
{...loading_status}
on:clear_status={() => gradio.dispatch("clear_status", loading_status)}
/>
{visible}
variant={"solid"}
border_mode={dragging ? "focus" : "base"}
padding={false}
{elem_id}
{elem_classes}
{height}
{width}
{container}
{scale}
{min_width}
allow_overflow={false}
>
<StatusTracker
autoscroll={gradio.autoscroll}
i18n={gradio.i18n}
{...loading_status}
on:clear_status={() => gradio.dispatch("clear_status", loading_status)}
/>
{#if mode == "receive" && modality === "video"}
<StaticVideo
bind:value={value}
{on_change_cb}
{label}
{show_label}
{server}
{rtc_configuration}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
/>
{:else if mode == "receive" && modality === "audio"}
<StaticAudio
bind:value={value}
{on_change_cb}
{label}
{show_label}
{server}
{rtc_configuration}
{icon}
{icon_button_color}
{pulse_color}
i18n={gradio.i18n}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
/>
{:else if (mode === "send-receive" || mode == "send") && (modality === "video" || modality == "audio-video")}
<Video
bind:value={value}
{label}
{show_label}
active_source={"webcam"}
include_audio={modality === "audio-video"}
{server}
{rtc_configuration}
{time_limit}
{mode}
{track_constraints}
{rtp_params}
{on_change_cb}
{icon}
{icon_button_color}
{pulse_color}
{button_labels}
on:clear={() => gradio.dispatch("clear")}
on:play={() => gradio.dispatch("play")}
on:pause={() => gradio.dispatch("pause")}
on:upload={() => gradio.dispatch("upload")}
on:stop={() => gradio.dispatch("stop")}
on:end={() => gradio.dispatch("end")}
on:start_recording={() => gradio.dispatch("start_recording")}
on:stop_recording={() => gradio.dispatch("stop_recording")}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
i18n={gradio.i18n}
stream_handler={(...args) => gradio.client.stream(...args)}
>
<UploadText i18n={gradio.i18n} type="video" />
</Video>
{:else if (mode === "send-receive" || mode === "send") && modality === "audio"}
<InteractiveAudio
bind:value={value}
{on_change_cb}
{label}
{show_label}
{server}
{rtc_configuration}
{time_limit}
{track_constraints}
{mode}
{rtp_params}
i18n={gradio.i18n}
{icon}
{icon_button_color}
{pulse_color}
{button_labels}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
on:warning={({ detail }) => gradio.dispatch("warning", detail)}
/>
{/if}
{#if mode == "receive" && modality === "video"}
<StaticVideo
bind:value
{on_change_cb}
{label}
{show_label}
{server}
{rtc_configuration}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
/>
{:else if mode == "receive" && modality === "audio"}
<StaticAudio
bind:value
{on_change_cb}
{label}
{show_label}
{server}
{rtc_configuration}
{icon}
{icon_button_color}
{pulse_color}
{icon_radius}
i18n={gradio.i18n}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
/>
{:else if (mode === "send-receive" || mode == "send") && (modality === "video" || modality == "audio-video")}
<Video
bind:value
{label}
{show_label}
active_source={"webcam"}
include_audio={modality === "audio-video"}
{server}
{rtc_configuration}
{time_limit}
{mode}
{track_constraints}
{rtp_params}
{on_change_cb}
{reject_cb}
{icon}
{icon_button_color}
{pulse_color}
{icon_radius}
{button_labels}
on:clear={() => gradio.dispatch("clear")}
on:play={() => gradio.dispatch("play")}
on:pause={() => gradio.dispatch("pause")}
on:upload={() => gradio.dispatch("upload")}
on:stop={() => gradio.dispatch("stop")}
on:end={() => gradio.dispatch("end")}
on:start_recording={() => gradio.dispatch("start_recording")}
on:stop_recording={() => gradio.dispatch("stop_recording")}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
i18n={gradio.i18n}
stream_handler={(...args) => gradio.client.stream(...args)}
>
<UploadText i18n={gradio.i18n} type="video" />
</Video>
{:else if (mode === "send-receive" || mode === "send") && modality === "audio"}
<InteractiveAudio
bind:value
{on_change_cb}
{label}
{show_label}
{server}
{rtc_configuration}
{time_limit}
{track_constraints}
{mode}
{rtp_params}
i18n={gradio.i18n}
{icon}
{reject_cb}
{icon_button_color}
{icon_radius}
{pulse_color}
{button_labels}
on:tick={() => gradio.dispatch("tick")}
on:error={({ detail }) => gradio.dispatch("error", detail)}
on:warning={({ detail }) => gradio.dispatch("warning", detail)}
/>
{/if}
</Block>

15
frontend/package-lock.json generated
View File

@@ -25,7 +25,8 @@
},
"devDependencies": {
"@gradio/preview": "0.12.0",
"prettier": "3.3.3"
"prettier": "^3.3.3",
"prettier-plugin-svelte": "^3.3.3"
},
"peerDependencies": {
"svelte": "^4.0.0"
@@ -4118,6 +4119,7 @@
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz",
"integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==",
"dev": true,
"license": "MIT",
"bin": {
"prettier": "bin/prettier.cjs"
},
@@ -4128,6 +4130,17 @@
"url": "https://github.com/prettier/prettier?sponsor=1"
}
},
"node_modules/prettier-plugin-svelte": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/prettier-plugin-svelte/-/prettier-plugin-svelte-3.3.3.tgz",
"integrity": "sha512-yViK9zqQ+H2qZD1w/bH7W8i+bVfKrD8GIFjkFe4Thl6kCT9SlAsXVNmt3jCvQOCsnOhcvYgsoVlRV/Eu6x5nNw==",
"dev": true,
"license": "MIT",
"peerDependencies": {
"prettier": "^3.0.0",
"svelte": "^3.2.0 || ^4.0.0-next.0 || ^5.0.0-next.0"
}
},
"node_modules/prismjs": {
"version": "1.29.0",
"resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz",

3
frontend/package.json
View File

@@ -23,7 +23,8 @@
},
"devDependencies": {
"@gradio/preview": "0.12.0",
"prettier": "3.3.3"
"prettier": "^3.3.3",
"prettier-plugin-svelte": "^3.3.3"
},
"exports": {
"./package.json": "./package.json",

245
frontend/shared/AudioWave.svelte
View File

@@ -1,8 +1,8 @@
<script lang="ts">
import { onDestroy } from 'svelte';
import type {ComponentType} from 'svelte';
import { onDestroy } from "svelte";
import type { ComponentType } from "svelte";
import PulsingIcon from './PulsingIcon.svelte';
import PulsingIcon from "./PulsingIcon.svelte";
export let numBars = 16;
export let stream_state: "open" | "closed" | "waiting" = "closed";
@@ -10,18 +10,20 @@
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let pending: boolean = false;
export let icon_radius: number = 50;
let audioContext: AudioContext;
let analyser: AnalyserNode;
let dataArray: Uint8Array;
let animationId: number;
let pulseScale = 1;
export let pulseScale = 1;
$: containerWidth = icon
$: containerWidth = icon
? "128px"
: `calc((var(--boxSize) + var(--gutter)) * ${numBars})`;
$: if(stream_state === "open") setupAudioContext();
$: if (stream_state === "open") setupAudioContext();
onDestroy(() => {
if (animationId) {
@@ -33,12 +35,15 @@
});
function setupAudioContext() {
// @ts-ignore
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(audio_source_callback());
const source = audioContext.createMediaStreamSource(
audio_source_callback(),
);
source.connect(analyser);
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.8;
dataArray = new Uint8Array(analyser.frequencyBinCount);
@@ -48,110 +53,121 @@
function updateVisualization() {
analyser.getByteFrequencyData(dataArray);
// Update bars
const bars = document.querySelectorAll('.gradio-webrtc-waveContainer .gradio-webrtc-box');
for (let i = 0; i < bars.length; i++) {
const barHeight = (dataArray[i] / 255) * 2;
bars[i].style.transform = `scaleY(${Math.max(0.1, barHeight)})`;
}
// Update bars
const bars = document.querySelectorAll(
".gradio-webrtc-waveContainer .gradio-webrtc-box",
);
for (let i = 0; i < bars.length; i++) {
const barHeight = (dataArray[i] / 255) * 2;
// @ts-ignore
bars[i].style.transform = `scaleY(${Math.max(0.1, barHeight)})`;
}
animationId = requestAnimationFrame(updateVisualization);
}
</script>
<div class="gradio-webrtc-waveContainer">
{#if icon}
<div class="gradio-webrtc-icon-container">
<div
class="gradio-webrtc-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
<PulsingIcon
{stream_state}
{pulse_color}
{icon}
{icon_button_color}
{audio_source_callback}/>
{#if icon && !pending}
<div class="gradio-webrtc-icon-container">
<div
class="gradio-webrtc-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
<PulsingIcon
{stream_state}
{pulse_color}
{icon}
{icon_button_color}
{icon_radius}
{audio_source_callback}
/>
</div>
</div>
</div>
{:else}
<div class="gradio-webrtc-boxContainer" style:width={containerWidth}>
{#each Array(numBars) as _}
<div class="gradio-webrtc-box"></div>
{/each}
</div>
{/if}
{:else if pending}
<div class="dots">
<div class="dot" style:background-color={pulse_color} />
<div class="dot" style:background-color={pulse_color} />
<div class="dot" style:background-color={pulse_color} />
</div>
{:else}
<div class="gradio-webrtc-boxContainer" style:width={containerWidth}>
{#each Array(numBars) as _}
<div class="gradio-webrtc-box"></div>
{/each}
</div>
{/if}
</div>
<style>
.gradio-webrtc-waveContainer {
position: relative;
display: flex;
min-height: 100px;
max-height: 128px;
justify-content: center;
align-items: center;
}
.gradio-webrtc-waveContainer {
position: relative;
display: flex;
min-height: 100px;
max-height: 128px;
justify-content: center;
align-items: center;
}
.gradio-webrtc-boxContainer {
display: flex;
justify-content: space-between;
height: 64px;
--boxSize: 8px;
--gutter: 4px;
}
.gradio-webrtc-boxContainer {
display: flex;
justify-content: space-between;
height: 64px;
--boxSize: 8px;
--gutter: 4px;
}
.gradio-webrtc-box {
height: 100%;
width: var(--boxSize);
background: var(--color-accent);
border-radius: 8px;
transition: transform 0.05s ease;
}
.gradio-webrtc-box {
height: 100%;
width: var(--boxSize);
background: var(--color-accent);
border-radius: 8px;
transition: transform 0.05s ease;
}
.gradio-webrtc-icon-container {
position: relative;
width: 128px;
height: 128px;
display: flex;
justify-content: center;
align-items: center;
}
.gradio-webrtc-icon-container {
position: relative;
width: 128px;
height: 128px;
display: flex;
justify-content: center;
align-items: center;
}
.gradio-webrtc-icon {
position: relative;
width: 48px;
height: 48px;
border-radius: 50%;
transition: transform 0.1s ease;
display: flex;
justify-content: center;
align-items: center;
z-index: 2;
}
.gradio-webrtc-icon {
position: relative;
width: 48px;
height: 48px;
border-radius: 50%;
transition: transform 0.1s ease;
display: flex;
justify-content: center;
align-items: center;
z-index: 2;
}
.icon-image {
width: 32px;
height: 32px;
object-fit: contain;
filter: brightness(0) invert(1);
}
.icon-image {
width: 32px;
height: 32px;
object-fit: contain;
filter: brightness(0) invert(1);
}
.pulse-ring {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
width: 48px;
height: 48px;
border-radius: 50%;
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
opacity: 0.5;
}
.pulse-ring {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
width: 48px;
height: 48px;
border-radius: 50%;
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
opacity: 0.5;
}
@keyframes pulse {
@keyframes pulse {
0% {
transform: translate(-50%, -50%) scale(1);
opacity: 0.5;
@@ -161,4 +177,39 @@
opacity: 0;
}
}
</style>
.dots {
display: flex;
gap: 8px;
align-items: center;
height: 64px;
}
.dot {
width: 12px;
height: 12px;
border-radius: 50%;
opacity: 0.5;
animation: pulse 1.5s infinite;
}
.dot:nth-child(2) {
animation-delay: 0.2s;
}
.dot:nth-child(3) {
animation-delay: 0.4s;
}
@keyframes pulse {
0%,
100% {
opacity: 0.4;
transform: scale(1);
}
50% {
opacity: 1;
transform: scale(1.1);
}
}
</style>

938
frontend/shared/InteractiveAudio.svelte
View File

@@ -1,454 +1,584 @@
<script lang="ts">
import {
BlockLabel,
} from "@gradio/atoms";
import type { I18nFormatter } from "@gradio/utils";
import { createEventDispatcher } from "svelte";
import { onMount } from "svelte";
import { fade } from "svelte/transition";
import { StreamingBar } from "@gradio/statustracker";
import {
Circle,
Square,
Spinner,
Music,
DropdownArrow,
Microphone
} from "@gradio/icons";
import { BlockLabel } from "@gradio/atoms";
import type { I18nFormatter } from "@gradio/utils";
import { createEventDispatcher } from "svelte";
import { onMount } from "svelte";
import { fade } from "svelte/transition";
import { StreamingBar } from "@gradio/statustracker";
import {
Circle,
Spinner,
Music,
DropdownArrow,
VolumeMuted,
VolumeHigh,
Microphone,
} from "@gradio/icons";
import MicrophoneMuted from "./MicrophoneMuted.svelte";
import { start, stop } from "./webrtc_utils";
import { get_devices, set_available_devices } from "./stream_utils";
import AudioWave from "./AudioWave.svelte";
import WebcamPermissions from "./WebcamPermissions.svelte";
import { start, stop } from "./webrtc_utils";
import { get_devices, set_available_devices } from "./stream_utils";
import AudioWave from "./AudioWave.svelte";
import WebcamPermissions from "./WebcamPermissions.svelte";
import PulsingIcon from "./PulsingIcon.svelte";
export let mode: "send-receive" | "send";
export let value: string | null = null;
export let label: string | undefined = undefined;
export let show_label = true;
export let rtc_configuration: Object | null = null;
export let i18n: I18nFormatter;
export let time_limit: number | null = null;
export let track_constraints: MediaTrackConstraints = {};
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let on_change_cb: (mg: "tick" | "change") => void;
export let reject_cb: (msg: object) => void;
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let icon_radius: number = 50;
export let button_labels: { start: string; stop: string; waiting: string };
let pending = false;
export let mode: "send-receive" | "send";
export let value: string | null = null;
export let label: string | undefined = undefined;
export let show_label = true;
export let rtc_configuration: Object | null = null;
export let i18n: I18nFormatter;
export let time_limit: number | null = null;
export let track_constraints: MediaTrackConstraints = {};
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let on_change_cb: (mg: "tick" | "change") => void;
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let button_labels: {start: string, stop: string, waiting: string};
let stopword_recognized = false;
let stopword_recognized = false;
let notification_sound;
let notification_sound;
onMount(() => {
if (value === "__webrtc_value__") {
notification_sound = new Audio(
"https://huggingface.co/datasets/freddyaboulton/bucket/resolve/main/pop-sounds.mp3",
);
}
});
onMount(() => {
if (value === "__webrtc_value__") {
notification_sound = new Audio("https://huggingface.co/datasets/freddyaboulton/bucket/resolve/main/pop-sounds.mp3");
}
let _on_change_cb = (msg: "change" | "tick" | "stopword" | any) => {
if (msg === "stopword") {
stopword_recognized = true;
setTimeout(() => {
stopword_recognized = false;
}, 3000);
} else if (msg.type === "end_stream") {
stream_state = "closed";
stop(pc);
on_change_cb(msg);
} else {
console.debug("calling on_change_cb with msg", msg);
on_change_cb(msg);
}
};
let options_open = false;
let _time_limit: number | null = null;
export let server: {
offer: (body: any) => Promise<any>;
};
let stream_state: "open" | "closed" | "waiting" = "closed";
let audio_player: HTMLAudioElement;
let pc: RTCPeerConnection;
let _webrtc_id = null;
let stream: MediaStream;
let available_audio_devices: MediaDeviceInfo[];
let selected_device: MediaDeviceInfo | null = null;
let mic_accessed = false;
let is_muted = false;
let is_mic_muted = false;
const audio_source_callback = () => {
if (mode === "send") return stream;
else return audio_player.srcObject as MediaStream;
};
const dispatch = createEventDispatcher<{
tick: undefined;
state_change: undefined;
error: string;
play: undefined;
stop: undefined;
}>();
async function access_mic(): Promise<void> {
try {
const constraints = selected_device
? {
deviceId: { exact: selected_device.deviceId },
...track_constraints,
}
: track_constraints;
const stream_ = await navigator.mediaDevices.getUserMedia({
audio: constraints,
});
stream = stream_;
} catch (err) {
if (!navigator.mediaDevices) {
dispatch("error", i18n("audio.no_device_support"));
return;
}
if (err instanceof DOMException && err.name == "NotAllowedError") {
dispatch("error", i18n("audio.allow_recording_access"));
return;
}
throw err;
}
available_audio_devices = set_available_devices(
await get_devices(),
"audioinput",
);
mic_accessed = true;
const used_devices = stream
.getTracks()
.map((track) => track.getSettings()?.deviceId)[0];
selected_device = used_devices
? available_audio_devices.find(
(device) => device.deviceId === used_devices,
) || available_audio_devices[0]
: available_audio_devices[0];
}
async function start_stream(): Promise<void> {
if (stream_state === "open") {
stop(pc);
stream_state = "closed";
_time_limit = null;
await access_mic();
return;
}
_webrtc_id = Math.random().toString(36).substring(2);
value = _webrtc_id;
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange", async (event) => {
switch (pc.connectionState) {
case "connected":
console.info("connected");
stream_state = "open";
_time_limit = time_limit;
break;
case "disconnected":
console.info("closed");
stream_state = "closed";
_time_limit = null;
stop(pc);
break;
case "failed":
console.info("failed");
stream_state = "closed";
_time_limit = null;
dispatch("error", "Connection failed!");
stop(pc);
break;
default:
break;
}
});
stream_state = "waiting";
stream = null;
let _on_change_cb = (msg: "change" | "tick" | "stopword") => {
console.log("msg", msg);
if (msg === "stopword") {
console.log("stopword recognized");
stopword_recognized = true;
setTimeout(() => {
stopword_recognized = false;
}, 3000);
} else {
console.log("calling on_change_cb with msg", msg);
on_change_cb(msg);
}
try {
await access_mic();
} catch (err) {
if (!navigator.mediaDevices) {
dispatch("error", i18n("audio.no_device_support"));
return;
}
if (err instanceof DOMException && err.name == "NotAllowedError") {
dispatch("error", i18n("audio.allow_recording_access"));
return;
}
throw err;
}
if (stream == null) return;
const additional_message_cb = (msg: object) => {
// @ts-ignore
if (msg.type === "log" && msg.data === "pause_detected") {
pending = true;
// @ts-ignore
} else if (msg.type === "log" && msg.data === "response_starting") {
pending = false;
}
};
let options_open = false;
const timeoutId = setTimeout(() => {
// @ts-ignore
_on_change_cb({ type: "connection_timeout" });
}, 5000);
let _time_limit: number | null = null;
export let server: {
offer: (body: any) => Promise<any>;
start(
stream,
pc,
mode === "send" ? null : audio_player,
server.offer,
_webrtc_id,
"audio",
_on_change_cb,
rtp_params,
additional_message_cb,
reject_cb,
)
.then((connection) => {
clearTimeout(timeoutId);
pc = connection;
})
.catch(() => {
console.info("catching");
clearTimeout(timeoutId);
stream_state = "closed";
});
}
function handle_click_outside(event: MouseEvent): void {
event.preventDefault();
event.stopPropagation();
options_open = false;
}
function click_outside(node: Node, cb: any): any {
const handle_click = (event: MouseEvent): void => {
if (
node &&
!node.contains(event.target as Node) &&
!event.defaultPrevented
) {
cb(event);
}
};
let stream_state: "open" | "closed" | "waiting" = "closed";
let audio_player: HTMLAudioElement;
let pc: RTCPeerConnection;
let _webrtc_id = null;
let stream: MediaStream;
let available_audio_devices: MediaDeviceInfo[];
let selected_device: MediaDeviceInfo | null = null;
let mic_accessed = false;
document.addEventListener("click", handle_click, true);
const audio_source_callback = () => {
console.log("stream in callback", stream);
if(mode==="send") return stream;
else return audio_player.srcObject as MediaStream
return {
destroy() {
document.removeEventListener("click", handle_click, true);
},
};
}
const handle_device_change = async (event: InputEvent): Promise<void> => {
const target = event.target as HTMLInputElement;
const device_id = target.value;
stream = await navigator.mediaDevices.getUserMedia({
audio: { deviceId: { exact: device_id }, ...track_constraints },
});
selected_device =
available_audio_devices.find((device) => device.deviceId === device_id) ||
null;
options_open = false;
};
function toggleMute(): void {
if (audio_player) {
audio_player.muted = !audio_player.muted;
is_muted = audio_player.muted;
}
}
const dispatch = createEventDispatcher<{
tick: undefined;
state_change: undefined;
error: string
play: undefined;
stop: undefined;
}>();
async function access_mic(): Promise<void> {
try {
const constraints = selected_device ? { deviceId: { exact: selected_device.deviceId }, ...track_constraints } : track_constraints;
const stream_ = await navigator.mediaDevices.getUserMedia({ audio: constraints });
stream = stream_;
} catch (err) {
if (!navigator.mediaDevices) {
dispatch("error", i18n("audio.no_device_support"));
return;
}
if (err instanceof DOMException && err.name == "NotAllowedError") {
dispatch("error", i18n("audio.allow_recording_access"));
return;
}
throw err;
}
available_audio_devices = set_available_devices(await get_devices(), "audioinput");
mic_accessed = true;
const used_devices = stream
.getTracks()
.map((track) => track.getSettings()?.deviceId)[0];
selected_device = used_devices
? available_audio_devices.find((device) => device.deviceId === used_devices) ||
available_audio_devices[0]
: available_audio_devices[0];
}
async function start_stream(): Promise<void> {
if( stream_state === "open"){
stop(pc);
stream_state = "closed";
_time_limit = null;
await access_mic();
return;
}
_webrtc_id = Math.random().toString(36).substring(2);
value = _webrtc_id;
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange",
async (event) => {
switch(pc.connectionState) {
case "connected":
console.info("connected");
stream_state = "open";
_time_limit = time_limit;
break;
case "disconnected":
console.info("closed");
stream_state = "closed";
_time_limit = null;
stop(pc);
break;
default:
break;
}
}
)
stream_state = "waiting"
stream = null
try {
await access_mic();
} catch (err) {
if (!navigator.mediaDevices) {
dispatch("error", i18n("audio.no_device_support"));
return;
}
if (err instanceof DOMException && err.name == "NotAllowedError") {
dispatch("error", i18n("audio.allow_recording_access"));
return;
}
throw err;
}
if (stream == null) return;
start(stream, pc, mode === "send" ? null: audio_player, server.offer, _webrtc_id, "audio", _on_change_cb, rtp_params).then((connection) => {
pc = connection;
}).catch(() => {
console.info("catching")
dispatch("error", "Too many concurrent users. Come back later!");
});
}
function handle_click_outside(event: MouseEvent): void {
event.preventDefault();
event.stopPropagation();
options_open = false;
}
function click_outside(node: Node, cb: any): any {
const handle_click = (event: MouseEvent): void => {
if (
node &&
!node.contains(event.target as Node) &&
!event.defaultPrevented
) {
cb(event);
}
};
document.addEventListener("click", handle_click, true);
return {
destroy() {
document.removeEventListener("click", handle_click, true);
}
};
}
const handle_device_change = async (event: InputEvent): Promise<void> => {
const target = event.target as HTMLInputElement;
const device_id = target.value;
stream = await navigator.mediaDevices.getUserMedia({ audio: {deviceId: { exact: device_id }, ...track_constraints }});
selected_device =
available_audio_devices.find(
(device) => device.deviceId === device_id
) || null;
options_open = false;
};
$: if(stopword_recognized){
notification_sound.play();
function toggleMuteMicrophone(): void {
if (stream && stream.getAudioTracks().length > 0) {
const audioTrack = stream.getAudioTracks()[0];
audioTrack.enabled = !audioTrack.enabled;
is_mic_muted = !audioTrack.enabled;
}
}
$: if (stopword_recognized) {
notification_sound.play();
}
</script>
<BlockLabel
{show_label}
Icon={Music}
float={false}
label={label || i18n("audio.audio")}
{show_label}
Icon={Music}
float={false}
label={label || i18n("audio.audio")}
/>
<div class="audio-container">
<audio
class="standard-player"
class:hidden={value === "__webrtc_value__"}
on:load
bind:this={audio_player}
on:ended={() => dispatch("stop")}
on:play={() => dispatch("play")}
<audio
class="standard-player"
class:hidden={value === "__webrtc_value__"}
on:load
bind:this={audio_player}
on:ended={() => dispatch("stop")}
on:play={() => dispatch("play")}
/>
{#if !mic_accessed}
<div
in:fade={{ delay: 100, duration: 200 }}
title="grant webcam access"
style="height: 100%"
>
<WebcamPermissions
icon={Microphone}
on:click={async () => access_mic()}
/>
</div>
{:else}
<AudioWave
{audio_source_callback}
{stream_state}
{icon}
{icon_button_color}
{pulse_color}
{pending}
{icon_radius}
/>
{#if !mic_accessed}
<div
in:fade={{ delay: 100, duration: 200 }}
title="grant webcam access"
style="height: 100%"
>
<WebcamPermissions icon={Microphone} on:click={async () => access_mic()} />
</div>
{:else}
<AudioWave {audio_source_callback} {stream_state} {icon} {icon_button_color} {pulse_color}/>
<StreamingBar time_limit={_time_limit} />
<div class="button-wrap" class:pulse={stopword_recognized}>
<button
on:click={start_stream}
aria-label={"start stream"}
>
{#if stream_state === "waiting"}
<div class="icon-with-text">
<div class="icon color-primary" title="spinner">
<Spinner />
</div>
{button_labels.waiting || i18n("audio.waiting")}
</div>
{:else if stream_state === "open"}
<div class="icon-with-text">
<div class="icon color-primary" title="stop recording">
<Square />
</div>
{button_labels.stop || i18n("audio.stop")}
</div>
{:else}
<div class="icon-with-text">
<div class="icon color-primary" title="start recording">
<Circle />
</div>
{button_labels.start || i18n("audio.record")}
</div>
{/if}
</button>
{#if stream_state === "closed"}
<button
class="icon"
on:click={() => (options_open = true)}
aria-label="select input source"
>
<DropdownArrow />
</button>
<StreamingBar time_limit={_time_limit} />
<div class="button-wrap" class:pulse={stopword_recognized}>
<button on:click={start_stream} aria-label={"start stream"}>
{#if stream_state === "waiting"}
<div class="icon-with-text">
<div class="icon color-primary" title="spinner">
<Spinner />
</div>
{button_labels.waiting || "Connecting..."}
</div>
{:else if stream_state === "open"}
<div class="icon-with-text">
{#if mode === "send-receive"}
<div
class="icon"
title="stop recording"
style={`fill: ${icon_button_color}; stroke: ${icon_button_color}; color: ${icon_button_color};`}
>
<PulsingIcon
audio_source_callback={() => stream}
stream_state={"open"}
icon={Circle}
{icon_button_color}
{pulse_color}
/>
</div>
{:else}
<div class="icon color-primary" title="start recording">
<Circle />
</div>
{/if}
{#if options_open && selected_device}
<select
class="select-wrap"
aria-label="select source"
use:click_outside={handle_click_outside}
on:change={handle_device_change}
>
<button
class="inset-icon"
on:click|stopPropagation={() => (options_open = false)}
>
<DropdownArrow />
</button>
{#if available_audio_devices.length === 0}
<option value="">{i18n("common.no_devices")}</option>
{:else}
{#each available_audio_devices as device}
<option
value={device.deviceId}
selected={selected_device.deviceId === device.deviceId}
>
{device.label}
</option>
{/each}
{/if}
</select>
{button_labels.stop || i18n("audio.stop")}
</div>
{:else}
<div class="icon-with-text">
<div class="icon color-primary" title="start recording">
<Circle />
</div>
{button_labels.start || i18n("audio.record")}
</div>
{/if}
</div>
{/if}
</button>
{#if stream_state === "closed"}
<button
class="icon"
on:click={() => (options_open = true)}
aria-label="select input source"
>
<DropdownArrow />
</button>
{/if}
{#if stream_state === "open" && mode === "send-receive"}
<button
class="mute-button"
on:click={toggleMute}
aria-label={is_muted ? "unmute audio" : "mute audio"}
>
<div
class="icon"
style={`fill: ${icon_button_color}; stroke: ${icon_button_color}; color: ${icon_button_color};`}
>
{#if is_muted}
<VolumeMuted />
{:else}
<VolumeHigh />
{/if}
</div>
</button>
{/if}
{#if stream_state === "open" && mode.includes("send")}
<button
class="mute-button"
on:click={toggleMuteMicrophone}
aria-label={is_mic_muted ? "unmute mic" : "mute mic"}
>
<div
class="icon"
style={`fill: ${icon_button_color}; stroke: ${icon_button_color}; color: ${icon_button_color};`}
>
{#if is_mic_muted}
<MicrophoneMuted />
{:else}
<Microphone />
{/if}
</div>
</button>
{/if}
{#if options_open && selected_device}
<select
class="select-wrap"
aria-label="select source"
use:click_outside={handle_click_outside}
on:change={handle_device_change}
>
<button
class="inset-icon"
on:click|stopPropagation={() => (options_open = false)}
>
<DropdownArrow />
</button>
{#if available_audio_devices.length === 0}
<option value="">{i18n("common.no_devices")}</option>
{:else}
{#each available_audio_devices as device}
<option
value={device.deviceId}
selected={selected_device.deviceId === device.deviceId}
>
{device.label}
</option>
{/each}
{/if}
</select>
{/if}
</div>
{/if}
</div>
<style>
.audio-container {
display: flex;
height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
.audio-container {
display: flex;
height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
:global(::part(wrapper)) {
margin-bottom: var(--size-2);
}
.standard-player {
width: 100%;
padding: var(--size-2);
}
:global(::part(wrapper)) {
margin-bottom: var(--size-2);
.hidden {
display: none;
}
.button-wrap {
margin-top: var(--size-2);
margin-bottom: var(--size-2);
background-color: var(--block-background-fill);
border: 1px solid var(--border-color-primary);
border-radius: var(--radius-xl);
padding: var(--size-1-5);
display: flex;
bottom: var(--size-2);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
line-height: var(--size-3);
color: var(--button-secondary-text-color);
}
@keyframes pulse {
0% {
transform: scale(1);
box-shadow: 0 0 0 0 rgba(var(--primary-500-rgb), 0.7);
}
.standard-player {
width: 100%;
padding: var(--size-2);
70% {
transform: scale(1.25);
box-shadow: 0 0 0 10px rgba(var(--primary-500-rgb), 0);
}
.hidden {
display: none;
100% {
transform: scale(1);
box-shadow: 0 0 0 0 rgba(var(--primary-500-rgb), 0);
}
}
.pulse {
animation: pulse 1s infinite;
}
.button-wrap {
margin-top: var(--size-2);
margin-bottom: var(--size-2);
background-color: var(--block-background-fill);
border: 1px solid var(--border-color-primary);
border-radius: var(--radius-xl);
padding: var(--size-1-5);
display: flex;
bottom: var(--size-2);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
line-height: var(--size-3);
color: var(--button-secondary-text-color);
}
.icon-with-text {
min-width: var(--size-16);
align-items: center;
margin: 0 var(--spacing-xl);
display: flex;
justify-content: space-evenly;
gap: var(--size-2);
}
@keyframes pulse {
0% {
transform: scale(1);
box-shadow: 0 0 0 0 rgba(var(--primary-500-rgb), 0.7);
}
70% {
transform: scale(1.25);
box-shadow: 0 0 0 10px rgba(var(--primary-500-rgb), 0);
}
100% {
transform: scale(1);
box-shadow: 0 0 0 0 rgba(var(--primary-500-rgb), 0);
}
@media (--screen-md) {
button {
bottom: var(--size-4);
}
}
.pulse {
animation: pulse 1s infinite;
@media (--screen-xl) {
button {
bottom: var(--size-8);
}
}
.icon-with-text {
min-width: var(--size-16);
align-items: center;
margin: 0 var(--spacing-xl);
display: flex;
justify-content: space-evenly;
gap: var(--size-2);
}
.icon {
width: 18px;
height: 18px;
display: flex;
justify-content: space-between;
align-items: center;
}
@media (--screen-md) {
button {
bottom: var(--size-4);
}
}
.color-primary {
fill: var(--primary-600);
stroke: var(--primary-600);
color: var(--primary-600);
}
@media (--screen-xl) {
button {
bottom: var(--size-8);
}
}
.select-wrap {
-webkit-appearance: none;
-moz-appearance: none;
appearance: none;
color: var(--button-secondary-text-color);
background-color: transparent;
width: 95%;
font-size: var(--text-md);
position: absolute;
bottom: var(--size-2);
background-color: var(--block-background-fill);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
z-index: var(--layer-top);
border: 1px solid var(--border-color-primary);
text-align: left;
line-height: var(--size-4);
white-space: nowrap;
text-overflow: ellipsis;
left: 50%;
transform: translate(-50%, 0);
max-width: var(--size-52);
}
.icon {
width: 18px;
height: 18px;
display: flex;
justify-content: space-between;
align-items: center;
}
.select-wrap > option {
padding: 0.25rem 0.5rem;
border-bottom: 1px solid var(--border-color-accent);
padding-right: var(--size-8);
text-overflow: ellipsis;
overflow: hidden;
}
.color-primary {
fill: var(--primary-600);
stroke: var(--primary-600);
color: var(--primary-600);
}
.select-wrap > option:hover {
background-color: var(--color-accent);
}
.select-wrap {
-webkit-appearance: none;
-moz-appearance: none;
appearance: none;
color: var(--button-secondary-text-color);
background-color: transparent;
width: 95%;
font-size: var(--text-md);
position: absolute;
bottom: var(--size-2);
background-color: var(--block-background-fill);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
z-index: var(--layer-top);
border: 1px solid var(--border-color-primary);
text-align: left;
line-height: var(--size-4);
white-space: nowrap;
text-overflow: ellipsis;
left: 50%;
transform: translate(-50%, 0);
max-width: var(--size-52);
}
.select-wrap > option:last-child {
border: none;
}
.select-wrap > option {
padding: 0.25rem 0.5rem;
border-bottom: 1px solid var(--border-color-accent);
padding-right: var(--size-8);
text-overflow: ellipsis;
overflow: hidden;
}
.select-wrap > option:hover {
background-color: var(--color-accent);
}
.select-wrap > option:last-child {
border: none;
}
</style>
.mute-button {
background-color: var(--block-background-fill);
padding-right: var(--size-2);
display: flex;
color: var(--button-secondary-text-color);
}
</style>

151
frontend/shared/InteractiveVideo.svelte
View File

@@ -1,89 +1,90 @@
<script lang="ts">
import { createEventDispatcher } from "svelte";
import type { ComponentType } from "svelte";
import type { FileData, Client } from "@gradio/client";
import { BlockLabel } from "@gradio/atoms";
import Webcam from "./Webcam.svelte";
import { Video } from "@gradio/icons";
import { createEventDispatcher } from "svelte";
import type { ComponentType } from "svelte";
import type { FileData, Client } from "@gradio/client";
import { BlockLabel } from "@gradio/atoms";
import Webcam from "./Webcam.svelte";
import { Video } from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
import type { I18nFormatter } from "@gradio/utils";
export let value: string = null;
export let label: string | undefined = undefined;
export let show_label = true;
export let include_audio: boolean;
export let i18n: I18nFormatter;
export let active_source: "webcam" | "upload" = "webcam";
export let handle_reset_value: () => void = () => {};
export let stream_handler: Client["stream"];
export let time_limit: number | null = null;
export let button_labels: {start: string, stop: string, waiting: string};
export let server: {
offer: (body: any) => Promise<any>;
};
export let rtc_configuration: Object;
export let track_constraints: MediaTrackConstraints = {};
export let mode: "send" | "send-receive";
export let on_change_cb: (msg: "change" | "tick") => void;
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let value: string = null;
export let label: string | undefined = undefined;
export let show_label = true;
export let include_audio: boolean;
export let i18n: I18nFormatter;
export let active_source: "webcam" | "upload" = "webcam";
export let handle_reset_value: () => void = () => {};
export let stream_handler: Client["stream"];
export let time_limit: number | null = null;
export let button_labels: { start: string; stop: string; waiting: string };
export let server: {
offer: (body: any) => Promise<any>;
};
export let rtc_configuration: Object;
export let track_constraints: MediaTrackConstraints = {};
export let mode: "send" | "send-receive";
export let on_change_cb: (msg: "change" | "tick") => void;
export let reject_cb: (msg: object) => void;
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let icon_radius: number = 50;
const dispatch = createEventDispatcher<{
change: FileData | null;
clear?: never;
play?: never;
pause?: never;
end?: never;
drag: boolean;
error: string;
upload: FileData;
start_recording?: never;
stop_recording?: never;
tick: never;
}>();
let dragging = false;
$: dispatch("drag", dragging);
$: console.log("value", value)
const dispatch = createEventDispatcher<{
change: FileData | null;
clear?: never;
play?: never;
pause?: never;
end?: never;
drag: boolean;
error: string;
upload: FileData;
start_recording?: never;
stop_recording?: never;
tick: never;
}>();
let dragging = false;
$: dispatch("drag", dragging);
</script>
<BlockLabel {show_label} Icon={Video} label={label || "Video"} />
<div data-testid="video" class="video-container">
<Webcam
{rtc_configuration}
{include_audio}
{time_limit}
{track_constraints}
{mode}
{rtp_params}
{on_change_cb}
{icon}
{icon_button_color}
{pulse_color}
{button_labels}
on:error
on:start_recording
on:stop_recording
on:tick
{i18n}
stream_every={0.5}
{server}
bind:webrtc_id={value}
/>
<Webcam
{rtc_configuration}
{include_audio}
{time_limit}
{track_constraints}
{mode}
{rtp_params}
{on_change_cb}
{icon}
{icon_button_color}
{pulse_color}
{icon_radius}
{button_labels}
on:error
on:start_recording
on:stop_recording
on:tick
{i18n}
stream_every={0.5}
{server}
bind:webrtc_id={value}
{reject_cb}
/>
<!-- <SelectSource {sources} bind:active_source /> -->
<!-- <SelectSource {sources} bind:active_source /> -->
</div>
<style>
.video-container {
display: flex;
height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
.video-container {
display: flex;
height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
</style>

20
frontend/shared/MicrophoneMuted.svelte Normal file
View File

@@ -0,0 +1,20 @@
<svg
xmlns="http://www.w3.org/2000/svg"
width="100%"
height="100%"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
class="feather feather-mic"
><path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" /><path
d="M19 10v2a7 7 0 0 1-14 0v-2"
/><line x1="12" y1="19" x2="12" y2="23" /><line
x1="8"
y1="23"
x2="16"
y2="23"
/><line x1="1" y1="1" x2="23" y2="23" /></svg
>
Side by Side

After

Width:  |  Height:  |  Size: 489 B

206
frontend/shared/PulsingIcon.svelte
View File

@@ -1,95 +1,102 @@
<script lang="ts">
import { onDestroy } from 'svelte';
import type {ComponentType} from 'svelte';
export let stream_state: "open" | "closed" | "waiting" = "closed";
export let audio_source_callback: () => MediaStream;
export let icon: string | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
import { onDestroy } from "svelte";
import type { ComponentType } from "svelte";
let audioContext: AudioContext;
let analyser: AnalyserNode;
let dataArray: Uint8Array;
let animationId: number;
let pulseScale = 1;
let pulseIntensity = 0;
$: if(stream_state === "open") setupAudioContext();
onDestroy(() => {
if (animationId) {
cancelAnimationFrame(animationId);
}
if (audioContext) {
audioContext.close();
}
});
function setupAudioContext() {
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(audio_source_callback());
source.connect(analyser);
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.8;
dataArray = new Uint8Array(analyser.frequencyBinCount);
updateVisualization();
}
function updateVisualization() {
analyser.getByteFrequencyData(dataArray);
export let stream_state: "open" | "closed" | "waiting" = "closed";
export let audio_source_callback: () => MediaStream;
export let icon: string | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let icon_radius: number = 50;
// Calculate average amplitude for pulse effect
const average = Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
const normalizedAverage = average / 255;
pulseScale = 1 + (normalizedAverage * 0.15);
pulseIntensity = normalizedAverage;
animationId = requestAnimationFrame(updateVisualization);
let audioContext: AudioContext;
let analyser: AnalyserNode;
let dataArray: Uint8Array;
let animationId: number;
let pulseScale = 1;
let pulseIntensity = 0;
$: if (stream_state === "open") setupAudioContext();
onDestroy(() => {
if (animationId) {
cancelAnimationFrame(animationId);
}
$: maxPulseScale = 1 + (pulseIntensity * 10); // Scale from 1x to 3x based on intensity
</script>
if (audioContext) {
audioContext.close();
}
});
function setupAudioContext() {
// @ts-ignore
audioContext = new (window.AudioContext || window.webkitAudioContext)();
analyser = audioContext.createAnalyser();
const source = audioContext.createMediaStreamSource(
audio_source_callback(),
);
source.connect(analyser);
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.8;
dataArray = new Uint8Array(analyser.frequencyBinCount);
updateVisualization();
}
function updateVisualization() {
analyser.getByteFrequencyData(dataArray);
// Calculate average amplitude for pulse effect
const average =
Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
const normalizedAverage = average / 255;
pulseScale = 1 + normalizedAverage * 0.15;
pulseIntensity = normalizedAverage;
animationId = requestAnimationFrame(updateVisualization);
}
$: maxPulseScale = 1 + pulseIntensity * 10; // Scale from 1x to 3x based on intensity
</script>
<div class="gradio-webrtc-icon-wrapper">
<div class="gradio-webrtc-icon-wrapper">
<div class="gradio-webrtc-pulsing-icon-container">
{#if pulseIntensity > 0}
{#each Array(3) as _, i}
<div
class="pulse-ring"
style:background={pulse_color}
style:animation-delay={`${i * 0.4}s`}
style:--max-scale={maxPulseScale}
style:opacity={0.5 * pulseIntensity}
/>
{/each}
{/if}
<div
class="gradio-webrtc-pulsing-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
{#if typeof icon === "string"}
<img
src={icon}
alt="Audio visualization icon"
class="icon-image"
/>
{:else}
<svelte:component this={icon} />
<div class="gradio-webrtc-pulsing-icon-container">
{#if pulseIntensity > 0}
{#each Array(3) as _, i}
<div
class="pulse-ring"
style:background={pulse_color}
style:animation-delay={`${i * 0.4}s`}
style:--max-scale={maxPulseScale}
style:opacity={0.5 * pulseIntensity}
/>
{/each}
{/if}
<div
class="gradio-webrtc-pulsing-icon"
style:transform={`scale(${pulseScale})`}
style:background={icon_button_color}
>
{#if typeof icon === "string"}
<img
src={icon}
alt="Audio visualization icon"
class="icon-image"
style:border-radius={`${icon_radius}%`}
/>
{:else if icon === undefined}
<div></div>
{:else}
<div>
<svelte:component this={icon} />
</div>
{/if}
</div>
</div>
</div>
</div>
<style>
.gradio-webrtc-icon-wrapper {
position: relative;
display: flex;
@@ -97,7 +104,7 @@
justify-content: center;
align-items: center;
}
}
.gradio-webrtc-pulsing-icon-container {
position: relative;
width: 100%;
@@ -106,7 +113,7 @@
justify-content: center;
align-items: center;
}
}
.gradio-webrtc-pulsing-icon {
position: relative;
width: 100%;
@@ -118,14 +125,13 @@
align-items: center;
z-index: 2;
}
}
.icon-image {
width: 100%;
height: 100%;
object-fit: contain;
object-fit: contain;
}
}
.pulse-ring {
position: absolute;
top: 50%;
@@ -136,16 +142,18 @@
border-radius: 50%;
animation: pulse 2s cubic-bezier(0.4, 0, 0.6, 1) infinite;
opacity: 0.5;
min-width: 18px;
min-height: 18px;
}
}
@keyframes pulse {
@keyframes pulse {
0% {
transform: translate(-50%, -50%) scale(1);
opacity: 0.5;
}
100% {
transform: translate(-50%, -50%) scale(var(--max-scale, 3));
opacity: 0;
0% {
transform: translate(-50%, -50%) scale(1);
opacity: 0.5;
}
}
100% {
transform: translate(-50%, -50%) scale(var(--max-scale, 3));
opacity: 0;
}
}
</style>

244
frontend/shared/StaticAudio.svelte
View File

@@ -1,135 +1,157 @@
<script lang="ts">
import { Empty } from "@gradio/atoms";
import {
BlockLabel,
} from "@gradio/atoms";
import { Music } from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
import { createEventDispatcher } from "svelte";
import { onMount } from "svelte";
import { Empty } from "@gradio/atoms";
import { BlockLabel } from "@gradio/atoms";
import { Music } from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
import { createEventDispatcher } from "svelte";
import { onMount } from "svelte";
import { start, stop } from "./webrtc_utils";
import AudioWave from "./AudioWave.svelte";
import { start, stop } from "./webrtc_utils";
import AudioWave from "./AudioWave.svelte";
export let value: string | null = null;
export let label: string | undefined = undefined;
export let show_label = true;
export let rtc_configuration: Object | null = null;
export let i18n: I18nFormatter;
export let on_change_cb: (msg: "change" | "tick") => void;
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let icon_radius: number = 50;
export let value: string | null = null;
export let label: string | undefined = undefined;
export let show_label = true;
export let rtc_configuration: Object | null = null;
export let i18n: I18nFormatter;
export let on_change_cb: (msg: "change" | "tick") => void;
export let icon: string | undefined = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let server: {
offer: (body: any) => Promise<any>;
};
export let server: {
offer: (body: any) => Promise<any>;
};
let stream_state: "open" | "closed" | "waiting" = "closed";
let audio_player: HTMLAudioElement;
let pc: RTCPeerConnection;
let _webrtc_id = Math.random().toString(36).substring(2);
let stream_state: "open" | "closed" | "waiting" = "closed";
let audio_player: HTMLAudioElement;
let pc: RTCPeerConnection;
let _webrtc_id = Math.random().toString(36).substring(2);
const dispatch = createEventDispatcher<{
tick: undefined;
error: string
play: undefined;
stop: undefined;
}>();
onMount(() => {
window.setInterval(() => {
if (stream_state == "open") {
dispatch("tick");
}
}, 1000);
}
)
async function start_stream(value: string): Promise<string> {
if( value === "start_webrtc_stream") {
stream_state = "waiting";
_webrtc_id = Math.random().toString(36).substring(2)
value = _webrtc_id;
console.log("set value to ", value);
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange",
async (event) => {
switch(pc.connectionState) {
case "connected":
console.info("connected");
stream_state = "open";
break;
case "disconnected":
console.info("closed");
stop(pc);
break;
default:
break;
}
}
)
let stream = null;
start(stream, pc, audio_player, server.offer, _webrtc_id, "audio", on_change_cb).then((connection) => {
pc = connection;
}).catch(() => {
console.info("catching")
dispatch("error", "Too many concurrent users. Come back later!");
});
}
return value;
let _on_change_cb = (msg: "change" | "tick" | "stopword" | any) => {
if (msg.type === "end_stream") {
on_change_cb(msg);
stream_state = "closed";
stop(pc);
} else {
console.debug("calling on_change_cb with msg", msg);
on_change_cb(msg);
}
};
$: start_stream(value).then((val) => {
value = val;
});
const dispatch = createEventDispatcher<{
tick: undefined;
error: string;
play: undefined;
stop: undefined;
}>();
async function start_stream(value: string): Promise<string> {
if (value === "start_webrtc_stream") {
stream_state = "waiting";
_webrtc_id = Math.random().toString(36).substring(2);
value = _webrtc_id;
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange", async (event) => {
switch (pc.connectionState) {
case "connected":
console.info("connected");
stream_state = "open";
dispatch("tick");
break;
case "disconnected":
console.info("closed");
stop(pc);
break;
case "failed":
stream_state = "closed";
dispatch("error", "Connection failed!");
stop(pc);
break;
default:
break;
}
});
let stream = null;
const timeoutId = setTimeout(() => {
// @ts-ignore
on_change_cb({ type: "connection_timeout" });
}, 5000);
start(
stream,
pc,
audio_player,
server.offer,
_webrtc_id,
"audio",
_on_change_cb,
)
.then((connection) => {
clearTimeout(timeoutId);
pc = connection;
})
.catch(() => {
clearTimeout(timeoutId);
console.info("catching");
dispatch("error", "Too many concurrent users. Come back later!");
});
}
return value;
}
$: start_stream(value).then((val) => {
value = val;
});
</script>
<BlockLabel
{show_label}
Icon={Music}
float={false}
label={label || i18n("audio.audio")}
{show_label}
Icon={Music}
float={false}
label={label || i18n("audio.audio")}
/>
<audio
class="standard-player"
class:hidden={true}
on:load
bind:this={audio_player}
on:ended={() => dispatch("stop")}
on:play={() => dispatch("play")}
class="standard-player"
class:hidden={true}
on:load
bind:this={audio_player}
on:ended={() => dispatch("stop")}
on:play={() => dispatch("play")}
/>
{#if value !== "__webrtc_value__"}
<div class="audio-container">
<AudioWave audio_source_callback={() => audio_player.srcObject} {stream_state} {icon} {icon_button_color} {pulse_color}/>
</div>
<div class="audio-container">
<AudioWave
audio_source_callback={() => audio_player.srcObject}
{stream_state}
{icon}
{icon_button_color}
{pulse_color}
{icon_radius}
/>
</div>
{/if}
{#if value === "__webrtc_value__"}
<Empty size="small">
<Music />
</Empty>
<Empty size="small">
<Music />
</Empty>
{/if}
<style>
.audio-container {
display: flex;
height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
.audio-container {
display: flex;
height: 100%;
flex-direction: column;
justify-content: center;
align-items: center;
}
.standard-player {
width: 100%;
}
.standard-player {
width: 100%;
}
.hidden {
display: none;
}
</style>
.hidden {
display: none;
}
</style>

205
frontend/shared/StaticVideo.svelte
View File

@@ -1,119 +1,132 @@
<script lang="ts">
import { createEventDispatcher, onMount} from "svelte";
import {
BlockLabel,
Empty
} from "@gradio/atoms";
import { Video } from "@gradio/icons";
import { createEventDispatcher, onMount } from "svelte";
import { BlockLabel, Empty } from "@gradio/atoms";
import { Video } from "@gradio/icons";
import { start, stop } from "./webrtc_utils";
import { start, stop } from "./webrtc_utils";
export let value: string | null = null;
export let label: string | undefined = undefined;
export let show_label = true;
export let rtc_configuration: Object | null = null;
export let on_change_cb: (msg: "change" | "tick") => void;
export let server: {
offer: (body: any) => Promise<any>;
};
export let value: string | null = null;
export let label: string | undefined = undefined;
export let show_label = true;
export let rtc_configuration: Object | null = null;
export let on_change_cb: (msg: "change" | "tick") => void;
export let server: {
offer: (body: any) => Promise<any>;
};
let video_element: HTMLVideoElement;
let video_element: HTMLVideoElement;
let _webrtc_id = Math.random().toString(36).substring(2);
let _webrtc_id = Math.random().toString(36).substring(2);
let pc: RTCPeerConnection;
let pc: RTCPeerConnection;
const dispatch = createEventDispatcher<{
error: string;
tick: undefined;
}>();
let stream_state = "closed";
const dispatch = createEventDispatcher<{
error: string;
tick: undefined;
}>();
onMount(() => {
window.setInterval(() => {
if (stream_state == "open") {
dispatch("tick");
}
}, 1000);
}
)
let _on_change_cb = (msg: "change" | "tick" | "stopword" | any) => {
if (msg.type === "end_stream") {
on_change_cb(msg);
stream_state = "closed";
stop(pc);
} else {
console.debug("calling on_change_cb with msg", msg);
on_change_cb(msg);
}
};
$: if( value === "start_webrtc_stream") {
_webrtc_id = Math.random().toString(36).substring(2);
value = _webrtc_id;
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange",
async (event) => {
switch(pc.connectionState) {
case "connected":
console.log("connected");
stream_state = "open";
break;
case "disconnected":
console.log("closed");
stop(pc);
break;
default:
break;
}
}
)
start(null, pc, video_element, server.offer, _webrtc_id, "video", on_change_cb).then((connection) => {
pc = connection;
}).catch(() => {
console.log("catching")
dispatch("error", "Too many concurrent users. Come back later!");
});
}
let stream_state = "closed";
$: if (value === "start_webrtc_stream") {
_webrtc_id = Math.random().toString(36).substring(2);
value = _webrtc_id;
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange", async (event) => {
switch (pc.connectionState) {
case "connected":
stream_state = "open";
dispatch("tick");
break;
case "disconnected":
stop(pc);
break;
case "failed":
stream_state = "closed";
dispatch("error", "Connection failed!");
stop(pc);
break;
default:
break;
}
});
const timeoutId = setTimeout(() => {
// @ts-ignore
on_change_cb({ type: "connection_timeout" });
}, 5000);
start(
null,
pc,
video_element,
server.offer,
_webrtc_id,
"video",
_on_change_cb,
)
.then((connection) => {
clearTimeout(timeoutId);
pc = connection;
})
.catch(() => {
clearTimeout(timeoutId);
dispatch("error", "Too many concurrent users. Come back later!");
});
}
</script>
<BlockLabel {show_label} Icon={Video} label={label || "Video"} />
{#if value === "__webrtc_value__"}
<Empty unpadded_box={true} size="large"><Video /></Empty>
<Empty unpadded_box={true} size="large"><Video /></Empty>
{/if}
<div class="wrap">
<video
class:hidden={value === "__webrtc_value__"}
bind:this={video_element}
autoplay={true}
on:loadeddata={dispatch.bind(null, "loadeddata")}
on:click={dispatch.bind(null, "click")}
on:play={dispatch.bind(null, "play")}
on:pause={dispatch.bind(null, "pause")}
on:ended={dispatch.bind(null, "ended")}
on:mouseover={dispatch.bind(null, "mouseover")}
on:mouseout={dispatch.bind(null, "mouseout")}
on:focus={dispatch.bind(null, "focus")}
on:blur={dispatch.bind(null, "blur")}
on:load
data-testid={$$props["data-testid"]}
crossorigin="anonymous"
>
<track kind="captions" />
</video>
<video
class:hidden={value === "__webrtc_value__"}
bind:this={video_element}
autoplay={true}
on:loadeddata={dispatch.bind(null, "loadeddata")}
on:click={dispatch.bind(null, "click")}
on:play={dispatch.bind(null, "play")}
on:pause={dispatch.bind(null, "pause")}
on:ended={dispatch.bind(null, "ended")}
on:mouseover={dispatch.bind(null, "mouseover")}
on:mouseout={dispatch.bind(null, "mouseout")}
on:focus={dispatch.bind(null, "focus")}
on:blur={dispatch.bind(null, "blur")}
on:load
data-testid={$$props["data-testid"]}
crossorigin="anonymous"
>
<track kind="captions" />
</video>
</div>
<style>
.hidden {
display: none;
}
.hidden {
display: none;
}
.wrap {
position: relative;
background-color: var(--background-fill-secondary);
height: var(--size-full);
width: var(--size-full);
border-radius: var(--radius-xl);
}
.wrap :global(video) {
height: var(--size-full);
width: var(--size-full);
}
.wrap {
position: relative;
background-color: var(--background-fill-secondary);
height: var(--size-full);
width: var(--size-full);
border-radius: var(--radius-xl);
}
.wrap :global(video) {
height: var(--size-full);
width: var(--size-full);
}
</style>

815
frontend/shared/Webcam.svelte
View File

@@ -1,435 +1,474 @@
<script lang="ts">
import { createEventDispatcher, onMount } from "svelte";
import type { ComponentType } from "svelte";
import {
Circle,
Square,
DropdownArrow,
Spinner,
Microphone as Mic
} from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
import { StreamingBar } from "@gradio/statustracker";
import WebcamPermissions from "./WebcamPermissions.svelte";
import { fade } from "svelte/transition";
import {
get_devices,
get_video_stream,
set_available_devices
} from "./stream_utils";
import { start, stop } from "./webrtc_utils";
import PulsingIcon from "./PulsingIcon.svelte";
import { createEventDispatcher, onMount } from "svelte";
import type { ComponentType } from "svelte";
import {
Circle,
Square,
DropdownArrow,
Spinner,
Microphone as Mic,
} from "@gradio/icons";
import type { I18nFormatter } from "@gradio/utils";
import { StreamingBar } from "@gradio/statustracker";
import WebcamPermissions from "./WebcamPermissions.svelte";
import { fade } from "svelte/transition";
import {
get_devices,
get_video_stream,
set_available_devices,
} from "./stream_utils";
import { start, stop } from "./webrtc_utils";
import PulsingIcon from "./PulsingIcon.svelte";
let video_source: HTMLVideoElement;
let available_video_devices: MediaDeviceInfo[] = [];
let selected_device: MediaDeviceInfo | null = null;
let _time_limit: number | null = null;
export let time_limit: number | null = null;
let stream_state: "open" | "waiting" | "closed" = "closed";
export let on_change_cb: (msg: "tick" | "change") => void;
export let mode: "send-receive" | "send";
const _webrtc_id = Math.random().toString(36).substring(2);
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let pulse_color: string = "var(--color-accent)";
export let button_labels: {start: string, stop: string, waiting: string};
let video_source: HTMLVideoElement;
let available_video_devices: MediaDeviceInfo[] = [];
let selected_device: MediaDeviceInfo | null = null;
let _time_limit: number | null = null;
export let time_limit: number | null = null;
let stream_state: "open" | "waiting" | "closed" = "closed";
export let on_change_cb: (msg: "tick" | "change") => void;
export let reject_cb: (msg: object) => void;
export let mode: "send-receive" | "send";
const _webrtc_id = Math.random().toString(36).substring(2);
export let rtp_params: RTCRtpParameters = {} as RTCRtpParameters;
export let icon: string | undefined | ComponentType = undefined;
export let icon_button_color: string = "var(--color-accent)";
export let icon_radius: number = 50;
export let pulse_color: string = "var(--color-accent)";
export let button_labels: { start: string; stop: string; waiting: string };
export const modify_stream: (state: "open" | "closed" | "waiting") => void = (
state: "open" | "closed" | "waiting"
) => {
if (state === "closed") {
_time_limit = null;
stream_state = "closed";
} else if (state === "waiting") {
stream_state = "waiting";
} else {
stream_state = "open";
}
};
export const modify_stream: (state: "open" | "closed" | "waiting") => void = (
state: "open" | "closed" | "waiting",
) => {
if (state === "closed") {
_time_limit = null;
stream_state = "closed";
} else if (state === "waiting") {
stream_state = "waiting";
} else {
stream_state = "open";
}
};
let canvas: HTMLCanvasElement;
export let track_constraints: MediaTrackConstraints | null = null;
export let rtc_configuration: Object;
export let stream_every = 1;
export let server: {
offer: (body: any) => Promise<any>;
};
let canvas: HTMLCanvasElement;
export let track_constraints: MediaTrackConstraints | null = null;
export let rtc_configuration: Object;
export let stream_every = 1;
export let server: {
offer: (body: any) => Promise<any>;
};
export let include_audio: boolean;
export let i18n: I18nFormatter;
export let include_audio: boolean;
export let i18n: I18nFormatter;
const dispatch = createEventDispatcher<{
tick: undefined;
error: string;
start_recording: undefined;
stop_recording: undefined;
close_stream: undefined;
}>();
const dispatch = createEventDispatcher<{
tick: undefined;
error: string;
start_recording: undefined;
stop_recording: undefined;
close_stream: undefined;
}>();
onMount(() => (canvas = document.createElement("canvas")));
onMount(() => (canvas = document.createElement("canvas")));
const handle_device_change = async (event: InputEvent): Promise<void> => {
const target = event.target as HTMLInputElement;
const device_id = target.value;
const handle_device_change = async (event: InputEvent): Promise<void> => {
const target = event.target as HTMLInputElement;
const device_id = target.value;
await get_video_stream(include_audio, video_source, device_id, track_constraints).then(
async (local_stream) => {
stream = local_stream;
selected_device =
available_video_devices.find(
(device) => device.deviceId === device_id
) || null;
options_open = false;
}
);
};
await get_video_stream(
include_audio,
video_source,
device_id,
track_constraints,
).then(async (local_stream) => {
stream = local_stream;
selected_device =
available_video_devices.find(
(device) => device.deviceId === device_id,
) || null;
options_open = false;
});
};
async function access_webcam(): Promise<void> {
try {
get_video_stream(include_audio, video_source, null, track_constraints)
.then(async (local_stream) => {
webcam_accessed = true;
available_video_devices = await get_devices();
stream = local_stream;
})
.then(() => set_available_devices(available_video_devices))
.then((devices) => {
available_video_devices = devices;
async function access_webcam(): Promise<void> {
try {
get_video_stream(include_audio, video_source, null, track_constraints)
.then(async (local_stream) => {
webcam_accessed = true;
available_video_devices = await get_devices();
stream = local_stream;
})
.then(() => set_available_devices(available_video_devices))
.then((devices) => {
available_video_devices = devices;
const used_devices = stream
.getTracks()
.map((track) => track.getSettings()?.deviceId)[0];
const used_devices = stream
.getTracks()
.map((track) => track.getSettings()?.deviceId)[0];
selected_device = used_devices
? devices.find((device) => device.deviceId === used_devices) ||
available_video_devices[0]
: available_video_devices[0];
});
selected_device = used_devices
? devices.find((device) => device.deviceId === used_devices) ||
available_video_devices[0]
: available_video_devices[0];
});
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
dispatch("error", i18n("image.no_webcam_support"));
}
} catch (err) {
if (err instanceof DOMException && err.name == "NotAllowedError") {
dispatch("error", i18n("image.allow_webcam_access"));
} else {
throw err;
}
}
}
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
dispatch("error", i18n("image.no_webcam_support"));
}
} catch (err) {
if (err instanceof DOMException && err.name == "NotAllowedError") {
dispatch("error", i18n("image.allow_webcam_access"));
} else {
throw err;
}
}
}
let recording = false;
let stream: MediaStream;
let _on_change_cb = (msg: "change" | "tick" | "stopword" | any) => {
if (msg.type === "end_stream") {
on_change_cb(msg);
stream_state = "closed";
stop(pc);
access_webcam();
} else {
console.debug("calling on_change_cb with msg", msg);
on_change_cb(msg);
}
};
let webcam_accessed = false;
let pc: RTCPeerConnection;
export let webrtc_id;
let recording = false;
let stream: MediaStream;
async function start_webrtc(): Promise<void> {
if (stream_state === 'closed') {
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange",
async (event) => {
switch(pc.connectionState) {
case "connected":
stream_state = "open";
_time_limit = time_limit;
dispatch("tick");
break;
case "disconnected":
stream_state = "closed";
_time_limit = null;
stop(pc);
await access_webcam();
break;
default:
break;
}
}
)
stream_state = "waiting"
webrtc_id = Math.random().toString(36).substring(2);
start(stream, pc, mode === "send" ? null: video_source, server.offer, webrtc_id, "video", on_change_cb, rtp_params).then((connection) => {
pc = connection;
}).catch(() => {
console.info("catching")
stream_state = "closed";
dispatch("error", "Too many concurrent users. Come back later!");
});
} else {
stop(pc);
let webcam_accessed = false;
let pc: RTCPeerConnection;
export let webrtc_id;
async function start_webrtc(): Promise<void> {
if (stream_state === "closed") {
pc = new RTCPeerConnection(rtc_configuration);
pc.addEventListener("connectionstatechange", async (event) => {
switch (pc.connectionState) {
case "connected":
stream_state = "open";
_time_limit = time_limit;
dispatch("tick");
break;
case "disconnected":
stream_state = "closed";
_time_limit = null;
_time_limit = null;
stop(pc);
await access_webcam();
break;
case "failed":
stream_state = "closed";
_time_limit = null;
dispatch("error", "Connection failed!");
stop(pc);
break;
default:
break;
}
}
});
stream_state = "waiting";
webrtc_id = Math.random().toString(36).substring(2);
const timeoutId = setTimeout(() => {
// @ts-ignore
on_change_cb({ type: "connection_timeout" });
}, 5000);
let options_open = false;
start(
stream,
pc,
mode === "send" ? null : video_source,
server.offer,
webrtc_id,
"video",
_on_change_cb,
rtp_params,
undefined,
reject_cb,
)
.then((connection) => {
clearTimeout(timeoutId);
pc = connection;
})
.catch(() => {
clearTimeout(timeoutId);
console.info("catching");
stream_state = "closed";
});
} else {
stop(pc);
stream_state = "closed";
_time_limit = null;
await access_webcam();
}
}
export function click_outside(node: Node, cb: any): any {
const handle_click = (event: MouseEvent): void => {
if (
node &&
!node.contains(event.target as Node) &&
!event.defaultPrevented
) {
cb(event);
}
};
let options_open = false;
document.addEventListener("click", handle_click, true);
export function click_outside(node: Node, cb: any): any {
const handle_click = (event: MouseEvent): void => {
if (
node &&
!node.contains(event.target as Node) &&
!event.defaultPrevented
) {
cb(event);
}
};
return {
destroy() {
document.removeEventListener("click", handle_click, true);
}
};
}
document.addEventListener("click", handle_click, true);
function handle_click_outside(event: MouseEvent): void {
event.preventDefault();
event.stopPropagation();
options_open = false;
}
return {
destroy() {
document.removeEventListener("click", handle_click, true);
},
};
}
const audio_source_callback = () => video_source.srcObject as MediaStream;
function handle_click_outside(event: MouseEvent): void {
event.preventDefault();
event.stopPropagation();
options_open = false;
}
const audio_source_callback = () => video_source.srcObject as MediaStream;
</script>
<div class="wrap">
<StreamingBar time_limit={_time_limit} />
{#if stream_state === "open" && include_audio}
<div class="audio-indicator">
<PulsingIcon
stream_state={stream_state}
audio_source_callback={audio_source_callback}
icon={icon || Mic}
icon_button_color={icon_button_color}
pulse_color={pulse_color}
/>
</div>
{/if}
<!-- svelte-ignore a11y-media-has-caption -->
<!-- need to suppress for video streaming https://github.com/sveltejs/svelte/issues/5967 -->
<video
bind:this={video_source}
class:hide={!webcam_accessed}
class:flip={(stream_state != "open") || (stream_state === "open" && include_audio)}
autoplay={true}
playsinline={true}
/>
<!-- svelte-ignore a11y-missing-attribute -->
{#if !webcam_accessed}
<div
in:fade={{ delay: 100, duration: 200 }}
title="grant webcam access"
style="height: 100%"
>
<WebcamPermissions on:click={async () => access_webcam()} />
</div>
{:else}
<div class="button-wrap">
<button
on:click={start_webrtc}
aria-label={"start stream"}
>
{#if stream_state === "waiting"}
<div class="icon-with-text">
<div class="icon color-primary" title="spinner">
<Spinner />
</div>
{button_labels.waiting || i18n("audio.waiting")}
</div>
{:else if stream_state === "open"}
<div class="icon-with-text">
<div class="icon color-primary" title="stop recording">
<Square />
</div>
{button_labels.stop || i18n("audio.stop")}
</div>
{:else}
<div class="icon-with-text">
<div class="icon color-primary" title="start recording">
<Circle />
</div>
{button_labels.start || i18n("audio.record")}
</div>
{/if}
</button>
{#if !recording}
<button
class="icon"
on:click={() => (options_open = true)}
aria-label="select input source"
>
<DropdownArrow />
</button>
{/if}
</div>
{#if options_open && selected_device}
<select
class="select-wrap"
aria-label="select source"
use:click_outside={handle_click_outside}
on:change={handle_device_change}
>
<button
class="inset-icon"
on:click|stopPropagation={() => (options_open = false)}
>
<DropdownArrow />
</button>
{#if available_video_devices.length === 0}
<option value="">{i18n("common.no_devices")}</option>
{:else}
{#each available_video_devices as device}
<option
value={device.deviceId}
selected={selected_device.deviceId === device.deviceId}
>
{device.label}
</option>
{/each}
{/if}
</select>
{/if}
{/if}
<StreamingBar time_limit={_time_limit} />
{#if stream_state === "open" && include_audio}
<div class="audio-indicator">
<PulsingIcon
{stream_state}
{audio_source_callback}
icon={icon || Mic}
{icon_button_color}
{pulse_color}
{icon_radius}
/>
</div>
{/if}
<!-- svelte-ignore a11y-media-has-caption -->
<!-- need to suppress for video streaming https://github.com/sveltejs/svelte/issues/5967 -->
<video
bind:this={video_source}
class:hide={!webcam_accessed}
class:flip={stream_state != "open" ||
(stream_state === "open" && include_audio)}
autoplay={true}
playsinline={true}
/>
<!-- svelte-ignore a11y-missing-attribute -->
{#if !webcam_accessed}
<div
in:fade={{ delay: 100, duration: 200 }}
title="grant webcam access"
style="height: 100%"
>
<WebcamPermissions on:click={async () => access_webcam()} />
</div>
{:else}
<div class="button-wrap">
<button on:click={start_webrtc} aria-label={"start stream"}>
{#if stream_state === "waiting"}
<div class="icon-with-text">
<div class="icon color-primary" title="spinner">
<Spinner />
</div>
{button_labels.waiting || i18n("audio.waiting")}
</div>
{:else if stream_state === "open"}
<div class="icon-with-text">
<div class="icon color-primary" title="stop recording">
<Square />
</div>
{button_labels.stop || i18n("audio.stop")}
</div>
{:else}
<div class="icon-with-text">
<div class="icon color-primary" title="start recording">
<Circle />
</div>
{button_labels.start || i18n("audio.record")}
</div>
{/if}
</button>
{#if !recording}
<button
class="icon"
on:click={() => (options_open = true)}
aria-label="select input source"
>
<DropdownArrow />
</button>
{/if}
</div>
{#if options_open && selected_device}
<select
class="select-wrap"
aria-label="select source"
use:click_outside={handle_click_outside}
on:change={handle_device_change}
>
<button
class="inset-icon"
on:click|stopPropagation={() => (options_open = false)}
>
<DropdownArrow />
</button>
{#if available_video_devices.length === 0}
<option value="">{i18n("common.no_devices")}</option>
{:else}
{#each available_video_devices as device}
<option
value={device.deviceId}
selected={selected_device.deviceId === device.deviceId}
>
{device.label}
</option>
{/each}
{/if}
</select>
{/if}
{/if}
</div>
<style>
.wrap {
position: relative;
width: var(--size-full);
height: var(--size-full);
}
.wrap {
position: relative;
width: var(--size-full);
height: var(--size-full);
}
.hide {
display: none;
}
.hide {
display: none;
}
video {
width: var(--size-full);
height: var(--size-full);
object-fit: cover;
}
video {
width: var(--size-full);
height: var(--size-full);
object-fit: cover;
}
.button-wrap {
position: absolute;
background-color: var(--block-background-fill);
border: 1px solid var(--border-color-primary);
border-radius: var(--radius-xl);
padding: var(--size-1-5);
display: flex;
bottom: var(--size-2);
left: 50%;
transform: translate(-50%, 0);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
line-height: var(--size-3);
color: var(--button-secondary-text-color);
}
.button-wrap {
position: absolute;
background-color: var(--block-background-fill);
border: 1px solid var(--border-color-primary);
border-radius: var(--radius-xl);
padding: var(--size-1-5);
display: flex;
bottom: var(--size-2);
left: 50%;
transform: translate(-50%, 0);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
line-height: var(--size-3);
color: var(--button-secondary-text-color);
}
.icon-with-text {
min-width: var(--size-16);
align-items: center;
margin: 0 var(--spacing-xl);
display: flex;
justify-content: space-evenly;
/* Add gap between icon and text */
gap: var(--size-2);
}
.icon-with-text {
min-width: var(--size-16);
align-items: center;
margin: 0 var(--spacing-xl);
display: flex;
justify-content: space-evenly;
/* Add gap between icon and text */
gap: var(--size-2);
}
.audio-indicator {
position: absolute;
top: var(--size-2);
right: var(--size-2);
z-index: var(--layer-2);
height: var(--size-5);
width: var(--size-5);
.audio-indicator {
position: absolute;
top: var(--size-2);
right: var(--size-2);
z-index: var(--layer-2);
height: var(--size-5);
width: var(--size-5);
}
@media (--screen-md) {
button {
bottom: var(--size-4);
}
}
@media (--screen-md) {
button {
bottom: var(--size-4);
}
}
@media (--screen-xl) {
button {
bottom: var(--size-8);
}
}
@media (--screen-xl) {
button {
bottom: var(--size-8);
}
}
.icon {
width: 18px;
height: 18px;
display: flex;
justify-content: space-between;
align-items: center;
}
.icon {
width: 18px;
height: 18px;
display: flex;
justify-content: space-between;
align-items: center;
}
.color-primary {
fill: var(--primary-600);
stroke: var(--primary-600);
color: var(--primary-600);
}
.color-primary {
fill: var(--primary-600);
stroke: var(--primary-600);
color: var(--primary-600);
}
.flip {
transform: scaleX(-1);
}
.flip {
transform: scaleX(-1);
}
.select-wrap {
-webkit-appearance: none;
-moz-appearance: none;
appearance: none;
color: var(--button-secondary-text-color);
background-color: transparent;
width: 95%;
font-size: var(--text-md);
position: absolute;
bottom: var(--size-2);
background-color: var(--block-background-fill);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
z-index: var(--layer-top);
border: 1px solid var(--border-color-primary);
text-align: left;
line-height: var(--size-4);
white-space: nowrap;
text-overflow: ellipsis;
left: 50%;
transform: translate(-50%, 0);
max-width: var(--size-52);
}
.select-wrap {
-webkit-appearance: none;
-moz-appearance: none;
appearance: none;
color: var(--button-secondary-text-color);
background-color: transparent;
width: 95%;
font-size: var(--text-md);
position: absolute;
bottom: var(--size-2);
background-color: var(--block-background-fill);
box-shadow: var(--shadow-drop-lg);
border-radius: var(--radius-xl);
z-index: var(--layer-top);
border: 1px solid var(--border-color-primary);
text-align: left;
line-height: var(--size-4);
white-space: nowrap;
text-overflow: ellipsis;
left: 50%;
transform: translate(-50%, 0);
max-width: var(--size-52);
}
.select-wrap > option {
padding: 0.25rem 0.5rem;
border-bottom: 1px solid var(--border-color-accent);
padding-right: var(--size-8);
text-overflow: ellipsis;
overflow: hidden;
}
.select-wrap > option {
padding: 0.25rem 0.5rem;
border-bottom: 1px solid var(--border-color-accent);
padding-right: var(--size-8);
text-overflow: ellipsis;
overflow: hidden;
}
.select-wrap > option:hover {
background-color: var(--color-accent);
}
.select-wrap > option:hover {
background-color: var(--color-accent);
}
.select-wrap > option:last-child {
border: none;
}
.select-wrap > option:last-child {
border: none;
}
.inset-icon {
position: absolute;
top: 5px;
right: -6.5px;
width: var(--size-10);
height: var(--size-5);
opacity: 0.8;
}
.inset-icon {
position: absolute;
top: 5px;
right: -6.5px;
width: var(--size-10);
height: var(--size-5);
opacity: 0.8;
}
@media (--screen-md) {
.wrap {
font-size: var(--text-lg);
}
}
</style>
@media (--screen-md) {
.wrap {
font-size: var(--text-lg);
}
}
</style>

75
frontend/shared/WebcamPermissions.svelte
View File

@@ -1,49 +1,50 @@
<script lang="ts">
import { Webcam } from "@gradio/icons";
import { createEventDispatcher } from "svelte";
import { Webcam } from "@gradio/icons";
import { createEventDispatcher } from "svelte";
export let icon = Webcam;
$: text = icon === Webcam ? "Click to Access Webcam" : "Click to Access Microphone";
export let icon = Webcam;
$: text =
icon === Webcam ? "Click to Access Webcam" : "Click to Access Microphone";
const dispatch = createEventDispatcher<{
click: undefined;
}>();
const dispatch = createEventDispatcher<{
click: undefined;
}>();
</script>
<button style:height="100%" on:click={() => dispatch("click")}>
<div class="wrap">
<span class="icon-wrap">
<svelte:component this={icon} />
</span>
{text}
</div>
<div class="wrap">
<span class="icon-wrap">
<svelte:component this={icon} />
</span>
{text}
</div>
</button>
<style>
button {
cursor: pointer;
width: var(--size-full);
}
button {
cursor: pointer;
width: var(--size-full);
}
.wrap {
display: flex;
flex-direction: column;
justify-content: center;
align-items: center;
min-height: var(--size-60);
color: var(--block-label-text-color);
height: 100%;
padding-top: var(--size-3);
}
.wrap {
display: flex;
flex-direction: column;
justify-content: center;
align-items: center;
min-height: var(--size-60);
color: var(--block-label-text-color);
height: 100%;
padding-top: var(--size-3);
}
.icon-wrap {
width: 30px;
margin-bottom: var(--spacing-lg);
}
.icon-wrap {
width: 30px;
margin-bottom: var(--spacing-lg);
}
@media (--screen-md) {
.wrap {
font-size: var(--text-lg);
}
}
</style>
@media (--screen-md) {
.wrap {
font-size: var(--text-lg);
}
}
</style>

64
frontend/shared/webrtc_utils.ts
View File

@@ -53,6 +53,8 @@ export async function start(
modality: "video" | "audio" = "video",
on_change_cb: (msg: "change" | "tick") => void = () => {},
rtp_params = {},
additional_message_cb: (msg: object) => void = () => {},
reject_cb: (msg: object) => void = () => {},
) {
pc = createPeerConnection(pc, node);
const data_channel = pc.createDataChannel("text");
@@ -70,17 +72,20 @@ export async function start(
} catch (e) {
console.debug("Error parsing JSON");
}
console.log("event_json", event_json);
if (
event.data === "change" ||
event.data === "tick" ||
event.data === "stopword" ||
event_json?.type === "warning" ||
event_json?.type === "error"
event_json?.type === "error" ||
event_json?.type === "send_input" ||
event_json?.type === "fetch_output" ||
event_json?.type === "stopword" ||
event_json?.type === "end_stream"
) {
console.debug(`${event.data} event received`);
on_change_cb(event_json ?? event.data);
}
additional_message_cb(event_json ?? event.data);
};
if (stream) {
@@ -97,15 +102,20 @@ export async function start(
pc.addTransceiver(modality, { direction: "recvonly" });
}
await negotiate(pc, server_fn, webrtc_id);
await negotiate(pc, server_fn, webrtc_id, reject_cb);
return pc;
}
function make_offer(server_fn: any, body): Promise<object> {
function make_offer(
server_fn: any,
body,
reject_cb: (msg: object) => void = () => {},
): Promise<object> {
return new Promise((resolve, reject) => {
server_fn(body).then((data) => {
console.debug("data", data);
if (data?.status === "failed") {
reject_cb(data);
console.debug("rejecting");
reject("error");
}
@@ -118,37 +128,35 @@ async function negotiate(
pc: RTCPeerConnection,
server_fn: any,
webrtc_id: string,
reject_cb: (msg: object) => void = () => {},
): Promise<void> {
pc.onicecandidate = ({ candidate }) => {
if (candidate) {
console.debug("Sending ICE candidate", candidate);
server_fn({
candidate: candidate.toJSON(),
webrtc_id: webrtc_id,
type: "ice-candidate",
}).catch((err) => console.error("Error sending ICE candidate:", err));
}
};
return pc
.createOffer()
.then((offer) => {
return pc.setLocalDescription(offer);
})
.then(() => {
// wait for ICE gathering to complete
return new Promise<void>((resolve) => {
console.debug("ice gathering state", pc.iceGatheringState);
if (pc.iceGatheringState === "complete") {
resolve();
} else {
const checkState = () => {
if (pc.iceGatheringState === "complete") {
console.debug("ice complete");
pc.removeEventListener("icegatheringstatechange", checkState);
resolve();
}
};
pc.addEventListener("icegatheringstatechange", checkState);
}
});
})
.then(() => {
var offer = pc.localDescription;
return make_offer(server_fn, {
sdp: offer.sdp,
type: offer.type,
webrtc_id: webrtc_id,
});
return make_offer(
server_fn,
{
sdp: offer.sdp,
type: offer.type,
webrtc_id: webrtc_id,
},
reject_cb,
);
})
.then((response) => {
return response;