mirror of
https://github.com/HumanAIGC-Engineering/gradio-webrtc.git
synced 2026-02-05 09:59:22 +08:00
Working draft
This commit is contained in:
freddyaboulton
87
frontend/shared/InteractiveVideo.svelte
Normal file
87
frontend/shared/InteractiveVideo.svelte
Normal file
@@ -0,0 +1,87 @@
|
||||
<script lang="ts">
|
||||
import { createEventDispatcher } from "svelte";
|
||||
import type { FileData, Client } from "@gradio/client";
|
||||
import { BlockLabel } from "@gradio/atoms";
|
||||
import Webcam from "./Webcam.svelte";
|
||||
import { Video } from "@gradio/icons";
|
||||
|
||||
import type { I18nFormatter } from "@gradio/utils";
|
||||
|
||||
export let value: string = null;
|
||||
export let label: string | undefined = undefined;
|
||||
export let show_label = true;
|
||||
export let include_audio: boolean;
|
||||
export let root: string;
|
||||
export let i18n: I18nFormatter;
|
||||
export let active_source: "webcam" | "upload" = "webcam";
|
||||
export let handle_reset_value: () => void = () => {};
|
||||
export let stream_handler: Client["stream"];
|
||||
export let server: {
|
||||
offer: (body: any) => Promise<any>;
|
||||
};
|
||||
|
||||
let has_change_history = false;
|
||||
|
||||
const dispatch = createEventDispatcher<{
|
||||
change: FileData | null;
|
||||
clear?: never;
|
||||
play?: never;
|
||||
pause?: never;
|
||||
end?: never;
|
||||
drag: boolean;
|
||||
error: string;
|
||||
upload: FileData;
|
||||
start_recording?: never;
|
||||
stop_recording?: never;
|
||||
tick: never;
|
||||
}>();
|
||||
|
||||
let dragging = false;
|
||||
$: dispatch("drag", dragging);
|
||||
|
||||
$: console.log("interactive value", value);
|
||||
</script>
|
||||
|
||||
<BlockLabel {show_label} Icon={Video} label={label || "Video"} />
|
||||
<div data-testid="video" class="video-container">
|
||||
<Webcam
|
||||
{root}
|
||||
{include_audio}
|
||||
on:error
|
||||
on:start_recording
|
||||
on:stop_recording
|
||||
on:tick
|
||||
{i18n}
|
||||
stream_every={0.5}
|
||||
{server}
|
||||
bind:webrtc_id={value}
|
||||
/>
|
||||
|
||||
<!-- <SelectSource {sources} bind:active_source /> -->
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.file-name {
|
||||
padding: var(--size-6);
|
||||
font-size: var(--text-xxl);
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.file-size {
|
||||
padding: var(--size-2);
|
||||
font-size: var(--text-xl);
|
||||
}
|
||||
|
||||
.upload-container {
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.video-container {
|
||||
display: flex;
|
||||
height: 100%;
|
||||
flex-direction: column;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
}
|
||||
</style>
|
||||
270
frontend/shared/Player.svelte
Normal file
270
frontend/shared/Player.svelte
Normal file
@@ -0,0 +1,270 @@
|
||||
<script lang="ts">
|
||||
import { createEventDispatcher } from "svelte";
|
||||
import { Play, Pause, Maximise, Undo } from "@gradio/icons";
|
||||
import Video from "./Video.svelte";
|
||||
import VideoControls from "./VideoControls.svelte";
|
||||
import type { FileData, Client } from "@gradio/client";
|
||||
import { prepare_files } from "@gradio/client";
|
||||
import { format_time } from "@gradio/utils";
|
||||
import type { I18nFormatter } from "@gradio/utils";
|
||||
|
||||
export let root = "";
|
||||
export let src: string;
|
||||
export let subtitle: string | null = null;
|
||||
export let mirror: boolean;
|
||||
export let autoplay: boolean;
|
||||
export let loop: boolean;
|
||||
export let label = "test";
|
||||
export let interactive = false;
|
||||
export let handle_change: (video: FileData) => void = () => {};
|
||||
export let handle_reset_value: () => void = () => {};
|
||||
export let upload: Client["upload"];
|
||||
export let is_stream: boolean | undefined;
|
||||
export let i18n: I18nFormatter;
|
||||
export let show_download_button = false;
|
||||
export let value: FileData | null = null;
|
||||
export let handle_clear: () => void = () => {};
|
||||
export let has_change_history = false;
|
||||
|
||||
const dispatch = createEventDispatcher<{
|
||||
play: undefined;
|
||||
pause: undefined;
|
||||
stop: undefined;
|
||||
end: undefined;
|
||||
clear: undefined;
|
||||
}>();
|
||||
|
||||
let time = 0;
|
||||
let duration: number;
|
||||
let paused = true;
|
||||
let video: HTMLVideoElement;
|
||||
let processingVideo = false;
|
||||
|
||||
function handleMove(e: TouchEvent | MouseEvent): void {
|
||||
if (!duration) return;
|
||||
|
||||
if (e.type === "click") {
|
||||
handle_click(e as MouseEvent);
|
||||
return;
|
||||
}
|
||||
|
||||
if (e.type !== "touchmove" && !((e as MouseEvent).buttons & 1)) return;
|
||||
|
||||
const clientX =
|
||||
e.type === "touchmove"
|
||||
? (e as TouchEvent).touches[0].clientX
|
||||
: (e as MouseEvent).clientX;
|
||||
const { left, right } = (
|
||||
e.currentTarget as HTMLProgressElement
|
||||
).getBoundingClientRect();
|
||||
time = (duration * (clientX - left)) / (right - left);
|
||||
}
|
||||
|
||||
async function play_pause(): Promise<void> {
|
||||
if (document.fullscreenElement != video) {
|
||||
const isPlaying =
|
||||
video.currentTime > 0 &&
|
||||
!video.paused &&
|
||||
!video.ended &&
|
||||
video.readyState > video.HAVE_CURRENT_DATA;
|
||||
|
||||
if (!isPlaying) {
|
||||
await video.play();
|
||||
} else video.pause();
|
||||
}
|
||||
}
|
||||
|
||||
function handle_click(e: MouseEvent): void {
|
||||
const { left, right } = (
|
||||
e.currentTarget as HTMLProgressElement
|
||||
).getBoundingClientRect();
|
||||
time = (duration * (e.clientX - left)) / (right - left);
|
||||
}
|
||||
|
||||
function handle_end(): void {
|
||||
dispatch("stop");
|
||||
dispatch("end");
|
||||
}
|
||||
|
||||
const handle_trim_video = async (videoBlob: Blob): Promise<void> => {
|
||||
let _video_blob = new File([videoBlob], "video.mp4");
|
||||
const val = await prepare_files([_video_blob]);
|
||||
let value = ((await upload(val, root))?.filter(Boolean) as FileData[])[0];
|
||||
|
||||
handle_change(value);
|
||||
};
|
||||
|
||||
function open_full_screen(): void {
|
||||
video.requestFullscreen();
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="wrap">
|
||||
<div class="mirror-wrap" class:mirror>
|
||||
<Video
|
||||
{src}
|
||||
preload="auto"
|
||||
{autoplay}
|
||||
{loop}
|
||||
{is_stream}
|
||||
on:click={play_pause}
|
||||
on:play
|
||||
on:pause
|
||||
on:ended={handle_end}
|
||||
bind:currentTime={time}
|
||||
bind:duration
|
||||
bind:paused
|
||||
bind:node={video}
|
||||
data-testid={`${label}-player`}
|
||||
{processingVideo}
|
||||
on:load
|
||||
>
|
||||
<track kind="captions" src={subtitle} default />
|
||||
</Video>
|
||||
</div>
|
||||
|
||||
<div class="controls">
|
||||
<div class="inner">
|
||||
<span
|
||||
role="button"
|
||||
tabindex="0"
|
||||
class="icon"
|
||||
aria-label="play-pause-replay-button"
|
||||
on:click={play_pause}
|
||||
on:keydown={play_pause}
|
||||
>
|
||||
{#if time === duration}
|
||||
<Undo />
|
||||
{:else if paused}
|
||||
<Play />
|
||||
{:else}
|
||||
<Pause />
|
||||
{/if}
|
||||
</span>
|
||||
|
||||
<span class="time">{format_time(time)} / {format_time(duration)}</span>
|
||||
|
||||
<!-- TODO: implement accessible video timeline for 4.0 -->
|
||||
<!-- svelte-ignore a11y-click-events-have-key-events -->
|
||||
<!-- svelte-ignore a11y-no-noninteractive-element-interactions -->
|
||||
<progress
|
||||
value={time / duration || 0}
|
||||
on:mousemove={handleMove}
|
||||
on:touchmove|preventDefault={handleMove}
|
||||
on:click|stopPropagation|preventDefault={handle_click}
|
||||
/>
|
||||
|
||||
<div
|
||||
role="button"
|
||||
tabindex="0"
|
||||
class="icon"
|
||||
aria-label="full-screen"
|
||||
on:click={open_full_screen}
|
||||
on:keypress={open_full_screen}
|
||||
>
|
||||
<Maximise />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{#if interactive}
|
||||
<VideoControls
|
||||
videoElement={video}
|
||||
showRedo
|
||||
{handle_trim_video}
|
||||
{handle_reset_value}
|
||||
bind:processingVideo
|
||||
{value}
|
||||
{i18n}
|
||||
{show_download_button}
|
||||
{handle_clear}
|
||||
{has_change_history}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<style lang="postcss">
|
||||
span {
|
||||
text-shadow: 0 0 8px rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
|
||||
progress {
|
||||
margin-right: var(--size-3);
|
||||
border-radius: var(--radius-sm);
|
||||
width: var(--size-full);
|
||||
height: var(--size-2);
|
||||
}
|
||||
|
||||
progress::-webkit-progress-bar {
|
||||
border-radius: 2px;
|
||||
background-color: rgba(255, 255, 255, 0.2);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
progress::-webkit-progress-value {
|
||||
background-color: rgba(255, 255, 255, 0.9);
|
||||
}
|
||||
|
||||
.mirror {
|
||||
transform: scaleX(-1);
|
||||
}
|
||||
|
||||
.mirror-wrap {
|
||||
position: relative;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.controls {
|
||||
position: absolute;
|
||||
bottom: 0;
|
||||
opacity: 0;
|
||||
transition: 500ms;
|
||||
margin: var(--size-2);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--color-grey-800);
|
||||
padding: var(--size-2) var(--size-1);
|
||||
width: calc(100% - 0.375rem * 2);
|
||||
width: calc(100% - var(--size-2) * 2);
|
||||
}
|
||||
.wrap:hover .controls {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.inner {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding-right: var(--size-2);
|
||||
padding-left: var(--size-2);
|
||||
width: var(--size-full);
|
||||
height: var(--size-full);
|
||||
}
|
||||
|
||||
.icon {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
cursor: pointer;
|
||||
width: var(--size-6);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.time {
|
||||
flex-shrink: 0;
|
||||
margin-right: var(--size-3);
|
||||
margin-left: var(--size-3);
|
||||
color: white;
|
||||
font-size: var(--text-sm);
|
||||
font-family: var(--font-mono);
|
||||
}
|
||||
.wrap {
|
||||
position: relative;
|
||||
background-color: var(--background-fill-secondary);
|
||||
height: var(--size-full);
|
||||
width: var(--size-full);
|
||||
border-radius: var(--radius-xl);
|
||||
}
|
||||
.wrap :global(video) {
|
||||
height: var(--size-full);
|
||||
width: var(--size-full);
|
||||
}
|
||||
</style>
|
||||
197
frontend/shared/Video.svelte
Normal file
197
frontend/shared/Video.svelte
Normal file
@@ -0,0 +1,197 @@
|
||||
<script lang="ts">
|
||||
import type { HTMLVideoAttributes } from "svelte/elements";
|
||||
import { createEventDispatcher } from "svelte";
|
||||
import { loaded } from "./utils";
|
||||
|
||||
import { resolve_wasm_src } from "@gradio/wasm/svelte";
|
||||
|
||||
import Hls from "hls.js";
|
||||
|
||||
export let src: HTMLVideoAttributes["src"] = undefined;
|
||||
|
||||
export let muted: HTMLVideoAttributes["muted"] = undefined;
|
||||
export let playsinline: HTMLVideoAttributes["playsinline"] = undefined;
|
||||
export let preload: HTMLVideoAttributes["preload"] = undefined;
|
||||
export let autoplay: HTMLVideoAttributes["autoplay"] = undefined;
|
||||
export let controls: HTMLVideoAttributes["controls"] = undefined;
|
||||
|
||||
export let currentTime: number | undefined = undefined;
|
||||
export let duration: number | undefined = undefined;
|
||||
export let paused: boolean | undefined = undefined;
|
||||
|
||||
export let node: HTMLVideoElement | undefined = undefined;
|
||||
export let loop: boolean;
|
||||
export let is_stream;
|
||||
|
||||
export let processingVideo = false;
|
||||
|
||||
let resolved_src: typeof src;
|
||||
let stream_active = false;
|
||||
|
||||
// The `src` prop can be updated before the Promise from `resolve_wasm_src` is resolved.
|
||||
// In such a case, the resolved value for the old `src` has to be discarded,
|
||||
// This variable `latest_src` is used to pick up only the value resolved for the latest `src` prop.
|
||||
let latest_src: typeof src;
|
||||
$: {
|
||||
// In normal (non-Wasm) Gradio, the `<img>` element should be rendered with the passed `src` props immediately
|
||||
// without waiting for `resolve_wasm_src()` to resolve.
|
||||
// If it waits, a blank element is displayed until the async task finishes
|
||||
// and it leads to undesirable flickering.
|
||||
// So set `src` to `resolved_src` here.
|
||||
resolved_src = src;
|
||||
|
||||
latest_src = src;
|
||||
const resolving_src = src;
|
||||
resolve_wasm_src(resolving_src).then((s) => {
|
||||
if (latest_src === resolving_src) {
|
||||
resolved_src = s;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const dispatch = createEventDispatcher();
|
||||
|
||||
function load_stream(
|
||||
src: string | null | undefined,
|
||||
is_stream: boolean,
|
||||
node: HTMLVideoElement | undefined
|
||||
): void {
|
||||
if (!src || !is_stream) return;
|
||||
if (!node) return;
|
||||
if (Hls.isSupported() && !stream_active) {
|
||||
const hls = new Hls({
|
||||
maxBufferLength: 1, // 0.5 seconds (500 ms)
|
||||
maxMaxBufferLength: 1, // Maximum max buffer length in seconds
|
||||
lowLatencyMode: true // Enable low latency mode
|
||||
});
|
||||
hls.loadSource(src);
|
||||
hls.attachMedia(node);
|
||||
hls.on(Hls.Events.MANIFEST_PARSED, function () {
|
||||
(node as HTMLVideoElement).play();
|
||||
});
|
||||
hls.on(Hls.Events.ERROR, function (event, data) {
|
||||
console.error("HLS error:", event, data);
|
||||
if (data.fatal) {
|
||||
switch (data.type) {
|
||||
case Hls.ErrorTypes.NETWORK_ERROR:
|
||||
console.error(
|
||||
"Fatal network error encountered, trying to recover"
|
||||
);
|
||||
hls.startLoad();
|
||||
break;
|
||||
case Hls.ErrorTypes.MEDIA_ERROR:
|
||||
console.error("Fatal media error encountered, trying to recover");
|
||||
hls.recoverMediaError();
|
||||
break;
|
||||
default:
|
||||
console.error("Fatal error, cannot recover");
|
||||
hls.destroy();
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
stream_active = true;
|
||||
}
|
||||
}
|
||||
|
||||
$: src, (stream_active = false);
|
||||
|
||||
$: load_stream(src, is_stream, node);
|
||||
</script>
|
||||
|
||||
<!--
|
||||
The spread operator with `$$props` or `$$restProps` can't be used here
|
||||
to pass props from the parent component to the <video> element
|
||||
because of its unexpected behavior: https://github.com/sveltejs/svelte/issues/7404
|
||||
For example, if we add {...$$props} or {...$$restProps}, the boolean props aside it like `controls` will be compiled as string "true" or "false" on the actual DOM.
|
||||
Then, even when `controls` is false, the compiled DOM would be `<video controls="false">` which is equivalent to `<video controls>` since the string "false" is even truthy.
|
||||
-->
|
||||
<div class:hidden={!processingVideo} class="overlay">
|
||||
<span class="load-wrap">
|
||||
<span class="loader" />
|
||||
</span>
|
||||
</div>
|
||||
<video
|
||||
src={resolved_src}
|
||||
{muted}
|
||||
{playsinline}
|
||||
{preload}
|
||||
{autoplay}
|
||||
{controls}
|
||||
{loop}
|
||||
on:loadeddata={dispatch.bind(null, "loadeddata")}
|
||||
on:click={dispatch.bind(null, "click")}
|
||||
on:play={dispatch.bind(null, "play")}
|
||||
on:pause={dispatch.bind(null, "pause")}
|
||||
on:ended={dispatch.bind(null, "ended")}
|
||||
on:mouseover={dispatch.bind(null, "mouseover")}
|
||||
on:mouseout={dispatch.bind(null, "mouseout")}
|
||||
on:focus={dispatch.bind(null, "focus")}
|
||||
on:blur={dispatch.bind(null, "blur")}
|
||||
on:load
|
||||
bind:currentTime
|
||||
bind:duration
|
||||
bind:paused
|
||||
bind:this={node}
|
||||
use:loaded={{ autoplay: autoplay ?? false }}
|
||||
data-testid={$$props["data-testid"]}
|
||||
crossorigin="anonymous"
|
||||
>
|
||||
<slot />
|
||||
</video>
|
||||
|
||||
<style>
|
||||
.overlay {
|
||||
position: absolute;
|
||||
background-color: rgba(0, 0, 0, 0.4);
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.load-wrap {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.loader {
|
||||
display: flex;
|
||||
position: relative;
|
||||
background-color: var(--border-color-accent-subdued);
|
||||
animation: shadowPulse 2s linear infinite;
|
||||
box-shadow:
|
||||
-24px 0 var(--border-color-accent-subdued),
|
||||
24px 0 var(--border-color-accent-subdued);
|
||||
margin: var(--spacing-md);
|
||||
border-radius: 50%;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
scale: 0.5;
|
||||
}
|
||||
|
||||
@keyframes shadowPulse {
|
||||
33% {
|
||||
box-shadow:
|
||||
-24px 0 var(--border-color-accent-subdued),
|
||||
24px 0 #fff;
|
||||
background: #fff;
|
||||
}
|
||||
66% {
|
||||
box-shadow:
|
||||
-24px 0 #fff,
|
||||
24px 0 #fff;
|
||||
background: var(--border-color-accent-subdued);
|
||||
}
|
||||
100% {
|
||||
box-shadow:
|
||||
-24px 0 #fff,
|
||||
24px 0 var(--border-color-accent-subdued);
|
||||
background: #fff;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
202
frontend/shared/VideoControls.svelte
Normal file
202
frontend/shared/VideoControls.svelte
Normal file
@@ -0,0 +1,202 @@
|
||||
<script lang="ts">
|
||||
import { Undo, Trim, Clear } from "@gradio/icons";
|
||||
import VideoTimeline from "./VideoTimeline.svelte";
|
||||
import { trimVideo } from "./utils";
|
||||
import { FFmpeg } from "@ffmpeg/ffmpeg";
|
||||
import loadFfmpeg from "./utils";
|
||||
import { onMount } from "svelte";
|
||||
import { format_time } from "@gradio/utils";
|
||||
import { IconButton } from "@gradio/atoms";
|
||||
import { ModifyUpload } from "@gradio/upload";
|
||||
import type { FileData } from "@gradio/client";
|
||||
|
||||
export let videoElement: HTMLVideoElement;
|
||||
|
||||
export let showRedo = false;
|
||||
export let interactive = true;
|
||||
export let mode = "";
|
||||
export let handle_reset_value: () => void;
|
||||
export let handle_trim_video: (videoBlob: Blob) => void;
|
||||
export let processingVideo = false;
|
||||
export let i18n: (key: string) => string;
|
||||
export let value: FileData | null = null;
|
||||
export let show_download_button = false;
|
||||
export let handle_clear: () => void = () => {};
|
||||
export let has_change_history = false;
|
||||
|
||||
let ffmpeg: FFmpeg;
|
||||
|
||||
onMount(async () => {
|
||||
ffmpeg = await loadFfmpeg();
|
||||
});
|
||||
|
||||
$: if (mode === "edit" && trimmedDuration === null && videoElement)
|
||||
trimmedDuration = videoElement.duration;
|
||||
|
||||
let trimmedDuration: number | null = null;
|
||||
let dragStart = 0;
|
||||
let dragEnd = 0;
|
||||
|
||||
let loadingTimeline = false;
|
||||
|
||||
const toggleTrimmingMode = (): void => {
|
||||
if (mode === "edit") {
|
||||
mode = "";
|
||||
trimmedDuration = videoElement.duration;
|
||||
} else {
|
||||
mode = "edit";
|
||||
}
|
||||
};
|
||||
</script>
|
||||
|
||||
<div class="container" class:hidden={mode !== "edit"}>
|
||||
{#if mode === "edit"}
|
||||
<div class="timeline-wrapper">
|
||||
<VideoTimeline
|
||||
{videoElement}
|
||||
bind:dragStart
|
||||
bind:dragEnd
|
||||
bind:trimmedDuration
|
||||
bind:loadingTimeline
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<div class="controls" data-testid="waveform-controls">
|
||||
{#if mode === "edit" && trimmedDuration !== null}
|
||||
<time
|
||||
aria-label="duration of selected region in seconds"
|
||||
class:hidden={loadingTimeline}>{format_time(trimmedDuration)}</time
|
||||
>
|
||||
<div class="edit-buttons">
|
||||
<button
|
||||
class:hidden={loadingTimeline}
|
||||
class="text-button"
|
||||
on:click={() => {
|
||||
mode = "";
|
||||
processingVideo = true;
|
||||
trimVideo(ffmpeg, dragStart, dragEnd, videoElement)
|
||||
.then((videoBlob) => {
|
||||
handle_trim_video(videoBlob);
|
||||
})
|
||||
.then(() => {
|
||||
processingVideo = false;
|
||||
});
|
||||
}}>Trim</button
|
||||
>
|
||||
<button
|
||||
class="text-button"
|
||||
class:hidden={loadingTimeline}
|
||||
on:click={toggleTrimmingMode}>Cancel</button
|
||||
>
|
||||
</div>
|
||||
{:else}
|
||||
<div />
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ModifyUpload
|
||||
{i18n}
|
||||
on:clear={() => handle_clear()}
|
||||
download={show_download_button ? value?.url : null}
|
||||
>
|
||||
{#if showRedo && mode === ""}
|
||||
<IconButton
|
||||
Icon={Undo}
|
||||
label="Reset video to initial value"
|
||||
disabled={processingVideo || !has_change_history}
|
||||
on:click={() => {
|
||||
handle_reset_value();
|
||||
mode = "";
|
||||
}}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
{#if interactive && mode === ""}
|
||||
<IconButton
|
||||
Icon={Trim}
|
||||
label="Trim video to selection"
|
||||
disabled={processingVideo}
|
||||
on:click={toggleTrimmingMode}
|
||||
/>
|
||||
{/if}
|
||||
</ModifyUpload>
|
||||
|
||||
<style>
|
||||
.container {
|
||||
width: 100%;
|
||||
}
|
||||
time {
|
||||
color: var(--color-accent);
|
||||
font-weight: bold;
|
||||
padding-left: var(--spacing-xs);
|
||||
}
|
||||
|
||||
.timeline-wrapper {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.text-button {
|
||||
border: 1px solid var(--neutral-400);
|
||||
border-radius: var(--radius-sm);
|
||||
font-weight: 300;
|
||||
font-size: var(--size-3);
|
||||
text-align: center;
|
||||
color: var(--neutral-400);
|
||||
height: var(--size-5);
|
||||
font-weight: bold;
|
||||
padding: 0 5px;
|
||||
margin-left: 5px;
|
||||
}
|
||||
|
||||
.text-button:hover,
|
||||
.text-button:focus {
|
||||
color: var(--color-accent);
|
||||
border-color: var(--color-accent);
|
||||
}
|
||||
|
||||
.controls {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin: var(--spacing-lg);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.edit-buttons {
|
||||
display: flex;
|
||||
gap: var(--spacing-sm);
|
||||
}
|
||||
|
||||
@media (max-width: 320px) {
|
||||
.controls {
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
}
|
||||
|
||||
.edit-buttons {
|
||||
margin-top: var(--spacing-sm);
|
||||
}
|
||||
|
||||
.controls * {
|
||||
margin: var(--spacing-sm);
|
||||
}
|
||||
|
||||
.controls .text-button {
|
||||
margin-left: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.hidden {
|
||||
display: none;
|
||||
}
|
||||
</style>
|
||||
108
frontend/shared/VideoPreview.svelte
Normal file
108
frontend/shared/VideoPreview.svelte
Normal file
@@ -0,0 +1,108 @@
|
||||
<script lang="ts">
|
||||
import { createEventDispatcher, afterUpdate, tick } from "svelte";
|
||||
import {
|
||||
BlockLabel,
|
||||
Empty,
|
||||
IconButton,
|
||||
ShareButton,
|
||||
IconButtonWrapper
|
||||
} from "@gradio/atoms";
|
||||
import type { FileData, Client } from "@gradio/client";
|
||||
import { Video, Download } from "@gradio/icons";
|
||||
import { DownloadLink } from "@gradio/wasm/svelte";
|
||||
|
||||
import Player from "./Player.svelte";
|
||||
import type { I18nFormatter } from "js/core/src/gradio_helper";
|
||||
|
||||
export let value: FileData | null = null;
|
||||
export let subtitle: FileData | null = null;
|
||||
export let label: string | undefined = undefined;
|
||||
export let show_label = true;
|
||||
export let autoplay: boolean;
|
||||
export let show_share_button = true;
|
||||
export let show_download_button = true;
|
||||
export let loop: boolean;
|
||||
export let i18n: I18nFormatter;
|
||||
export let upload: Client["upload"];
|
||||
|
||||
let old_value: FileData | null = null;
|
||||
let old_subtitle: FileData | null = null;
|
||||
|
||||
const dispatch = createEventDispatcher<{
|
||||
change: FileData;
|
||||
play: undefined;
|
||||
pause: undefined;
|
||||
end: undefined;
|
||||
stop: undefined;
|
||||
}>();
|
||||
|
||||
$: value && dispatch("change", value);
|
||||
|
||||
afterUpdate(async () => {
|
||||
// needed to bust subtitle caching issues on Chrome
|
||||
if (
|
||||
value !== old_value &&
|
||||
subtitle !== old_subtitle &&
|
||||
old_subtitle !== null
|
||||
) {
|
||||
old_value = value;
|
||||
value = null;
|
||||
await tick();
|
||||
value = old_value;
|
||||
}
|
||||
old_value = value;
|
||||
old_subtitle = subtitle;
|
||||
});
|
||||
</script>
|
||||
|
||||
<BlockLabel {show_label} Icon={Video} label={label || "Video"} />
|
||||
{#if !value || value.url === undefined}
|
||||
<Empty unpadded_box={true} size="large"><Video /></Empty>
|
||||
{:else}
|
||||
{#key value.url}
|
||||
<Player
|
||||
src={value.url}
|
||||
subtitle={subtitle?.url}
|
||||
is_stream={value.is_stream}
|
||||
{autoplay}
|
||||
on:play
|
||||
on:pause
|
||||
on:stop
|
||||
on:end
|
||||
on:load
|
||||
mirror={false}
|
||||
{label}
|
||||
{loop}
|
||||
interactive={false}
|
||||
{upload}
|
||||
{i18n}
|
||||
/>
|
||||
{/key}
|
||||
<div data-testid="download-div">
|
||||
<IconButtonWrapper>
|
||||
{#if show_download_button}
|
||||
<DownloadLink
|
||||
href={value.is_stream
|
||||
? value.url?.replace("playlist.m3u8", "playlist-file")
|
||||
: value.url}
|
||||
download={value.orig_name || value.path}
|
||||
>
|
||||
<IconButton Icon={Download} label="Download" />
|
||||
</DownloadLink>
|
||||
{/if}
|
||||
{#if show_share_button}
|
||||
<ShareButton
|
||||
{i18n}
|
||||
on:error
|
||||
on:share
|
||||
{value}
|
||||
formatter={async (value) => {
|
||||
if (!value) return "";
|
||||
let url = await uploadToHuggingFace(value.data, "url");
|
||||
return url;
|
||||
}}
|
||||
/>
|
||||
{/if}
|
||||
</IconButtonWrapper>
|
||||
</div>
|
||||
{/if}
|
||||
279
frontend/shared/VideoTimeline.svelte
Normal file
279
frontend/shared/VideoTimeline.svelte
Normal file
@@ -0,0 +1,279 @@
|
||||
<script lang="ts">
|
||||
import { onMount, onDestroy } from "svelte";
|
||||
|
||||
export let videoElement: HTMLVideoElement;
|
||||
export let trimmedDuration: number | null;
|
||||
export let dragStart: number;
|
||||
export let dragEnd: number;
|
||||
export let loadingTimeline: boolean;
|
||||
|
||||
let thumbnails: string[] = [];
|
||||
let numberOfThumbnails = 10;
|
||||
let intervalId: ReturnType<typeof setInterval> | undefined;
|
||||
let videoDuration: number;
|
||||
|
||||
let leftHandlePosition = 0;
|
||||
let rightHandlePosition = 100;
|
||||
|
||||
let dragging: string | null = null;
|
||||
|
||||
const startDragging = (side: string | null): void => {
|
||||
dragging = side;
|
||||
};
|
||||
|
||||
$: loadingTimeline = thumbnails.length !== numberOfThumbnails;
|
||||
|
||||
const stopDragging = (): void => {
|
||||
dragging = null;
|
||||
};
|
||||
|
||||
const drag = (event: { clientX: number }, distance?: number): void => {
|
||||
if (dragging) {
|
||||
const timeline = document.getElementById("timeline");
|
||||
|
||||
if (!timeline) return;
|
||||
|
||||
const rect = timeline.getBoundingClientRect();
|
||||
let newPercentage = ((event.clientX - rect.left) / rect.width) * 100;
|
||||
|
||||
if (distance) {
|
||||
// Move handle based on arrow key press
|
||||
newPercentage =
|
||||
dragging === "left"
|
||||
? leftHandlePosition + distance
|
||||
: rightHandlePosition + distance;
|
||||
} else {
|
||||
// Move handle based on mouse drag
|
||||
newPercentage = ((event.clientX - rect.left) / rect.width) * 100;
|
||||
}
|
||||
|
||||
newPercentage = Math.max(0, Math.min(newPercentage, 100)); // Keep within 0 and 100
|
||||
|
||||
if (dragging === "left") {
|
||||
leftHandlePosition = Math.min(newPercentage, rightHandlePosition);
|
||||
|
||||
// Calculate the new time and set it for the videoElement
|
||||
const newTimeLeft = (leftHandlePosition / 100) * videoDuration;
|
||||
videoElement.currentTime = newTimeLeft;
|
||||
|
||||
dragStart = newTimeLeft;
|
||||
} else if (dragging === "right") {
|
||||
rightHandlePosition = Math.max(newPercentage, leftHandlePosition);
|
||||
|
||||
const newTimeRight = (rightHandlePosition / 100) * videoDuration;
|
||||
videoElement.currentTime = newTimeRight;
|
||||
|
||||
dragEnd = newTimeRight;
|
||||
}
|
||||
|
||||
const startTime = (leftHandlePosition / 100) * videoDuration;
|
||||
const endTime = (rightHandlePosition / 100) * videoDuration;
|
||||
trimmedDuration = endTime - startTime;
|
||||
|
||||
leftHandlePosition = leftHandlePosition;
|
||||
rightHandlePosition = rightHandlePosition;
|
||||
}
|
||||
};
|
||||
|
||||
const moveHandle = (e: KeyboardEvent): void => {
|
||||
if (dragging) {
|
||||
// Calculate the movement distance as a percentage of the video duration
|
||||
const distance = (1 / videoDuration) * 100;
|
||||
|
||||
if (e.key === "ArrowLeft") {
|
||||
drag({ clientX: 0 }, -distance);
|
||||
} else if (e.key === "ArrowRight") {
|
||||
drag({ clientX: 0 }, distance);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const generateThumbnail = (): void => {
|
||||
const canvas = document.createElement("canvas");
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) return;
|
||||
|
||||
canvas.width = videoElement.videoWidth;
|
||||
canvas.height = videoElement.videoHeight;
|
||||
|
||||
ctx.drawImage(videoElement, 0, 0, canvas.width, canvas.height);
|
||||
|
||||
const thumbnail: string = canvas.toDataURL("image/jpeg", 0.7);
|
||||
thumbnails = [...thumbnails, thumbnail];
|
||||
};
|
||||
|
||||
onMount(() => {
|
||||
const loadMetadata = (): void => {
|
||||
videoDuration = videoElement.duration;
|
||||
|
||||
const interval = videoDuration / numberOfThumbnails;
|
||||
let captures = 0;
|
||||
|
||||
const onSeeked = (): void => {
|
||||
generateThumbnail();
|
||||
captures++;
|
||||
|
||||
if (captures < numberOfThumbnails) {
|
||||
videoElement.currentTime += interval;
|
||||
} else {
|
||||
videoElement.removeEventListener("seeked", onSeeked);
|
||||
}
|
||||
};
|
||||
|
||||
videoElement.addEventListener("seeked", onSeeked);
|
||||
videoElement.currentTime = 0;
|
||||
};
|
||||
|
||||
if (videoElement.readyState >= 1) {
|
||||
loadMetadata();
|
||||
} else {
|
||||
videoElement.addEventListener("loadedmetadata", loadMetadata);
|
||||
}
|
||||
});
|
||||
|
||||
onDestroy(() => {
|
||||
window.removeEventListener("mousemove", drag);
|
||||
window.removeEventListener("mouseup", stopDragging);
|
||||
window.removeEventListener("keydown", moveHandle);
|
||||
|
||||
if (intervalId !== undefined) {
|
||||
clearInterval(intervalId);
|
||||
}
|
||||
});
|
||||
|
||||
onMount(() => {
|
||||
window.addEventListener("mousemove", drag);
|
||||
window.addEventListener("mouseup", stopDragging);
|
||||
window.addEventListener("keydown", moveHandle);
|
||||
});
|
||||
</script>
|
||||
|
||||
<div class="container">
|
||||
{#if loadingTimeline}
|
||||
<div class="load-wrap">
|
||||
<span aria-label="loading timeline" class="loader" />
|
||||
</div>
|
||||
{:else}
|
||||
<div id="timeline" class="thumbnail-wrapper">
|
||||
<button
|
||||
aria-label="start drag handle for trimming video"
|
||||
class="handle left"
|
||||
on:mousedown={() => startDragging("left")}
|
||||
on:blur={stopDragging}
|
||||
on:keydown={(e) => {
|
||||
if (e.key === "ArrowLeft" || e.key == "ArrowRight") {
|
||||
startDragging("left");
|
||||
}
|
||||
}}
|
||||
style="left: {leftHandlePosition}%;"
|
||||
/>
|
||||
|
||||
<div
|
||||
class="opaque-layer"
|
||||
style="left: {leftHandlePosition}%; right: {100 - rightHandlePosition}%"
|
||||
/>
|
||||
|
||||
{#each thumbnails as thumbnail, i (i)}
|
||||
<img src={thumbnail} alt={`frame-${i}`} draggable="false" />
|
||||
{/each}
|
||||
<button
|
||||
aria-label="end drag handle for trimming video"
|
||||
class="handle right"
|
||||
on:mousedown={() => startDragging("right")}
|
||||
on:blur={stopDragging}
|
||||
on:keydown={(e) => {
|
||||
if (e.key === "ArrowLeft" || e.key == "ArrowRight") {
|
||||
startDragging("right");
|
||||
}
|
||||
}}
|
||||
style="left: {rightHandlePosition}%;"
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.load-wrap {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
height: 100%;
|
||||
}
|
||||
.loader {
|
||||
display: flex;
|
||||
position: relative;
|
||||
background-color: var(--border-color-accent-subdued);
|
||||
animation: shadowPulse 2s linear infinite;
|
||||
box-shadow:
|
||||
-24px 0 var(--border-color-accent-subdued),
|
||||
24px 0 var(--border-color-accent-subdued);
|
||||
margin: var(--spacing-md);
|
||||
border-radius: 50%;
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
scale: 0.5;
|
||||
}
|
||||
|
||||
@keyframes shadowPulse {
|
||||
33% {
|
||||
box-shadow:
|
||||
-24px 0 var(--border-color-accent-subdued),
|
||||
24px 0 #fff;
|
||||
background: #fff;
|
||||
}
|
||||
66% {
|
||||
box-shadow:
|
||||
-24px 0 #fff,
|
||||
24px 0 #fff;
|
||||
background: var(--border-color-accent-subdued);
|
||||
}
|
||||
100% {
|
||||
box-shadow:
|
||||
-24px 0 #fff,
|
||||
24px 0 var(--border-color-accent-subdued);
|
||||
background: #fff;
|
||||
}
|
||||
}
|
||||
|
||||
.container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
margin: var(--spacing-lg) var(--spacing-lg) 0 var(--spacing-lg);
|
||||
}
|
||||
|
||||
#timeline {
|
||||
display: flex;
|
||||
height: var(--size-10);
|
||||
flex: 1;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
img {
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
object-fit: cover;
|
||||
height: var(--size-12);
|
||||
border: 1px solid var(--block-border-color);
|
||||
user-select: none;
|
||||
z-index: 1;
|
||||
}
|
||||
|
||||
.handle {
|
||||
width: 3px;
|
||||
background-color: var(--color-accent);
|
||||
cursor: ew-resize;
|
||||
height: var(--size-12);
|
||||
z-index: 3;
|
||||
position: absolute;
|
||||
}
|
||||
|
||||
.opaque-layer {
|
||||
background-color: rgba(230, 103, 40, 0.25);
|
||||
border: 1px solid var(--color-accent);
|
||||
height: var(--size-12);
|
||||
position: absolute;
|
||||
z-index: 2;
|
||||
}
|
||||
</style>
|
||||
402
frontend/shared/Webcam.svelte
Normal file
402
frontend/shared/Webcam.svelte
Normal file
@@ -0,0 +1,402 @@
|
||||
<script lang="ts">
|
||||
import { createEventDispatcher, onMount } from "svelte";
|
||||
import {
|
||||
Circle,
|
||||
Square,
|
||||
DropdownArrow,
|
||||
Spinner
|
||||
} from "@gradio/icons";
|
||||
import type { I18nFormatter } from "@gradio/utils";
|
||||
import { StreamingBar } from "@gradio/statustracker";
|
||||
import WebcamPermissions from "./WebcamPermissions.svelte";
|
||||
import { fade } from "svelte/transition";
|
||||
import {
|
||||
get_devices,
|
||||
get_video_stream,
|
||||
set_available_devices
|
||||
} from "./stream_utils";
|
||||
|
||||
import { start, stop } from "./webrtc_utils";
|
||||
|
||||
let video_source: HTMLVideoElement;
|
||||
let available_video_devices: MediaDeviceInfo[] = [];
|
||||
let selected_device: MediaDeviceInfo | null = null;
|
||||
let time_limit: number | null = null;
|
||||
let stream_state: "open" | "waiting" | "closed" = "closed";
|
||||
export const webrtc_id = Math.random().toString(36).substring(2);
|
||||
|
||||
export const modify_stream: (state: "open" | "closed" | "waiting") => void = (
|
||||
state: "open" | "closed" | "waiting"
|
||||
) => {
|
||||
if (state === "closed") {
|
||||
time_limit = null;
|
||||
stream_state = "closed";
|
||||
} else if (state === "waiting") {
|
||||
stream_state = "waiting";
|
||||
} else {
|
||||
stream_state = "open";
|
||||
}
|
||||
};
|
||||
|
||||
export const set_time_limit = (time: number): void => {
|
||||
if (recording) time_limit = time;
|
||||
};
|
||||
|
||||
let canvas: HTMLCanvasElement;
|
||||
export let pending = false;
|
||||
export let root = "";
|
||||
export let stream_every = 1;
|
||||
export let server: {
|
||||
offer: (body: any) => Promise<any>;
|
||||
};
|
||||
|
||||
export let include_audio: boolean;
|
||||
export let i18n: I18nFormatter;
|
||||
|
||||
const dispatch = createEventDispatcher<{
|
||||
tick: undefined;
|
||||
error: string;
|
||||
start_recording: undefined;
|
||||
stop_recording: undefined;
|
||||
close_stream: undefined;
|
||||
}>();
|
||||
|
||||
onMount(() => (canvas = document.createElement("canvas")));
|
||||
|
||||
const handle_device_change = async (event: InputEvent): Promise<void> => {
|
||||
const target = event.target as HTMLInputElement;
|
||||
const device_id = target.value;
|
||||
|
||||
await get_video_stream(include_audio, video_source, device_id).then(
|
||||
async (local_stream) => {
|
||||
stream = local_stream;
|
||||
selected_device =
|
||||
available_video_devices.find(
|
||||
(device) => device.deviceId === device_id
|
||||
) || null;
|
||||
options_open = false;
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
async function access_webcam(): Promise<void> {
|
||||
try {
|
||||
get_video_stream(include_audio, video_source)
|
||||
.then(async (local_stream) => {
|
||||
webcam_accessed = true;
|
||||
available_video_devices = await get_devices();
|
||||
stream = local_stream;
|
||||
})
|
||||
.then(() => set_available_devices(available_video_devices))
|
||||
.then((devices) => {
|
||||
available_video_devices = devices;
|
||||
|
||||
const used_devices = stream
|
||||
.getTracks()
|
||||
.map((track) => track.getSettings()?.deviceId)[0];
|
||||
|
||||
selected_device = used_devices
|
||||
? devices.find((device) => device.deviceId === used_devices) ||
|
||||
available_video_devices[0]
|
||||
: available_video_devices[0];
|
||||
});
|
||||
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
dispatch("error", i18n("image.no_webcam_support"));
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof DOMException && err.name == "NotAllowedError") {
|
||||
dispatch("error", i18n("image.allow_webcam_access"));
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let recording = false;
|
||||
let stream: MediaStream;
|
||||
|
||||
let webcam_accessed = false;
|
||||
let pc: RTCPeerConnection;
|
||||
|
||||
async function start_webrtc(): Promise<void> {
|
||||
if (stream_state === 'closed') {
|
||||
pc = new RTCPeerConnection();
|
||||
pc.addEventListener("connectionstatechange",
|
||||
(event) => {
|
||||
switch(pc.connectionState) {
|
||||
case "connected":
|
||||
stream_state = "open"
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
)
|
||||
stream_state = "waiting"
|
||||
start(stream, pc, video_source, server.offer, webrtc_id).then((connection) => {
|
||||
pc = connection;
|
||||
}).catch(() => {
|
||||
console.log("catching")
|
||||
stream_state = "closed";
|
||||
dispatch("error", "Too many concurrent users. Come back later!");
|
||||
});
|
||||
} else {
|
||||
stop(pc);
|
||||
stream_state = "closed";
|
||||
await access_webcam();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
window.setInterval(() => {
|
||||
if (stream_state == "open") {
|
||||
console.log("dispatching tick");
|
||||
dispatch("tick");
|
||||
}
|
||||
}, stream_every * 1000);
|
||||
|
||||
let options_open = false;
|
||||
|
||||
export function click_outside(node: Node, cb: any): any {
|
||||
const handle_click = (event: MouseEvent): void => {
|
||||
if (
|
||||
node &&
|
||||
!node.contains(event.target as Node) &&
|
||||
!event.defaultPrevented
|
||||
) {
|
||||
cb(event);
|
||||
}
|
||||
};
|
||||
|
||||
document.addEventListener("click", handle_click, true);
|
||||
|
||||
return {
|
||||
destroy() {
|
||||
document.removeEventListener("click", handle_click, true);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function handle_click_outside(event: MouseEvent): void {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
options_open = false;
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="wrap">
|
||||
<StreamingBar {time_limit} />
|
||||
<!-- svelte-ignore a11y-media-has-caption -->
|
||||
<!-- need to suppress for video streaming https://github.com/sveltejs/svelte/issues/5967 -->
|
||||
<video
|
||||
bind:this={video_source}
|
||||
class:hide={!webcam_accessed}
|
||||
class:flip={(stream_state != "open")}
|
||||
autoplay={true}
|
||||
playsinline={true}
|
||||
/>
|
||||
<!-- svelte-ignore a11y-missing-attribute -->
|
||||
{#if !webcam_accessed}
|
||||
<div
|
||||
in:fade={{ delay: 100, duration: 200 }}
|
||||
title="grant webcam access"
|
||||
style="height: 100%"
|
||||
>
|
||||
<WebcamPermissions on:click={async () => access_webcam()} />
|
||||
</div>
|
||||
{:else}
|
||||
<div class="button-wrap">
|
||||
<button
|
||||
on:click={start_webrtc}
|
||||
aria-label={"start stream"}
|
||||
>
|
||||
{#if stream_state === "waiting"}
|
||||
<div class="icon-with-text" style="width:var(--size-24);">
|
||||
<div class="icon color-primary" title="spinner">
|
||||
<Spinner />
|
||||
</div>
|
||||
{i18n("audio.waiting")}
|
||||
</div>
|
||||
{:else if stream_state === "open"}
|
||||
<div class="icon-with-text">
|
||||
<div class="icon color-primary" title="stop recording">
|
||||
<Square />
|
||||
</div>
|
||||
{i18n("audio.stop")}
|
||||
</div>
|
||||
{:else}
|
||||
<div class="icon-with-text">
|
||||
<div class="icon color-primary" title="start recording">
|
||||
<Circle />
|
||||
</div>
|
||||
{i18n("audio.record")}
|
||||
</div>
|
||||
{/if}
|
||||
</button>
|
||||
{#if !recording}
|
||||
<button
|
||||
class="icon"
|
||||
on:click={() => (options_open = true)}
|
||||
aria-label="select input source"
|
||||
>
|
||||
<DropdownArrow />
|
||||
</button>
|
||||
{/if}
|
||||
</div>
|
||||
{#if options_open && selected_device}
|
||||
<select
|
||||
class="select-wrap"
|
||||
aria-label="select source"
|
||||
use:click_outside={handle_click_outside}
|
||||
on:change={handle_device_change}
|
||||
>
|
||||
<button
|
||||
class="inset-icon"
|
||||
on:click|stopPropagation={() => (options_open = false)}
|
||||
>
|
||||
<DropdownArrow />
|
||||
</button>
|
||||
{#if available_video_devices.length === 0}
|
||||
<option value="">{i18n("common.no_devices")}</option>
|
||||
{:else}
|
||||
{#each available_video_devices as device}
|
||||
<option
|
||||
value={device.deviceId}
|
||||
selected={selected_device.deviceId === device.deviceId}
|
||||
>
|
||||
{device.label}
|
||||
</option>
|
||||
{/each}
|
||||
{/if}
|
||||
</select>
|
||||
{/if}
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.wrap {
|
||||
position: relative;
|
||||
width: var(--size-full);
|
||||
height: var(--size-full);
|
||||
}
|
||||
|
||||
.hide {
|
||||
display: none;
|
||||
}
|
||||
|
||||
video {
|
||||
width: var(--size-full);
|
||||
height: var(--size-full);
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.button-wrap {
|
||||
position: absolute;
|
||||
background-color: var(--block-background-fill);
|
||||
border: 1px solid var(--border-color-primary);
|
||||
border-radius: var(--radius-xl);
|
||||
padding: var(--size-1-5);
|
||||
display: flex;
|
||||
bottom: var(--size-2);
|
||||
left: 50%;
|
||||
transform: translate(-50%, 0);
|
||||
box-shadow: var(--shadow-drop-lg);
|
||||
border-radius: var(--radius-xl);
|
||||
line-height: var(--size-3);
|
||||
color: var(--button-secondary-text-color);
|
||||
}
|
||||
|
||||
.icon-with-text {
|
||||
width: var(--size-20);
|
||||
align-items: center;
|
||||
margin: 0 var(--spacing-xl);
|
||||
display: flex;
|
||||
justify-content: space-evenly;
|
||||
}
|
||||
|
||||
@media (--screen-md) {
|
||||
button {
|
||||
bottom: var(--size-4);
|
||||
}
|
||||
}
|
||||
|
||||
@media (--screen-xl) {
|
||||
button {
|
||||
bottom: var(--size-8);
|
||||
}
|
||||
}
|
||||
|
||||
.icon {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.color-primary {
|
||||
fill: var(--primary-600);
|
||||
stroke: var(--primary-600);
|
||||
color: var(--primary-600);
|
||||
}
|
||||
|
||||
.flip {
|
||||
transform: scaleX(-1);
|
||||
}
|
||||
|
||||
.select-wrap {
|
||||
-webkit-appearance: none;
|
||||
-moz-appearance: none;
|
||||
appearance: none;
|
||||
color: var(--button-secondary-text-color);
|
||||
background-color: transparent;
|
||||
width: 95%;
|
||||
font-size: var(--text-md);
|
||||
position: absolute;
|
||||
bottom: var(--size-2);
|
||||
background-color: var(--block-background-fill);
|
||||
box-shadow: var(--shadow-drop-lg);
|
||||
border-radius: var(--radius-xl);
|
||||
z-index: var(--layer-top);
|
||||
border: 1px solid var(--border-color-primary);
|
||||
text-align: left;
|
||||
line-height: var(--size-4);
|
||||
white-space: nowrap;
|
||||
text-overflow: ellipsis;
|
||||
left: 50%;
|
||||
transform: translate(-50%, 0);
|
||||
max-width: var(--size-52);
|
||||
}
|
||||
|
||||
.select-wrap > option {
|
||||
padding: 0.25rem 0.5rem;
|
||||
border-bottom: 1px solid var(--border-color-accent);
|
||||
padding-right: var(--size-8);
|
||||
text-overflow: ellipsis;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.select-wrap > option:hover {
|
||||
background-color: var(--color-accent);
|
||||
}
|
||||
|
||||
.select-wrap > option:last-child {
|
||||
border: none;
|
||||
}
|
||||
|
||||
.inset-icon {
|
||||
position: absolute;
|
||||
top: 5px;
|
||||
right: -6.5px;
|
||||
width: var(--size-10);
|
||||
height: var(--size-5);
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
@media (--screen-md) {
|
||||
.wrap {
|
||||
font-size: var(--text-lg);
|
||||
}
|
||||
}
|
||||
</style>
|
||||
46
frontend/shared/WebcamPermissions.svelte
Normal file
46
frontend/shared/WebcamPermissions.svelte
Normal file
@@ -0,0 +1,46 @@
|
||||
<script lang="ts">
|
||||
import { Webcam } from "@gradio/icons";
|
||||
import { createEventDispatcher } from "svelte";
|
||||
|
||||
const dispatch = createEventDispatcher<{
|
||||
click: undefined;
|
||||
}>();
|
||||
</script>
|
||||
|
||||
<button style:height="100%" on:click={() => dispatch("click")}>
|
||||
<div class="wrap">
|
||||
<span class="icon-wrap">
|
||||
<Webcam />
|
||||
</span>
|
||||
{"Click to Access Webcam"}
|
||||
</div>
|
||||
</button>
|
||||
|
||||
<style>
|
||||
button {
|
||||
cursor: pointer;
|
||||
width: var(--size-full);
|
||||
}
|
||||
|
||||
.wrap {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
min-height: var(--size-60);
|
||||
color: var(--block-label-text-color);
|
||||
height: 100%;
|
||||
padding-top: var(--size-3);
|
||||
}
|
||||
|
||||
.icon-wrap {
|
||||
width: 30px;
|
||||
margin-bottom: var(--spacing-lg);
|
||||
}
|
||||
|
||||
@media (--screen-md) {
|
||||
.wrap {
|
||||
font-size: var(--text-lg);
|
||||
}
|
||||
}
|
||||
</style>
|
||||
1
frontend/shared/index.ts
Normal file
1
frontend/shared/index.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { default as Video } from "./Video.svelte";
|
||||
49
frontend/shared/stream_utils.ts
Normal file
49
frontend/shared/stream_utils.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
export function get_devices(): Promise<MediaDeviceInfo[]> {
|
||||
return navigator.mediaDevices.enumerateDevices();
|
||||
}
|
||||
|
||||
export function handle_error(error: string): void {
|
||||
throw new Error(error);
|
||||
}
|
||||
|
||||
export function set_local_stream(
|
||||
local_stream: MediaStream | null,
|
||||
video_source: HTMLVideoElement
|
||||
): void {
|
||||
video_source.srcObject = local_stream;
|
||||
video_source.muted = true;
|
||||
video_source.play();
|
||||
}
|
||||
|
||||
export async function get_video_stream(
|
||||
include_audio: boolean,
|
||||
video_source: HTMLVideoElement,
|
||||
device_id?: string
|
||||
): Promise<MediaStream> {
|
||||
const size = {
|
||||
width: { ideal: 1920 },
|
||||
height: { ideal: 1440 }
|
||||
};
|
||||
|
||||
const constraints = {
|
||||
video: device_id ? { deviceId: { exact: device_id }, ...size } : size,
|
||||
audio: include_audio
|
||||
};
|
||||
|
||||
return navigator.mediaDevices
|
||||
.getUserMedia(constraints)
|
||||
.then((local_stream: MediaStream) => {
|
||||
set_local_stream(local_stream, video_source);
|
||||
return local_stream;
|
||||
});
|
||||
}
|
||||
|
||||
export function set_available_devices(
|
||||
devices: MediaDeviceInfo[]
|
||||
): MediaDeviceInfo[] {
|
||||
const cameras = devices.filter(
|
||||
(device: MediaDeviceInfo) => device.kind === "videoinput"
|
||||
);
|
||||
|
||||
return cameras;
|
||||
}
|
||||
146
frontend/shared/utils.ts
Normal file
146
frontend/shared/utils.ts
Normal file
@@ -0,0 +1,146 @@
|
||||
import { toBlobURL } from "@ffmpeg/util";
|
||||
import { FFmpeg } from "@ffmpeg/ffmpeg";
|
||||
import { lookup } from "mrmime";
|
||||
|
||||
export const prettyBytes = (bytes: number): string => {
|
||||
let units = ["B", "KB", "MB", "GB", "PB"];
|
||||
let i = 0;
|
||||
while (bytes > 1024) {
|
||||
bytes /= 1024;
|
||||
i++;
|
||||
}
|
||||
let unit = units[i];
|
||||
return bytes.toFixed(1) + " " + unit;
|
||||
};
|
||||
|
||||
export const playable = (): boolean => {
|
||||
// TODO: Fix this
|
||||
// let video_element = document.createElement("video");
|
||||
// let mime_type = mime.lookup(filename);
|
||||
// return video_element.canPlayType(mime_type) != "";
|
||||
return true; // FIX BEFORE COMMIT - mime import causing issues
|
||||
};
|
||||
|
||||
export function loaded(
|
||||
node: HTMLVideoElement,
|
||||
{ autoplay }: { autoplay: boolean }
|
||||
): any {
|
||||
async function handle_playback(): Promise<void> {
|
||||
if (!autoplay) return;
|
||||
await node.play();
|
||||
}
|
||||
|
||||
node.addEventListener("loadeddata", handle_playback);
|
||||
|
||||
return {
|
||||
destroy(): void {
|
||||
node.removeEventListener("loadeddata", handle_playback);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export default async function loadFfmpeg(): Promise<FFmpeg> {
|
||||
const ffmpeg = new FFmpeg();
|
||||
const baseURL = "https://unpkg.com/@ffmpeg/core@0.12.4/dist/esm";
|
||||
|
||||
await ffmpeg.load({
|
||||
coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, "text/javascript"),
|
||||
wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, "application/wasm")
|
||||
});
|
||||
|
||||
return ffmpeg;
|
||||
}
|
||||
|
||||
export function blob_to_data_url(blob: Blob): Promise<string> {
|
||||
return new Promise((fulfill, reject) => {
|
||||
let reader = new FileReader();
|
||||
reader.onerror = reject;
|
||||
reader.onload = () => fulfill(reader.result as string);
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
}
|
||||
|
||||
export async function trimVideo(
|
||||
ffmpeg: FFmpeg,
|
||||
startTime: number,
|
||||
endTime: number,
|
||||
videoElement: HTMLVideoElement
|
||||
): Promise<any> {
|
||||
const videoUrl = videoElement.src;
|
||||
const mimeType = lookup(videoElement.src) || "video/mp4";
|
||||
const blobUrl = await toBlobURL(videoUrl, mimeType);
|
||||
const response = await fetch(blobUrl);
|
||||
const vidBlob = await response.blob();
|
||||
const type = getVideoExtensionFromMimeType(mimeType) || "mp4";
|
||||
const inputName = `input.${type}`;
|
||||
const outputName = `output.${type}`;
|
||||
|
||||
try {
|
||||
if (startTime === 0 && endTime === 0) {
|
||||
return vidBlob;
|
||||
}
|
||||
|
||||
await ffmpeg.writeFile(
|
||||
inputName,
|
||||
new Uint8Array(await vidBlob.arrayBuffer())
|
||||
);
|
||||
|
||||
let command = [
|
||||
"-i",
|
||||
inputName,
|
||||
...(startTime !== 0 ? ["-ss", startTime.toString()] : []),
|
||||
...(endTime !== 0 ? ["-to", endTime.toString()] : []),
|
||||
"-c:a",
|
||||
"copy",
|
||||
outputName
|
||||
];
|
||||
|
||||
await ffmpeg.exec(command);
|
||||
const outputData = await ffmpeg.readFile(outputName);
|
||||
const outputBlob = new Blob([outputData], {
|
||||
type: `video/${type}`
|
||||
});
|
||||
|
||||
return outputBlob;
|
||||
} catch (error) {
|
||||
console.error("Error initializing FFmpeg:", error);
|
||||
return vidBlob;
|
||||
}
|
||||
}
|
||||
|
||||
const getVideoExtensionFromMimeType = (mimeType: string): string | null => {
|
||||
const videoMimeToExtensionMap: { [key: string]: string } = {
|
||||
"video/mp4": "mp4",
|
||||
"video/webm": "webm",
|
||||
"video/ogg": "ogv",
|
||||
"video/quicktime": "mov",
|
||||
"video/x-msvideo": "avi",
|
||||
"video/x-matroska": "mkv",
|
||||
"video/mpeg": "mpeg",
|
||||
"video/3gpp": "3gp",
|
||||
"video/3gpp2": "3g2",
|
||||
"video/h261": "h261",
|
||||
"video/h263": "h263",
|
||||
"video/h264": "h264",
|
||||
"video/jpeg": "jpgv",
|
||||
"video/jpm": "jpm",
|
||||
"video/mj2": "mj2",
|
||||
"video/mpv": "mpv",
|
||||
"video/vnd.ms-playready.media.pyv": "pyv",
|
||||
"video/vnd.uvvu.mp4": "uvu",
|
||||
"video/vnd.vivo": "viv",
|
||||
"video/x-f4v": "f4v",
|
||||
"video/x-fli": "fli",
|
||||
"video/x-flv": "flv",
|
||||
"video/x-m4v": "m4v",
|
||||
"video/x-ms-asf": "asf",
|
||||
"video/x-ms-wm": "wm",
|
||||
"video/x-ms-wmv": "wmv",
|
||||
"video/x-ms-wmx": "wmx",
|
||||
"video/x-ms-wvx": "wvx",
|
||||
"video/x-sgi-movie": "movie",
|
||||
"video/x-smv": "smv"
|
||||
};
|
||||
|
||||
return videoMimeToExtensionMap[mimeType] || null;
|
||||
};
|
||||
141
frontend/shared/webrtc_utils.ts
Normal file
141
frontend/shared/webrtc_utils.ts
Normal file
@@ -0,0 +1,141 @@
|
||||
export function createPeerConnection(pc, node) {
|
||||
// register some listeners to help debugging
|
||||
pc.addEventListener(
|
||||
"icegatheringstatechange",
|
||||
() => {
|
||||
console.log(pc.iceGatheringState);
|
||||
},
|
||||
false
|
||||
);
|
||||
|
||||
pc.addEventListener(
|
||||
"iceconnectionstatechange",
|
||||
() => {
|
||||
console.log(pc.iceConnectionState);
|
||||
},
|
||||
false
|
||||
);
|
||||
|
||||
pc.addEventListener(
|
||||
"signalingstatechange",
|
||||
() => {
|
||||
console.log(pc.signalingState);
|
||||
},
|
||||
false
|
||||
);
|
||||
|
||||
// connect audio / video from server to local
|
||||
pc.addEventListener("track", (evt) => {
|
||||
console.log("track event listener");
|
||||
if (evt.track.kind == "video") {
|
||||
console.log("streams", evt.streams);
|
||||
node.srcObject = evt.streams[0];
|
||||
console.log("node.srcOject", node.srcObject);
|
||||
}
|
||||
});
|
||||
|
||||
return pc;
|
||||
}
|
||||
|
||||
export async function start(stream, pc, node, server_fn, webrtc_id) {
|
||||
pc = createPeerConnection(pc, node);
|
||||
if (stream) {
|
||||
stream.getTracks().forEach((track) => {
|
||||
track.applyConstraints({ frameRate: { max: 30 } });
|
||||
|
||||
console.log("Track stream callback", track);
|
||||
pc.addTrack(track, stream);
|
||||
});
|
||||
} else {
|
||||
console.log("Creating transceiver!");
|
||||
pc.addTransceiver("video", { direction: "recvonly" });
|
||||
}
|
||||
|
||||
await negotiate(pc, server_fn, webrtc_id);
|
||||
return pc;
|
||||
}
|
||||
|
||||
function make_offer(server_fn: any, body): Promise<object> {
|
||||
return new Promise((resolve, reject) => {
|
||||
server_fn(body).then((data) => {
|
||||
console.log("data", data)
|
||||
if(data?.status === "failed") {
|
||||
console.log("rejecting")
|
||||
reject("error")
|
||||
}
|
||||
resolve(data);
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function negotiate(
|
||||
pc: RTCPeerConnection,
|
||||
server_fn: any,
|
||||
webrtc_id: string,
|
||||
): Promise<void> {
|
||||
return pc
|
||||
.createOffer()
|
||||
.then((offer) => {
|
||||
return pc.setLocalDescription(offer);
|
||||
})
|
||||
.then(() => {
|
||||
// wait for ICE gathering to complete
|
||||
return new Promise<void>((resolve) => {
|
||||
console.log("ice gathering state", pc.iceGatheringState);
|
||||
if (pc.iceGatheringState === "complete") {
|
||||
resolve();
|
||||
} else {
|
||||
const checkState = () => {
|
||||
if (pc.iceGatheringState === "complete") {
|
||||
console.log("ice complete");
|
||||
pc.removeEventListener("icegatheringstatechange", checkState);
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
pc.addEventListener("icegatheringstatechange", checkState);
|
||||
}
|
||||
});
|
||||
})
|
||||
.then(() => {
|
||||
var offer = pc.localDescription;
|
||||
return make_offer(
|
||||
server_fn,
|
||||
{
|
||||
sdp: offer.sdp,
|
||||
type: offer.type,
|
||||
webrtc_id: webrtc_id
|
||||
},
|
||||
);
|
||||
})
|
||||
.then((response) => {
|
||||
return response;
|
||||
})
|
||||
.then((answer) => {
|
||||
return pc.setRemoteDescription(answer);
|
||||
})
|
||||
}
|
||||
|
||||
export function stop(pc: RTCPeerConnection) {
|
||||
console.log("pc", pc);
|
||||
console.log("STOPPING");
|
||||
// close transceivers
|
||||
if (pc.getTransceivers) {
|
||||
pc.getTransceivers().forEach((transceiver) => {
|
||||
if (transceiver.stop) {
|
||||
transceiver.stop();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// close local audio / video
|
||||
if (pc.getSenders()) {
|
||||
pc.getSenders().forEach((sender) => {
|
||||
if (sender.track && sender.track.stop) sender.track.stop();
|
||||
});
|
||||
}
|
||||
|
||||
// close peer connection
|
||||
setTimeout(() => {
|
||||
pc.close();
|
||||
}, 500);
|
||||
}
|
||||
Reference in New Issue
Block a user