mirror of
https://github.com/HumanAIGC-Engineering/gradio-webrtc.git
synced 2026-02-04 09:29:23 +08:00
1559 lines
49 KiB
HTML
1559 lines
49 KiB
HTML
|
|
<!doctype html>
|
|
<html lang="en" class="no-js">
|
|
<head>
|
|
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
|
|
|
|
|
|
<link rel="canonical" href="https://fastrtc.org/userguide/api/">
|
|
|
|
|
|
<link rel="prev" href="../gradio/">
|
|
|
|
|
|
<link rel="next" href="../../cookbook/">
|
|
|
|
|
|
<link rel="icon" href="../../fastrtc_logo.png">
|
|
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.6.14">
|
|
|
|
|
|
|
|
<title>API - FastRTC</title>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/stylesheets/main.342714a4.min.css">
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
|
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../../stylesheets/extra.css">
|
|
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
|
|
|
|
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</head>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<body dir="ltr" data-md-color-scheme="fastrtc-dark" data-md-color-primary="indigo" data-md-color-accent="indigo">
|
|
|
|
|
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
|
<label class="md-overlay" for="__drawer"></label>
|
|
<div data-md-component="skip">
|
|
|
|
|
|
<a href="#connecting-via-api" class="md-skip">
|
|
Skip to content
|
|
</a>
|
|
|
|
</div>
|
|
<div data-md-component="announce">
|
|
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<header class="md-header md-header--shadow" data-md-component="header">
|
|
<nav class="md-header__inner md-grid" aria-label="Header">
|
|
<a href="../.." title="FastRTC"
|
|
class="md-header__button md-logo" aria-label="FastRTC" data-md-component="logo">
|
|
|
|
<img src="../../fastrtc_logo.png" alt="logo">
|
|
|
|
</a>
|
|
<label class="md-header__button md-icon" for="__drawer">
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
|
</label>
|
|
<div class="md-header__title" data-md-component="header-title">
|
|
<div class="md-header__ellipsis">
|
|
<div class="md-header__topic">
|
|
<span class="md-ellipsis">
|
|
FastRTC
|
|
</span>
|
|
</div>
|
|
<div class="md-header__topic" data-md-component="header-topic">
|
|
<span class="md-ellipsis">
|
|
|
|
API
|
|
|
|
</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div style="display: flex; align-items: center; margin-right: 1rem;">
|
|
<a href="https://hf.co/fastrtc" target="_blank" rel="noopener noreferrer">
|
|
<img src="/hf-logo.svg"
|
|
onerror="this.onerror=null; this.src='https://huggingface.co/datasets/freddyaboulton/bucket/resolve/main/hf-logo.svg';"
|
|
style="height: 24px; margin-right: 10px;">
|
|
</a>
|
|
<a href="https://gradio.app" target="_blank" rel="noopener noreferrer">
|
|
<img src="/gradio-logo.svg"
|
|
onerror="this.onerror=null; this.src='https://huggingface.co/datasets/freddyaboulton/bucket/resolve/main/gradio-logo.svg';"
|
|
style="height: 24px; margin-right: 10px;">
|
|
</a>
|
|
<a href="https://discord.gg/TSWU7HyaYu" target="_blank" rel="noopener noreferrer">
|
|
<img src="/Discord-Symbol-White.svg" style="height: 16px; margin-right: 10px;">
|
|
</a>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-header__source">
|
|
<a href="https://github.com/gradio-app/fastrtc" title="Go to repository" class="md-source" data-md-component="source">
|
|
<div class="md-source__icon md-icon">
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81"/></svg>
|
|
</div>
|
|
<div class="md-source__repository">
|
|
fastrtc
|
|
</div>
|
|
</a>
|
|
</div>
|
|
|
|
</nav>
|
|
|
|
</header>
|
|
|
|
<div class="md-container" data-md-component="container">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<main class="md-main" data-md-component="main">
|
|
<div class="md-main__inner md-grid">
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
|
|
<label class="md-nav__title" for="__drawer">
|
|
<a href="../.." title="FastRTC" class="md-nav__button md-logo" aria-label="FastRTC" data-md-component="logo">
|
|
|
|
<img src="../../fastrtc_logo.png" alt="logo">
|
|
|
|
</a>
|
|
FastRTC
|
|
</label>
|
|
|
|
<div class="md-nav__source">
|
|
<a href="https://github.com/gradio-app/fastrtc" title="Go to repository" class="md-source" data-md-component="source">
|
|
<div class="md-source__icon md-icon">
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81"/></svg>
|
|
</div>
|
|
<div class="md-source__repository">
|
|
fastrtc
|
|
</div>
|
|
</a>
|
|
</div>
|
|
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../.." class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Home
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" checked>
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
User Guide
|
|
|
|
</span>
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="true">
|
|
<label class="md-nav__title" for="__nav_2">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
User Guide
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../streams/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Core Concepts
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../audio/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Audio Streaming
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../video/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Video Streaming
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../audio-video/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Audio-Video Streaming
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../gradio/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Gradio
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active">
|
|
|
|
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__link md-nav__link--active" for="__toc">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
API
|
|
|
|
</span>
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<a href="./" class="md-nav__link md-nav__link--active">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
API
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#sample-code" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
Sample Code
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#message-format" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
Message Format
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#additional-inputs" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
Additional Inputs
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#additional-outputs" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
Additional Outputs
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#handling-errors" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
Handling Errors
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../cookbook/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Cookbook
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../deployment/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Deployment
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../advanced-configuration/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Advanced Configuration
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" >
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Plugin Ecosystem
|
|
|
|
</span>
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
|
|
<label class="md-nav__title" for="__nav_6">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Plugin Ecosystem
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../text_to_speech_gallery/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Text-to-Speech Gallery
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../speech_to_text_gallery/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Speech-to-Text Gallery
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../turn_taking_gallery/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Turn-taking Gallery
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../utils/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Utils
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../faq/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Frequently Asked Questions
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
|
|
|
|
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_9" >
|
|
|
|
|
|
<label class="md-nav__link" for="__nav_9" id="__nav_9_label" tabindex="0">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
API Reference
|
|
|
|
</span>
|
|
|
|
|
|
<span class="md-nav__icon md-icon"></span>
|
|
</label>
|
|
|
|
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
|
|
<label class="md-nav__title" for="__nav_9">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
API Reference
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../reference/stream/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Stream
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../reference/reply_on_pause/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Pause Detection Handlers
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../reference/stream_handlers/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Stream Handlers
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../reference/utils/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
Utils
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../reference/credentials/" class="md-nav__link">
|
|
|
|
|
|
|
|
<span class="md-ellipsis">
|
|
TURN Credentials
|
|
|
|
</span>
|
|
|
|
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">
|
|
<span class="md-nav__icon md-icon"></span>
|
|
Table of contents
|
|
</label>
|
|
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#sample-code" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
Sample Code
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#message-format" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
Message Format
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#additional-inputs" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
Additional Inputs
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#additional-outputs" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
Additional Outputs
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#handling-errors" class="md-nav__link">
|
|
<span class="md-ellipsis">
|
|
Handling Errors
|
|
</span>
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div class="md-content" data-md-component="content">
|
|
<article class="md-content__inner md-typeset">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<h1 id="connecting-via-api">Connecting via API</h1>
|
|
<p>Before continuing, select the <code>modality</code>, <code>mode</code> of your <code>Stream</code> and whether you're using <code>WebRTC</code> or <code>WebSocket</code>s.</p>
|
|
<div class="config-selector">
|
|
<div class="select-group">
|
|
<label for="connection">Connection</label>
|
|
<select id="connection" onchange="updateDocs()">
|
|
<option value="webrtc">WebRTC</option>
|
|
<option value="websocket">WebSocket</option>
|
|
</select>
|
|
</div>
|
|
<div class="select-group">
|
|
<label for="modality">Modality</label>
|
|
<select id="modality" onchange="updateDocs()">
|
|
<option value="audio">Audio</option>
|
|
<option value="video">Video</option>
|
|
<option value="audio-video">Audio-Video</option>
|
|
</select>
|
|
</div>
|
|
<div class="select-group">
|
|
<label for="mode">Mode</label>
|
|
<select id="mode" onchange="updateDocs()">
|
|
<option value="send-receive">Send-Receive</option>
|
|
<option value="receive">Receive</option>
|
|
<option value="send">Send</option>
|
|
</select>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<h3 id="sample-code">Sample Code</h3>
|
|
<div id="docs"></div>
|
|
|
|
<h3 id="message-format">Message Format</h3>
|
|
<p>Over both WebRTC and WebSocket, the server can send messages of the following format:</p>
|
|
<div class="language-json highlight"><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="p">{</span>
|
|
</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="w"> </span><span class="nt">"type"</span><span class="p">:</span><span class="w"> </span><span class="err">`se</span><span class="kc">n</span><span class="err">d_i</span><span class="kc">n</span><span class="err">pu</span><span class="kc">t</span><span class="err">`</span><span class="w"> </span><span class="err">|</span><span class="w"> </span><span class="err">`</span><span class="kc">fet</span><span class="err">ch_ou</span><span class="kc">t</span><span class="err">pu</span><span class="kc">t</span><span class="err">`</span><span class="w"> </span><span class="err">|</span><span class="w"> </span><span class="err">`s</span><span class="kc">t</span><span class="err">opword`</span><span class="w"> </span><span class="err">|</span><span class="w"> </span><span class="err">`error`</span><span class="w"> </span><span class="err">|</span><span class="w"> </span><span class="err">`war</span><span class="kc">n</span><span class="err">i</span><span class="kc">n</span><span class="err">g`</span><span class="w"> </span><span class="err">|</span><span class="w"> </span><span class="err">`log`</span><span class="p">,</span>
|
|
</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="w"> </span><span class="nt">"data"</span><span class="p">:</span><span class="w"> </span><span class="err">s</span><span class="kc">tr</span><span class="err">i</span><span class="kc">n</span><span class="err">g</span><span class="w"> </span><span class="err">|</span><span class="w"> </span><span class="err">objec</span><span class="kc">t</span>
|
|
</span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="p">}</span>
|
|
</span></code></pre></div>
|
|
<ul>
|
|
<li><code>send_input</code>: Send any input data for the handler to the server. See <a href="#additional-inputs"><code>Additional Inputs</code></a> for more details.</li>
|
|
<li><code>fetch_output</code>: An instance of <a href="#additional-outputs"><code>AdditionalOutputs</code></a> is sent to the server.</li>
|
|
<li><code>stopword</code>: The stopword has been detected. See <a href="../audio/#reply-on-stopwords"><code>ReplyOnStopWords</code></a> for more details.</li>
|
|
<li><code>error</code>: An error occurred. The <code>data</code> will be a string containing the error message.</li>
|
|
<li><code>warning</code>: A warning occurred. The <code>data</code> will be a string containing the warning message.</li>
|
|
<li><code>log</code>: A log message. The <code>data</code> will be a string containing the log message.</li>
|
|
</ul>
|
|
<p>The <code>ReplyOnPause</code> handler can also send the following <code>log</code> messages.</p>
|
|
<div class="language-json highlight"><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="p">{</span>
|
|
</span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="w"> </span><span class="nt">"type"</span><span class="p">:</span><span class="w"> </span><span class="s2">"log"</span><span class="p">,</span>
|
|
</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="w"> </span><span class="nt">"data"</span><span class="p">:</span><span class="w"> </span><span class="s2">"pause_detected"</span><span class="w"> </span><span class="err">|</span><span class="w"> </span><span class="s2">"response_starting"</span><span class="w"> </span><span class="err">|</span><span class="w"> </span><span class="s2">"started_talking"</span>
|
|
</span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="p">}</span>
|
|
</span></code></pre></div>
|
|
<div class="admonition tip">
|
|
<p class="admonition-title">Tip</p>
|
|
<p>When using WebRTC, the messages will be encoded as strings, so parse as JSON before using.</p>
|
|
</div>
|
|
<h3 id="additional-inputs">Additional Inputs</h3>
|
|
<p>When the <code>send_input</code> message is received, update the inputs of your handler however you like by using the <code>set_input</code> method of the <code>Stream</code> object.</p>
|
|
<p>A common pattern is to use a <code>POST</code> request to send the updated data. The first argument to the <code>set_input</code> method is the <code>webrtc_id</code> of the handler.</p>
|
|
<div class="language-python highlight"><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">pydantic</span><span class="w"> </span><span class="kn">import</span> <span class="n">BaseModel</span><span class="p">,</span> <span class="n">Field</span>
|
|
</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a>
|
|
</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="k">class</span><span class="w"> </span><span class="nc">InputData</span><span class="p">(</span><span class="n">BaseModel</span><span class="p">):</span>
|
|
</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a> <span class="n">webrtc_id</span><span class="p">:</span> <span class="nb">str</span>
|
|
</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a> <span class="n">conf_threshold</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">ge</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">le</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
|
</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a>
|
|
</span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a>
|
|
</span><span id="__span-2-8"><a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a><span class="nd">@app</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="s2">"/input_hook"</span><span class="p">)</span>
|
|
</span><span id="__span-2-9"><a id="__codelineno-2-9" name="__codelineno-2-9" href="#__codelineno-2-9"></a><span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">_</span><span class="p">(</span><span class="n">data</span><span class="p">:</span> <span class="n">InputData</span><span class="p">):</span>
|
|
</span><span id="__span-2-10"><a id="__codelineno-2-10" name="__codelineno-2-10" href="#__codelineno-2-10"></a> <span class="n">stream</span><span class="o">.</span><span class="n">set_input</span><span class="p">(</span><span class="n">data</span><span class="o">.</span><span class="n">webrtc_id</span><span class="p">,</span> <span class="n">data</span><span class="o">.</span><span class="n">conf_threshold</span><span class="p">)</span>
|
|
</span></code></pre></div>
|
|
<p>The updated data will be passed to the handler on the <strong>next</strong> call.</p>
|
|
<h3 id="additional-outputs">Additional Outputs</h3>
|
|
<p>The <code>fetch_output</code> message is sent to the client whenever an instance of <a href="../streams/#additional-outputs"><code>AdditionalOutputs</code></a> is available. You can access the latest output data by calling the <code>fetch_latest_output</code> method of the <code>Stream</code> object. </p>
|
|
<p>However, rather than fetching each output manually, a common pattern is to fetch the entire stream of output data by calling the <code>output_stream</code> method.</p>
|
|
<p>Here is an example:
|
|
<div class="language-python highlight"><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">fastapi.responses</span><span class="w"> </span><span class="kn">import</span> <span class="n">StreamingResponse</span>
|
|
</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a>
|
|
</span><span id="__span-3-3"><a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a><span class="nd">@app</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"/updates"</span><span class="p">)</span>
|
|
</span><span id="__span-3-4"><a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a><span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">stream_updates</span><span class="p">(</span><span class="n">webrtc_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
|
|
</span><span id="__span-3-5"><a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a> <span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">output_stream</span><span class="p">():</span>
|
|
</span><span id="__span-3-6"><a id="__codelineno-3-6" name="__codelineno-3-6" href="#__codelineno-3-6"></a> <span class="k">async</span> <span class="k">for</span> <span class="n">output</span> <span class="ow">in</span> <span class="n">stream</span><span class="o">.</span><span class="n">output_stream</span><span class="p">(</span><span class="n">webrtc_id</span><span class="p">):</span>
|
|
</span><span id="__span-3-7"><a id="__codelineno-3-7" name="__codelineno-3-7" href="#__codelineno-3-7"></a> <span class="c1"># Output is the AdditionalOutputs instance</span>
|
|
</span><span id="__span-3-8"><a id="__codelineno-3-8" name="__codelineno-3-8" href="#__codelineno-3-8"></a> <span class="c1"># Be sure to serialize it however you would like</span>
|
|
</span><span id="__span-3-9"><a id="__codelineno-3-9" name="__codelineno-3-9" href="#__codelineno-3-9"></a> <span class="k">yield</span> <span class="sa">f</span><span class="s2">"data: </span><span class="si">{</span><span class="n">output</span><span class="o">.</span><span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="si">}</span><span class="se">\n\n</span><span class="s2">"</span>
|
|
</span><span id="__span-3-10"><a id="__codelineno-3-10" name="__codelineno-3-10" href="#__codelineno-3-10"></a>
|
|
</span><span id="__span-3-11"><a id="__codelineno-3-11" name="__codelineno-3-11" href="#__codelineno-3-11"></a> <span class="k">return</span> <span class="n">StreamingResponse</span><span class="p">(</span>
|
|
</span><span id="__span-3-12"><a id="__codelineno-3-12" name="__codelineno-3-12" href="#__codelineno-3-12"></a> <span class="n">output_stream</span><span class="p">(),</span>
|
|
</span><span id="__span-3-13"><a id="__codelineno-3-13" name="__codelineno-3-13" href="#__codelineno-3-13"></a> <span class="n">media_type</span><span class="o">=</span><span class="s2">"text/event-stream"</span>
|
|
</span><span id="__span-3-14"><a id="__codelineno-3-14" name="__codelineno-3-14" href="#__codelineno-3-14"></a> <span class="p">)</span>
|
|
</span></code></pre></div></p>
|
|
<h3 id="handling-errors">Handling Errors</h3>
|
|
<p>When connecting via <code>WebRTC</code>, the server will respond to the <code>/webrtc/offer</code> route with a JSON response. If there are too many connections, the server will respond with a 200 error.</p>
|
|
<div class="language-json highlight"><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a><span class="p">{</span>
|
|
</span><span id="__span-4-2"><a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a><span class="w"> </span><span class="nt">"status"</span><span class="p">:</span><span class="w"> </span><span class="s2">"failed"</span><span class="p">,</span>
|
|
</span><span id="__span-4-3"><a id="__codelineno-4-3" name="__codelineno-4-3" href="#__codelineno-4-3"></a><span class="w"> </span><span class="nt">"meta"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
|
</span><span id="__span-4-4"><a id="__codelineno-4-4" name="__codelineno-4-4" href="#__codelineno-4-4"></a><span class="w"> </span><span class="nt">"error"</span><span class="p">:</span><span class="w"> </span><span class="s2">"concurrency_limit_reached"</span><span class="p">,</span>
|
|
</span><span id="__span-4-5"><a id="__codelineno-4-5" name="__codelineno-4-5" href="#__codelineno-4-5"></a><span class="w"> </span><span class="nt">"limit"</span><span class="p">:</span><span class="w"> </span><span class="mi">10</span>
|
|
</span><span id="__span-4-6"><a id="__codelineno-4-6" name="__codelineno-4-6" href="#__codelineno-4-6"></a><span class="w"> </span><span class="p">}</span>
|
|
</span></code></pre></div>
|
|
<p>Over <code>WebSocket</code>, the server will send the same message before closing the connection.</p>
|
|
<div class="admonition tip">
|
|
<p class="admonition-title">Tip</p>
|
|
<p>The server will sends a 200 status code because otherwise the gradio client will not be able to process the json response and display the error.</p>
|
|
</div>
|
|
<style>
|
|
.config-selector {
|
|
margin: 1em 0;
|
|
display: flex;
|
|
gap: 2em;
|
|
}
|
|
|
|
.select-group {
|
|
display: flex;
|
|
flex-direction: column;
|
|
gap: 0.5em;
|
|
}
|
|
|
|
.select-group label {
|
|
font-size: 0.8em;
|
|
font-weight: 600;
|
|
color: var(--md-default-fg-color--light);
|
|
}
|
|
|
|
.select-group select {
|
|
padding: 0.5em;
|
|
border: 1px solid var(--md-default-fg-color--lighter);
|
|
border-radius: 4px;
|
|
background-color: var(--md-code-bg-color);
|
|
color: var(--md-code-fg-color);
|
|
width: 150px;
|
|
font-size: 0.9em;
|
|
}
|
|
|
|
/* Style code blocks to match site theme */
|
|
.rendered-content pre {
|
|
background-color: var(--md-code-bg-color) !important;
|
|
color: var(--md-code-fg-color) !important;
|
|
padding: 1em;
|
|
border-radius: 4px;
|
|
}
|
|
|
|
.rendered-content code {
|
|
font-family: var(--md-code-font-family);
|
|
background-color: var(--md-code-bg-color) !important;
|
|
color: var(--md-code-fg-color) !important;
|
|
}
|
|
</style>
|
|
|
|
<script>
|
|
|
|
// doT.js
|
|
// 2011-2014, Laura Doktorova, https://github.com/olado/doT
|
|
// Licensed under the MIT license.
|
|
|
|
|
|
var doT = {
|
|
name: "doT",
|
|
version: "1.1.1",
|
|
templateSettings: {
|
|
evaluate: /\{\{([\s\S]+?(\}?)+)\}\}/g,
|
|
interpolate: /\{\{=([\s\S]+?)\}\}/g,
|
|
encode: /\{\{!([\s\S]+?)\}\}/g,
|
|
use: /\{\{#([\s\S]+?)\}\}/g,
|
|
useParams: /(^|[^\w$])def(?:\.|\[[\'\"])([\w$\.]+)(?:[\'\"]\])?\s*\:\s*([\w$\.]+|\"[^\"]+\"|\'[^\']+\'|\{[^\}]+\})/g,
|
|
define: /\{\{##\s*([\w\.$]+)\s*(\:|=)([\s\S]+?)#\}\}/g,
|
|
defineParams: /^\s*([\w$]+):([\s\S]+)/,
|
|
conditional: /\{\{\?(\?)?\s*([\s\S]*?)\s*\}\}/g,
|
|
iterate: /\{\{~\s*(?:\}\}|([\s\S]+?)\s*\:\s*([\w$]+)\s*(?:\:\s*([\w$]+))?\s*\}\})/g,
|
|
varname: "it",
|
|
strip: false,
|
|
append: true,
|
|
selfcontained: false,
|
|
doNotSkipEncoded: false
|
|
},
|
|
template: undefined, //fn, compile template
|
|
compile: undefined, //fn, for express
|
|
log: true
|
|
}, _globals;
|
|
|
|
doT.encodeHTMLSource = function (doNotSkipEncoded) {
|
|
var encodeHTMLRules = { "&": "&", "<": "<", ">": ">", '"': """, "'": "'", "/": "/" },
|
|
matchHTML = doNotSkipEncoded ? /[&<>"'\/]/g : /&(?!#?\w+;)|<|>|"|'|\//g;
|
|
return function (code) {
|
|
return code ? code.toString().replace(matchHTML, function (m) { return encodeHTMLRules[m] || m; }) : "";
|
|
};
|
|
};
|
|
|
|
_globals = (function () { return this || (0, eval)("this"); }());
|
|
|
|
/* istanbul ignore else */
|
|
if (typeof module !== "undefined" && module.exports) {
|
|
module.exports = doT;
|
|
} else if (typeof define === "function" && define.amd) {
|
|
define(function () { return doT; });
|
|
} else {
|
|
_globals.doT = doT;
|
|
}
|
|
|
|
var startend = {
|
|
append: { start: "'+(", end: ")+'", startencode: "'+encodeHTML(" },
|
|
split: { start: "';out+=(", end: ");out+='", startencode: "';out+=encodeHTML(" }
|
|
}, skip = /$^/;
|
|
|
|
function resolveDefs(c, block, def) {
|
|
return ((typeof block === "string") ? block : block.toString())
|
|
.replace(c.define || skip, function (m, code, assign, value) {
|
|
if (code.indexOf("def.") === 0) {
|
|
code = code.substring(4);
|
|
}
|
|
if (!(code in def)) {
|
|
if (assign === ":") {
|
|
if (c.defineParams) value.replace(c.defineParams, function (m, param, v) {
|
|
def[code] = { arg: param, text: v };
|
|
});
|
|
if (!(code in def)) def[code] = value;
|
|
} else {
|
|
new Function("def", "def['" + code + "']=" + value)(def);
|
|
}
|
|
}
|
|
return "";
|
|
})
|
|
.replace(c.use || skip, function (m, code) {
|
|
if (c.useParams) code = code.replace(c.useParams, function (m, s, d, param) {
|
|
if (def[d] && def[d].arg && param) {
|
|
var rw = (d + ":" + param).replace(/'|\\/g, "_");
|
|
def.__exp = def.__exp || {};
|
|
def.__exp[rw] = def[d].text.replace(new RegExp("(^|[^\\w$])" + def[d].arg + "([^\\w$])", "g"), "$1" + param + "$2");
|
|
return s + "def.__exp['" + rw + "']";
|
|
}
|
|
});
|
|
var v = new Function("def", "return " + code)(def);
|
|
return v ? resolveDefs(c, v, def) : v;
|
|
});
|
|
}
|
|
|
|
function unescape(code) {
|
|
return code.replace(/\\('|\\)/g, "$1").replace(/[\r\t\n]/g, " ");
|
|
}
|
|
|
|
doT.template = function (tmpl, c, def) {
|
|
c = c || doT.templateSettings;
|
|
var cse = c.append ? startend.append : startend.split, needhtmlencode, sid = 0, indv,
|
|
str = (c.use || c.define) ? resolveDefs(c, tmpl, def || {}) : tmpl;
|
|
|
|
str = ("var out='" + (c.strip ? str.replace(/(^|\r|\n)\t* +| +\t*(\r|\n|$)/g, " ")
|
|
.replace(/\r|\n|\t|\/\*[\s\S]*?\*\//g, "") : str)
|
|
.replace(/'|\\/g, "\\$&")
|
|
.replace(c.interpolate || skip, function (m, code) {
|
|
return cse.start + unescape(code) + cse.end;
|
|
})
|
|
.replace(c.encode || skip, function (m, code) {
|
|
needhtmlencode = true;
|
|
return cse.startencode + unescape(code) + cse.end;
|
|
})
|
|
.replace(c.conditional || skip, function (m, elsecase, code) {
|
|
return elsecase ?
|
|
(code ? "';}else if(" + unescape(code) + "){out+='" : "';}else{out+='") :
|
|
(code ? "';if(" + unescape(code) + "){out+='" : "';}out+='");
|
|
})
|
|
.replace(c.iterate || skip, function (m, iterate, vname, iname) {
|
|
if (!iterate) return "';} } out+='";
|
|
sid += 1; indv = iname || "i" + sid; iterate = unescape(iterate);
|
|
return "';var arr" + sid + "=" + iterate + ";if(arr" + sid + "){var " + vname + "," + indv + "=-1,l" + sid + "=arr" + sid + ".length-1;while(" + indv + "<l" + sid + "){"
|
|
+ vname + "=arr" + sid + "[" + indv + "+=1];out+='";
|
|
})
|
|
.replace(c.evaluate || skip, function (m, code) {
|
|
return "';" + unescape(code) + "out+='";
|
|
})
|
|
+ "';return out;")
|
|
.replace(/\n/g, "\\n").replace(/\t/g, '\\t').replace(/\r/g, "\\r")
|
|
.replace(/(\s|;|\}|^|\{)out\+='';/g, '$1').replace(/\+''/g, "");
|
|
//.replace(/(\s|;|\}|^|\{)out\+=''\+/g,'$1out+=');
|
|
|
|
if (needhtmlencode) {
|
|
if (!c.selfcontained && _globals && !_globals._encodeHTML) _globals._encodeHTML = doT.encodeHTMLSource(c.doNotSkipEncoded);
|
|
str = "var encodeHTML = typeof _encodeHTML !== 'undefined' ? _encodeHTML : ("
|
|
+ doT.encodeHTMLSource.toString() + "(" + (c.doNotSkipEncoded || '') + "));"
|
|
+ str;
|
|
}
|
|
try {
|
|
return new Function(c.varname, str);
|
|
} catch (e) {
|
|
/* istanbul ignore else */
|
|
if (typeof console !== "undefined") console.log("Could not create a template function: " + str);
|
|
throw e;
|
|
}
|
|
};
|
|
|
|
doT.compile = function (tmpl, def) {
|
|
return doT.template(tmpl, null, def);
|
|
};
|
|
|
|
// WebRTC template
|
|
|
|
const webrtcTemplate = doT.template(`
|
|
To connect to the server, you need to create a new RTCPeerConnection object and call the \`setupWebRTC\` function below.
|
|
{{? it.mode === "send-receive" || it.mode === "receive" }}
|
|
This code snippet assumes there is an html element with an id of \`{{=it.modality}}_output_component_id\` where the output will be displayed. It should be {{? it.modality === "audio"}}a \`<audio>\`{{??}}an \`<video>\`{{?}} element.
|
|
{{?}}
|
|
|
|
\`\`\`javascript
|
|
// pass any rtc_configuration params here
|
|
const pc = new RTCPeerConnection();
|
|
{{? it.mode === "send-receive" || it.mode === "receive" }}
|
|
const {{=it.modality}}_output_component = document.getElementById("{{=it.modality}}_output_component_id");
|
|
{{?}}
|
|
async function setupWebRTC(peerConnection) {
|
|
{{? it.mode === "send-receive" || it.mode === "send" }}
|
|
// Get {{=it.modality}} stream from webcam
|
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
{{=it.modality}}: true,
|
|
})
|
|
{{?}}
|
|
{{? it.mode === "send-receive" }}
|
|
// Send {{=it.modality}} stream to server
|
|
stream.getTracks().forEach(async (track) => {
|
|
const sender = pc.addTrack(track, stream);
|
|
})
|
|
{{?? it.mode === "send" }}
|
|
// Receive {{=it.modality}} stream from server
|
|
pc.addTransceiver({{=it.modality}}, { direction: "recvonly" })
|
|
{{?}}
|
|
{{? it.mode === "send-receive" || it.mode === "receive" }}
|
|
peerConnection.addEventListener("track", (evt) => {
|
|
if ({{=it.modality}}_output_component &&
|
|
{{=it.modality}}_output_component.srcObject !== evt.streams[0]) {
|
|
{{=it.modality}}_output_component.srcObject = evt.streams[0];
|
|
}
|
|
});
|
|
{{?}}
|
|
// Create data channel (needed!)
|
|
const dataChannel = peerConnection.createDataChannel("text");
|
|
|
|
// Create and send offer
|
|
const offer = await peerConnection.createOffer();
|
|
await peerConnection.setLocalDescription(offer);
|
|
|
|
let webrtc_id = Math.random().toString(36).substring(7)
|
|
|
|
// Send ICE candidates to server
|
|
// (especially needed when server is behind firewall)
|
|
peerConnection.onicecandidate = ({ candidate }) => {
|
|
if (candidate) {
|
|
console.debug("Sending ICE candidate", candidate);
|
|
fetch('/webrtc/offer', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
candidate: candidate.toJSON(),
|
|
webrtc_id: webrtc_id,
|
|
type: "ice-candidate",
|
|
})
|
|
})
|
|
}
|
|
};
|
|
|
|
// Send offer to server
|
|
const response = await fetch('/webrtc/offer', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
sdp: offer.sdp,
|
|
type: offer.type,
|
|
webrtc_id: webrtc_id
|
|
})
|
|
});
|
|
|
|
// Handle server response
|
|
const serverResponse = await response.json();
|
|
await peerConnection.setRemoteDescription(serverResponse);
|
|
}
|
|
\`\`\`
|
|
`);
|
|
|
|
// WebSocket template
|
|
const wsTemplate = doT.template(`
|
|
{{? it.modality !== "audio" || it.mode !== "send-receive" }}
|
|
WebSocket connections are currently only supported for audio in send-receive mode.
|
|
{{??}}
|
|
|
|
To connect to the server via WebSocket, you'll need to establish a WebSocket connection and handle audio processing. The code below assumes there is an HTML audio element for output playback.
|
|
|
|
The input audio must be mu-law encoded with a sample rate equal to the input_sample_rate of the handler you are connecting to. By default it is 48k Hz.
|
|
The out audio will also be mulaw encoded and the sample rate will be equal to the output_sample_rate of the handler. By default it is 48k Hz.
|
|
|
|
\`\`\`javascript
|
|
// Setup audio context and stream
|
|
const audioContext = new AudioContext();
|
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
audio: true
|
|
});
|
|
|
|
// Create WebSocket connection
|
|
const ws = new WebSocket(\`\${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//$\{window.location.host}/websocket/offer\`);
|
|
|
|
ws.onopen = () => {
|
|
// Send initial start message with unique ID
|
|
ws.send(JSON.stringify({
|
|
event: "start",
|
|
websocket_id: generateId() // Implement your own ID generator
|
|
}));
|
|
|
|
// Setup audio processing
|
|
const source = audioContext.createMediaStreamSource(stream);
|
|
const processor = audioContext.createScriptProcessor(2048, 1, 1);
|
|
source.connect(processor);
|
|
processor.connect(audioContext.destination);
|
|
|
|
processor.onaudioprocess = (e) => {
|
|
const inputData = e.inputBuffer.getChannelData(0);
|
|
const mulawData = convertToMulaw(inputData, audioContext.sampleRate);
|
|
const base64Audio = btoa(String.fromCharCode.apply(null, mulawData));
|
|
|
|
if (ws.readyState === WebSocket.OPEN) {
|
|
ws.send(JSON.stringify({
|
|
event: "media",
|
|
media: {
|
|
payload: base64Audio
|
|
}
|
|
}));
|
|
}
|
|
};
|
|
};
|
|
|
|
ws.onmessage = (event) => {
|
|
const data = JSON.parse(event.data);
|
|
if (data?.type === "send_input") {
|
|
fetch('/input_hook', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
// Send additional input data here
|
|
body: JSON.stringify({ webrtc_id: wsId })
|
|
});
|
|
}
|
|
if (data.event === "media") {
|
|
// Process received audio
|
|
const audioData = atob(data.media.payload);
|
|
const mulawData = new Uint8Array(audioData.length);
|
|
for (let i = 0; i < audioData.length; i++) {
|
|
mulawData[i] = audioData.charCodeAt(i);
|
|
}
|
|
|
|
// Convert mu-law to linear PCM
|
|
const linearData = alawmulaw.mulaw.decode(mulawData);
|
|
|
|
// Create an AudioBuffer
|
|
const audioBuffer = outputContext.createBuffer(1, linearData.length, sampleRate);
|
|
const channelData = audioBuffer.getChannelData(0);
|
|
|
|
// Fill the buffer with the decoded data
|
|
for (let i = 0; i < linearData.length; i++) {
|
|
channelData[i] = linearData[i] / 32768.0;
|
|
}
|
|
|
|
// Do something with Audio Buffer
|
|
}
|
|
};
|
|
\`\`\`
|
|
{{?}}
|
|
`);
|
|
|
|
function updateDocs() {
|
|
// Get selected values
|
|
const modality = document.getElementById('modality').value;
|
|
const mode = document.getElementById('mode').value;
|
|
const connection = document.getElementById('connection').value;
|
|
|
|
// Context for templates
|
|
const context = {
|
|
modality: modality,
|
|
mode: mode,
|
|
additional_inputs: true,
|
|
additional_outputs: true
|
|
};
|
|
|
|
// Choose template based on connection type
|
|
const template = connection === 'webrtc' ? webrtcTemplate : wsTemplate;
|
|
|
|
// Render docs with syntax highlighting
|
|
const html = template(context);
|
|
const docsDiv = document.getElementById('docs');
|
|
docsDiv.innerHTML = marked.parse(html);
|
|
docsDiv.className = 'rendered-content';
|
|
|
|
// Initialize any code blocks that were just added
|
|
document.querySelectorAll('pre code').forEach((block) => {
|
|
hljs.highlightElement(block);
|
|
});
|
|
}
|
|
|
|
// Initial render
|
|
document.addEventListener('DOMContentLoaded', updateDocs);
|
|
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</article>
|
|
</div>
|
|
|
|
|
|
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
|
</div>
|
|
|
|
</main>
|
|
|
|
<footer class="md-footer">
|
|
|
|
<div class="md-footer-meta md-typeset">
|
|
<div class="md-footer-meta__inner md-grid">
|
|
<div class="md-copyright">
|
|
|
|
|
|
Made with
|
|
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
|
Material for MkDocs
|
|
</a>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
<div class="md-dialog" data-md-component="dialog">
|
|
<div class="md-dialog__inner md-typeset"></div>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<script id="__config" type="application/json">{"base": "../..", "features": ["content.code.copy", "content.code.annotate", "navigation.indexes"], "search": "../../assets/javascripts/workers/search.d50fe291.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
|
|
|
|
|
|
<script src="../../assets/javascripts/bundle.13a4f30d.min.js"></script>
|
|
|
|
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
|
|
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
|
|
|
|
|
|
</body>
|
|
</html> |