mirror of
https://github.com/HumanAIGC-Engineering/gradio-webrtc.git
synced 2026-02-05 01:49:23 +08:00
853d6a06b5
* Add code * add code * add code * Rename messages * rename * add code * Add demo * docs + demos + bug fixes * add code * styles * user guide * Styles * Add code * misc docs updates * print nit * whisper + pr * url for images * whsiper update * Fix bugs * remove demo files * version number * Fix pypi readme * Fix * demos * Add llama code editor * Update llama code editor and object detection cookbook * Add more cookbook demos * add code * Fix links for PR deploys * add code * Fix the install * add tts * TTS docs * Typo * Pending bubbles for reply on pause * Stream redesign (#63) * better error handling * Websocket error handling * add code --------- Co-authored-by: Freddy Boulton <freddyboulton@hf-freddy.local> * remove docs from dist * Some docs typos * more typos * upload changes + docs * docs * better phone * update docs * add code * Make demos better * fix docs + websocket start_up * remove mention of FastAPI app * fastphone tweaks * add code * ReplyOnStopWord fixes * Fix cookbook * Fix pypi readme * add code * bump versions * sambanova cookbook * Fix tags * Llm voice chat * kyutai tag * Add error message to all index.html * STT module uses Moonshine * Not required from typing extensions * fix llm voice chat * Add vpn warning * demo fixes * demos * Add more ui args and gemini audio-video * update cookbook * version 9 --------- Co-authored-by: Freddy Boulton <freddyboulton@hf-freddy.local>
486 lines
15 KiB
HTML
486 lines
15 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Hello Computer 💻</title>
|
|
<style>
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
|
background-color: #f8f9fa;
|
|
color: #1a1a1a;
|
|
margin: 0;
|
|
padding: 20px;
|
|
height: 100vh;
|
|
box-sizing: border-box;
|
|
}
|
|
|
|
.container {
|
|
max-width: 800px;
|
|
margin: 0 auto;
|
|
height: calc(100% - 100px);
|
|
}
|
|
|
|
.logo {
|
|
text-align: center;
|
|
margin-bottom: 40px;
|
|
}
|
|
|
|
.chat-container {
|
|
background: white;
|
|
border-radius: 8px;
|
|
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
|
|
padding: 20px;
|
|
height: 90%;
|
|
box-sizing: border-box;
|
|
display: flex;
|
|
flex-direction: column;
|
|
}
|
|
|
|
.chat-messages {
|
|
flex-grow: 1;
|
|
overflow-y: auto;
|
|
margin-bottom: 20px;
|
|
padding: 10px;
|
|
}
|
|
|
|
.message {
|
|
margin-bottom: 20px;
|
|
padding: 12px;
|
|
border-radius: 8px;
|
|
font-size: 14px;
|
|
line-height: 1.5;
|
|
}
|
|
|
|
.message.user {
|
|
background-color: #e9ecef;
|
|
margin-left: 20%;
|
|
}
|
|
|
|
.message.assistant {
|
|
background-color: #f1f3f5;
|
|
margin-right: 20%;
|
|
}
|
|
|
|
.controls {
|
|
text-align: center;
|
|
margin-top: 20px;
|
|
}
|
|
|
|
button {
|
|
background-color: #0066cc;
|
|
color: white;
|
|
border: none;
|
|
padding: 12px 24px;
|
|
font-family: inherit;
|
|
font-size: 14px;
|
|
cursor: pointer;
|
|
transition: all 0.3s;
|
|
border-radius: 4px;
|
|
font-weight: 500;
|
|
}
|
|
|
|
button:hover {
|
|
background-color: #0052a3;
|
|
}
|
|
|
|
#audio-output {
|
|
display: none;
|
|
}
|
|
|
|
.icon-with-spinner {
|
|
display: flex;
|
|
align-items: center;
|
|
justify-content: center;
|
|
gap: 12px;
|
|
min-width: 180px;
|
|
}
|
|
|
|
.spinner {
|
|
width: 20px;
|
|
height: 20px;
|
|
border: 2px solid #ffffff;
|
|
border-top-color: transparent;
|
|
border-radius: 50%;
|
|
animation: spin 1s linear infinite;
|
|
flex-shrink: 0;
|
|
}
|
|
|
|
@keyframes spin {
|
|
to {
|
|
transform: rotate(360deg);
|
|
}
|
|
}
|
|
|
|
.pulse-container {
|
|
display: flex;
|
|
align-items: center;
|
|
justify-content: center;
|
|
gap: 12px;
|
|
min-width: 180px;
|
|
}
|
|
|
|
.pulse-circle {
|
|
width: 20px;
|
|
height: 20px;
|
|
border-radius: 50%;
|
|
background-color: #ffffff;
|
|
opacity: 0.2;
|
|
flex-shrink: 0;
|
|
transform: translateX(-0%) scale(var(--audio-level, 1));
|
|
transition: transform 0.1s ease;
|
|
}
|
|
|
|
/* Add styles for typing indicator */
|
|
.typing-indicator {
|
|
padding: 8px;
|
|
background-color: #f1f3f5;
|
|
border-radius: 8px;
|
|
margin-bottom: 10px;
|
|
display: none;
|
|
}
|
|
|
|
.dots {
|
|
display: inline-flex;
|
|
gap: 4px;
|
|
}
|
|
|
|
.dot {
|
|
width: 8px;
|
|
height: 8px;
|
|
background-color: #0066cc;
|
|
border-radius: 50%;
|
|
animation: pulse 1.5s infinite;
|
|
opacity: 0.5;
|
|
}
|
|
|
|
.dot:nth-child(2) {
|
|
animation-delay: 0.5s;
|
|
}
|
|
|
|
.dot:nth-child(3) {
|
|
animation-delay: 1s;
|
|
}
|
|
|
|
@keyframes pulse {
|
|
|
|
0%,
|
|
100% {
|
|
opacity: 0.5;
|
|
transform: scale(1);
|
|
}
|
|
|
|
50% {
|
|
opacity: 1;
|
|
transform: scale(1.2);
|
|
}
|
|
}
|
|
|
|
/* Add styles for toast notifications */
|
|
.toast {
|
|
position: fixed;
|
|
top: 20px;
|
|
left: 50%;
|
|
transform: translateX(-50%);
|
|
padding: 16px 24px;
|
|
border-radius: 4px;
|
|
font-size: 14px;
|
|
z-index: 1000;
|
|
display: none;
|
|
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2);
|
|
}
|
|
|
|
.toast.error {
|
|
background-color: #f44336;
|
|
color: white;
|
|
}
|
|
|
|
.toast.warning {
|
|
background-color: #ffd700;
|
|
color: black;
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<body>
|
|
<!-- Add toast element after body opening tag -->
|
|
<div id="error-toast" class="toast"></div>
|
|
<div class="container">
|
|
<div class="logo">
|
|
<h1>Hello Computer 💻</h1>
|
|
<h2 style="font-size: 1.2em; color: #666; margin-top: 10px;">Say 'Computer' before asking your question</h2>
|
|
</div>
|
|
<div class="chat-container">
|
|
<div class="chat-messages" id="chat-messages"></div>
|
|
<div class="typing-indicator" id="typing-indicator">
|
|
<div class="dots">
|
|
<div class="dot"></div>
|
|
<div class="dot"></div>
|
|
<div class="dot"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="controls">
|
|
<button id="start-button">Start Conversation</button>
|
|
</div>
|
|
</div>
|
|
<audio id="audio-output"></audio>
|
|
|
|
<script>
|
|
let peerConnection;
|
|
let webrtc_id;
|
|
const startButton = document.getElementById('start-button');
|
|
const chatMessages = document.getElementById('chat-messages');
|
|
|
|
let audioLevel = 0;
|
|
let animationFrame;
|
|
let audioContext, analyser, audioSource;
|
|
let messages = [];
|
|
let eventSource;
|
|
|
|
function updateButtonState() {
|
|
const button = document.getElementById('start-button');
|
|
if (peerConnection && (peerConnection.connectionState === 'connecting' || peerConnection.connectionState === 'new')) {
|
|
button.innerHTML = `
|
|
<div class="icon-with-spinner">
|
|
<div class="spinner"></div>
|
|
<span>Connecting...</span>
|
|
</div>
|
|
`;
|
|
} else if (peerConnection && peerConnection.connectionState === 'connected') {
|
|
button.innerHTML = `
|
|
<div class="pulse-container">
|
|
<div class="pulse-circle"></div>
|
|
<span>Stop Conversation</span>
|
|
</div>
|
|
`;
|
|
} else {
|
|
button.innerHTML = 'Start Conversation';
|
|
}
|
|
}
|
|
|
|
function setupAudioVisualization(stream) {
|
|
audioContext = new (window.AudioContext || window.webkitAudioContext)();
|
|
analyser = audioContext.createAnalyser();
|
|
audioSource = audioContext.createMediaStreamSource(stream);
|
|
audioSource.connect(analyser);
|
|
analyser.fftSize = 64;
|
|
const dataArray = new Uint8Array(analyser.frequencyBinCount);
|
|
|
|
function updateAudioLevel() {
|
|
analyser.getByteFrequencyData(dataArray);
|
|
const average = Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
|
|
audioLevel = average / 255;
|
|
|
|
const pulseCircle = document.querySelector('.pulse-circle');
|
|
if (pulseCircle) {
|
|
pulseCircle.style.setProperty('--audio-level', 1 + audioLevel);
|
|
}
|
|
|
|
animationFrame = requestAnimationFrame(updateAudioLevel);
|
|
}
|
|
updateAudioLevel();
|
|
}
|
|
|
|
function showError(message) {
|
|
const toast = document.getElementById('error-toast');
|
|
toast.textContent = message;
|
|
toast.className = 'toast error';
|
|
toast.style.display = 'block';
|
|
|
|
// Hide toast after 5 seconds
|
|
setTimeout(() => {
|
|
toast.style.display = 'none';
|
|
}, 5000);
|
|
}
|
|
|
|
function handleMessage(event) {
|
|
const eventJson = JSON.parse(event.data);
|
|
const typingIndicator = document.getElementById('typing-indicator');
|
|
|
|
if (eventJson.type === "error") {
|
|
showError(eventJson.message);
|
|
} else if (eventJson.type === "send_input") {
|
|
fetch('/input_hook', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify({
|
|
webrtc_id: webrtc_id,
|
|
chatbot: messages,
|
|
state: messages
|
|
})
|
|
});
|
|
} else if (eventJson.type === "log") {
|
|
if (eventJson.data === "pause_detected") {
|
|
typingIndicator.style.display = 'block';
|
|
chatMessages.scrollTop = chatMessages.scrollHeight;
|
|
} else if (eventJson.data === "response_starting") {
|
|
typingIndicator.style.display = 'none';
|
|
}
|
|
}
|
|
}
|
|
|
|
async function setupWebRTC() {
|
|
const config = __RTC_CONFIGURATION__;
|
|
peerConnection = new RTCPeerConnection(config);
|
|
|
|
const timeoutId = setTimeout(() => {
|
|
const toast = document.getElementById('error-toast');
|
|
toast.textContent = "Connection is taking longer than usual. Are you on a VPN?";
|
|
toast.className = 'toast warning';
|
|
toast.style.display = 'block';
|
|
|
|
// Hide warning after 5 seconds
|
|
setTimeout(() => {
|
|
toast.style.display = 'none';
|
|
}, 5000);
|
|
}, 5000);
|
|
|
|
try {
|
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
audio: true
|
|
});
|
|
|
|
setupAudioVisualization(stream);
|
|
|
|
stream.getTracks().forEach(track => {
|
|
peerConnection.addTrack(track, stream);
|
|
});
|
|
|
|
const dataChannel = peerConnection.createDataChannel('text');
|
|
dataChannel.onmessage = handleMessage;
|
|
|
|
const offer = await peerConnection.createOffer();
|
|
await peerConnection.setLocalDescription(offer);
|
|
|
|
await new Promise((resolve) => {
|
|
if (peerConnection.iceGatheringState === "complete") {
|
|
resolve();
|
|
} else {
|
|
const checkState = () => {
|
|
if (peerConnection.iceGatheringState === "complete") {
|
|
peerConnection.removeEventListener("icegatheringstatechange", checkState);
|
|
resolve();
|
|
}
|
|
};
|
|
peerConnection.addEventListener("icegatheringstatechange", checkState);
|
|
}
|
|
});
|
|
|
|
peerConnection.addEventListener('connectionstatechange', () => {
|
|
console.log('connectionstatechange', peerConnection.connectionState);
|
|
if (peerConnection.connectionState === 'connected') {
|
|
clearTimeout(timeoutId);
|
|
const toast = document.getElementById('error-toast');
|
|
toast.style.display = 'none';
|
|
}
|
|
updateButtonState();
|
|
});
|
|
|
|
webrtc_id = Math.random().toString(36).substring(7);
|
|
|
|
const response = await fetch('/webrtc/offer', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
sdp: peerConnection.localDescription.sdp,
|
|
type: peerConnection.localDescription.type,
|
|
webrtc_id: webrtc_id
|
|
})
|
|
});
|
|
|
|
const serverResponse = await response.json();
|
|
|
|
if (serverResponse.status === 'failed') {
|
|
showError(serverResponse.meta.error === 'concurrency_limit_reached'
|
|
? `Too many connections. Maximum limit is ${serverResponse.meta.limit}`
|
|
: serverResponse.meta.error);
|
|
stop();
|
|
return;
|
|
}
|
|
|
|
await peerConnection.setRemoteDescription(serverResponse);
|
|
|
|
eventSource = new EventSource('/outputs?webrtc_id=' + webrtc_id);
|
|
eventSource.addEventListener("output", (event) => {
|
|
const eventJson = JSON.parse(event.data);
|
|
console.log(eventJson);
|
|
messages.push(eventJson.message);
|
|
addMessage(eventJson.message.role, eventJson.audio ?? eventJson.message.content);
|
|
});
|
|
} catch (err) {
|
|
clearTimeout(timeoutId);
|
|
console.error('Error setting up WebRTC:', err);
|
|
showError('Failed to establish connection. Please try again.');
|
|
stop();
|
|
}
|
|
}
|
|
|
|
function addMessage(role, content) {
|
|
const messageDiv = document.createElement('div');
|
|
messageDiv.classList.add('message', role);
|
|
|
|
if (role === 'user') {
|
|
// Create audio element for user messages
|
|
const audio = document.createElement('audio');
|
|
audio.controls = true;
|
|
audio.src = content;
|
|
messageDiv.appendChild(audio);
|
|
} else {
|
|
// Text content for assistant messages
|
|
messageDiv.textContent = content;
|
|
}
|
|
|
|
chatMessages.appendChild(messageDiv);
|
|
chatMessages.scrollTop = chatMessages.scrollHeight;
|
|
}
|
|
|
|
function stop() {
|
|
if (eventSource) {
|
|
eventSource.close();
|
|
eventSource = null;
|
|
}
|
|
|
|
if (animationFrame) {
|
|
cancelAnimationFrame(animationFrame);
|
|
}
|
|
if (audioContext) {
|
|
audioContext.close();
|
|
audioContext = null;
|
|
analyser = null;
|
|
audioSource = null;
|
|
}
|
|
if (peerConnection) {
|
|
if (peerConnection.getTransceivers) {
|
|
peerConnection.getTransceivers().forEach(transceiver => {
|
|
if (transceiver.stop) {
|
|
transceiver.stop();
|
|
}
|
|
});
|
|
}
|
|
|
|
if (peerConnection.getSenders) {
|
|
peerConnection.getSenders().forEach(sender => {
|
|
if (sender.track && sender.track.stop) sender.track.stop();
|
|
});
|
|
}
|
|
peerConnection.close();
|
|
}
|
|
updateButtonState();
|
|
audioLevel = 0;
|
|
}
|
|
|
|
startButton.addEventListener('click', () => {
|
|
if (!peerConnection || peerConnection.connectionState !== 'connected') {
|
|
setupWebRTC();
|
|
} else {
|
|
stop();
|
|
}
|
|
});
|
|
</script>
|
|
</body>
|
|
|
|
</html> |